<commit_before>package opengl

// #cgo LDFLAGS: -framework OpenGL
//
// #include <stdlib.h>
// #include <OpenGL/gl.h>
import "C"
import (
	"fmt"
	"github.com/hajimehoshi/go-ebiten/graphics"
	"github.com/hajimehoshi/go-ebiten/graphics/matrix"
	"github.com/hajimehoshi/go-ebiten/graphics/texture"
	"image"
	"math"
	"unsafe"
)

type Context struct {
	screenId               graphics.RenderTargetId
	screenWidth            int
	screenHeight           int
	screenScale            int
	textures               map[graphics.TextureId]*texture.Texture
	renderTargets          map[graphics.RenderTargetId]*RenderTarget
	renderTargetToTexture  map[graphics.RenderTargetId]graphics.TextureId
	currentOffscreen       *RenderTarget
	mainFramebufferTexture *RenderTarget
	projectionMatrix       [16]float32
}

func newContext(screenWidth, screenHeight, screenScale int) *Context {
	context := &Context{
		screenWidth:           screenWidth,
		screenHeight:          screenHeight,
		screenScale:           screenScale,
		textures:              map[graphics.TextureId]*texture.Texture{},
		renderTargets:         map[graphics.RenderTargetId]*RenderTarget{},
		renderTargetToTexture: map[graphics.RenderTargetId]graphics.TextureId{},
	}
	return context
}

func (context *Context) Init() {
	// The main framebuffer should be created sooner than any other
	// framebuffers!
	mainFramebuffer := C.GLint(0)
	C.glGetIntegerv(C.GL_FRAMEBUFFER_BINDING, &mainFramebuffer)

	var err error
	context.mainFramebufferTexture, err = newRenderTargetWithFramebuffer(
		context.screenWidth*context.screenScale,
		context.screenHeight*context.screenScale,
		C.GLuint(mainFramebuffer))
	if err != nil {
		panic("creating main framebuffer failed: " + err.Error())
	}

	initializeShaders()

	context.screenId, err = context.NewRenderTarget(
		context.screenWidth, context.screenHeight)
	if err != nil {
		panic("initializing the offscreen failed: " + err.Error())
	}
}

func (context *Context) ToTexture(renderTargetId graphics.RenderTargetId) graphics.TextureId {
	return context.renderTargetToTexture[renderTargetId]
}

func (context *Context) Clear() {
	context.Fill(0, 0, 0)
}

func (context *Context) Fill(r, g, b uint8) {
	const max = float64(math.MaxUint8)
	C.glClearColor(
		C.GLclampf(float64(r)/max),
		C.GLclampf(float64(g)/max),
		C.GLclampf(float64(b)/max),
		1)
	C.glClear(C.GL_COLOR_BUFFER_BIT)
}

func (context *Context) DrawTexture(
	textureId graphics.TextureId,
	geometryMatrix matrix.Geometry, colorMatrix matrix.Color) {
	texture, ok := context.textures[textureId]
	if !ok {
		panic("invalid texture ID")
	}
	// TODO: fix this
	source := graphics.Rect{0, 0, texture.Width(), texture.Height()}
	locations := []graphics.TexturePart{{0, 0, source}}
	context.DrawTextureParts(textureId, locations,
		geometryMatrix, colorMatrix)
}

func (context *Context) DrawTextureParts(
	textureId graphics.TextureId, parts []graphics.TexturePart,
	geometryMatrix matrix.Geometry, colorMatrix matrix.Color) {

	texture, ok := context.textures[textureId]
	if !ok {
		panic("invalid texture ID")
	}

	shaderProgram := context.setShaderProgram(geometryMatrix, colorMatrix)
	C.glBindTexture(C.GL_TEXTURE_2D, texture.Native().(C.GLuint))

	vertexAttrLocation := getAttributeLocation(shaderProgram, "vertex")
	textureAttrLocation := getAttributeLocation(shaderProgram, "texture")

	C.glEnableClientState(C.GL_VERTEX_ARRAY)
	C.glEnableClientState(C.GL_TEXTURE_COORD_ARRAY)
	C.glEnableVertexAttribArray(C.GLuint(vertexAttrLocation))
	C.glEnableVertexAttribArray(C.GLuint(textureAttrLocation))
	// TODO: Optimization
	for _, part := range parts {
		x1 := float32(part.LocationX)
		x2 := float32(part.LocationX + part.Source.Width)
		y1 := float32(part.LocationY)
		y2 := float32(part.LocationY + part.Source.Height)
		vertex := [...]float32{
			x1, y1,
			x2, y1,
			x1, y2,
			x2, y2,
		}

		src := part.Source
		tu1 := float32(texture.U(src.X))
		tu2 := float32(texture.U(src.X + src.Width))
		tv1 := float32(texture.V(src.Y))
		tv2 := float32(texture.V(src.Y + src.Height))
		texCoord := [...]float32{
			tu1, tv1,
			tu2, tv1,
			tu1, tv2,
			tu2, tv2,
		}
		C.glVertexAttribPointer(C.GLuint(vertexAttrLocation), 2,
			C.GL_FLOAT, C.GL_FALSE,
			0, unsafe.Pointer(&vertex[0]))
		C.glVertexAttribPointer(C.GLuint(textureAttrLocation), 2,
			C.GL_FLOAT, C.GL_FALSE,
			0, unsafe.Pointer(&texCoord[0]))
		C.glDrawArrays(C.GL_TRIANGLE_STRIP, 0, 4)
	}
	C.glDisableVertexAttribArray(C.GLuint(textureAttrLocation))
	C.glDisableVertexAttribArray(C.GLuint(vertexAttrLocation))
	C.glDisableClientState(C.GL_TEXTURE_COORD_ARRAY)
	C.glDisableClientState(C.GL_VERTEX_ARRAY)
}

func (context *Context) ResetOffscreen() {
	context.SetOffscreen(context.screenId)
}

func (context *Context) SetOffscreen(renderTargetId graphics.RenderTargetId) {
	renderTarget := context.renderTargets[renderTargetId]
	context.setOffscreen(renderTarget)
}

func (context *Context) setOffscreen(renderTarget *RenderTarget) {
	context.currentOffscreen = renderTarget

	C.glFlush()

	C.glBindFramebuffer(C.GL_FRAMEBUFFER, renderTarget.framebuffer)
	err := C.glCheckFramebufferStatus(C.GL_FRAMEBUFFER)
	if err != C.GL_FRAMEBUFFER_COMPLETE {
		panic(fmt.Sprintf("glBindFramebuffer failed: %d", err))
	}

	C.glEnable(C.GL_BLEND)
	C.glBlendFuncSeparate(C.GL_SRC_ALPHA, C.GL_ONE_MINUS_SRC_ALPHA,
		C.GL_ZERO, C.GL_ONE)

	context.currentOffscreen.SetAsViewport(context)
}

func (context *Context) SetViewport(x, y, width, height int) {
	C.glViewport(C.GLint(x), C.GLint(y), C.GLsizei(width), C.GLsizei(height))

	matrix := graphics.OrthoProjectionMatrix(x, width, y, height)
	if context.currentOffscreen == context.mainFramebufferTexture {
		// Flip Y and move to fit with the top of the window.
		matrix[1][1] *= -1
		actualHeight := context.screenHeight * context.screenScale
		matrix[1][3] += float64(actualHeight) / float64(height) * 2
	}

	for j := 0; j < 4; j++ {
		for i := 0; i < 4; i++ {
			context.projectionMatrix[i+j*4] = float32(matrix[i][j])
		}
	}

	// TODO: call 'setShaderProgram' here?
}

func (context *Context) setMainFramebufferOffscreen() {
	context.setOffscreen(context.mainFramebufferTexture)
}

func (context *Context) flush() {
	C.glFlush()
}

func (context *Context) setShaderProgram(
	geometryMatrix matrix.Geometry, colorMatrix matrix.Color) (program C.GLuint) {
	if colorMatrix.IsIdentity() {
		program = regularShaderProgram
	} else {
		program = colorMatrixShaderProgram
	}
	// TODO: cache and skip?
	C.glUseProgram(program)

	C.glUniformMatrix4fv(getUniformLocation(program, "projection_matrix"),
		1, C.GL_FALSE,
		(*C.GLfloat)(&context.projectionMatrix[0]))

	a := float32(geometryMatrix.Elements[0][0])
	b := float32(geometryMatrix.Elements[0][1])
	c := float32(geometryMatrix.Elements[1][0])
	d := float32(geometryMatrix.Elements[1][1])
	tx := float32(geometryMatrix.Elements[0][2])
	ty := float32(geometryMatrix.Elements[1][2])
	glModelviewMatrix := [...]float32{
		a, c, 0, 0,
		b, d, 0, 0,
		0, 0, 1, 0,
		tx, ty, 0, 1,
	}
	C.glUniformMatrix4fv(getUniformLocation(program, "modelview_matrix"),
		1, C.GL_FALSE,
		(*C.GLfloat)(&glModelviewMatrix[0]))

	C.glUniform1i(getUniformLocation(program, "texture"), 0)

	if program != colorMatrixShaderProgram {
		return
	}

	e := [4][5]float32{}
	for i := 0; i < 4; i++ {
		for j := 0; j < 5; j++ {
			e[i][j] = float32(colorMatrix.Elements[i][j])
		}
	}

	glColorMatrix := [...]float32{
		e[0][0], e[1][0], e[2][0], e[3][0],
		e[0][1], e[1][1], e[2][1], e[3][1],
		e[0][2], e[1][2], e[2][2], e[3][2],
		e[0][3], e[1][3], e[2][3], e[3][3],
	}
	C.glUniformMatrix4fv(getUniformLocation(program, "color_matrix"),
		1, C.GL_FALSE, (*C.GLfloat)(&glColorMatrix[0]))

	glColorMatrixTranslation := [...]float32{
		e[0][4], e[1][4], e[2][4], e[3][4],
	}
	C.glUniform4fv(getUniformLocation(program, "color_matrix_translation"),
		1, (*C.GLfloat)(&glColorMatrixTranslation[0]))

	return
}

func (context *Context) NewRenderTarget(width, height int) (
	graphics.RenderTargetId, error) {
	renderTarget, err := newRenderTarget(width, height)
	if err != nil {
		return 0, nil
	}
	renderTargetId := graphics.RenderTargetId(<-newId)
	textureId := graphics.TextureId(<-newId)
	context.renderTargets[renderTargetId] = renderTarget
	context.textures[textureId] = renderTarget.texture
	context.renderTargetToTexture[renderTargetId] = textureId

	context.setOffscreen(renderTarget)
	context.Clear()
	// TODO: Is it OK to revert he main framebuffer?
	context.setMainFramebufferOffscreen()

	return renderTargetId, nil
}

func (context *Context) NewTextureFromImage(img image.Image) (
	graphics.TextureId, error) {
	texture, err := texture.NewFromImage(img, &NativeTextureCreator{})
	if err != nil {
		return 0, err
	}
	textureId := graphics.TextureId(<-newId)
	context.textures[textureId] = texture
	return textureId, nil
}

var newId chan int

func init() {
	newId = make(chan int)
	go func() {
		for i := 0; ; i++ {
			newId <- i
		}
	}()
}
<commit_msg>Optimize rendering textures<commit_after>package opengl

// #cgo LDFLAGS: -framework OpenGL
//
// #include <stdlib.h>
// #include <OpenGL/gl.h>
import "C"
import (
	"fmt"
	"github.com/hajimehoshi/go-ebiten/graphics"
	"github.com/hajimehoshi/go-ebiten/graphics/matrix"
	"github.com/hajimehoshi/go-ebiten/graphics/texture"
	"image"
	"math"
	"unsafe"
)

type Context struct {
	screenId               graphics.RenderTargetId
	screenWidth            int
	screenHeight           int
	screenScale            int
	textures               map[graphics.TextureId]*texture.Texture
	renderTargets          map[graphics.RenderTargetId]*RenderTarget
	renderTargetToTexture  map[graphics.RenderTargetId]graphics.TextureId
	currentOffscreen       *RenderTarget
	mainFramebufferTexture *RenderTarget
	projectionMatrix       [16]float32
}

func newContext(screenWidth, screenHeight, screenScale int) *Context {
	context := &Context{
		screenWidth:           screenWidth,
		screenHeight:          screenHeight,
		screenScale:           screenScale,
		textures:              map[graphics.TextureId]*texture.Texture{},
		renderTargets:         map[graphics.RenderTargetId]*RenderTarget{},
		renderTargetToTexture: map[graphics.RenderTargetId]graphics.TextureId{},
	}
	return context
}

func (context *Context) Init() {
	// The main framebuffer should be created sooner than any other
	// framebuffers!
	mainFramebuffer := C.GLint(0)
	C.glGetIntegerv(C.GL_FRAMEBUFFER_BINDING, &mainFramebuffer)

	var err error
	context.mainFramebufferTexture, err = newRenderTargetWithFramebuffer(
		context.screenWidth*context.screenScale,
		context.screenHeight*context.screenScale,
		C.GLuint(mainFramebuffer))
	if err != nil {
		panic("creating main framebuffer failed: " + err.Error())
	}

	initializeShaders()

	context.screenId, err = context.NewRenderTarget(
		context.screenWidth, context.screenHeight)
	if err != nil {
		panic("initializing the offscreen failed: " + err.Error())
	}
}

func (context *Context) ToTexture(renderTargetId graphics.RenderTargetId) graphics.TextureId {
	return context.renderTargetToTexture[renderTargetId]
}

func (context *Context) Clear() {
	context.Fill(0, 0, 0)
}

func (context *Context) Fill(r, g, b uint8) {
	const max = float64(math.MaxUint8)
	C.glClearColor(
		C.GLclampf(float64(r)/max),
		C.GLclampf(float64(g)/max),
		C.GLclampf(float64(b)/max),
		1)
	C.glClear(C.GL_COLOR_BUFFER_BIT)
}

func (context *Context) DrawTexture(
	textureId graphics.TextureId,
	geometryMatrix matrix.Geometry, colorMatrix matrix.Color) {
	texture, ok := context.textures[textureId]
	if !ok {
		panic("invalid texture ID")
	}
	// TODO: fix this
	source := graphics.Rect{0, 0, texture.Width(), texture.Height()}
	locations := []graphics.TexturePart{{0, 0, source}}
	context.DrawTextureParts(textureId, locations,
		geometryMatrix, colorMatrix)
}

func (context *Context) DrawTextureParts(
	textureId graphics.TextureId, parts []graphics.TexturePart,
	geometryMatrix matrix.Geometry, colorMatrix matrix.Color) {

	texture, ok := context.textures[textureId]
	if !ok {
		panic("invalid texture ID")
	}
	if len(parts) == 0 {
		return
	}

	shaderProgram := context.setShaderProgram(geometryMatrix, colorMatrix)
	C.glBindTexture(C.GL_TEXTURE_2D, texture.Native().(C.GLuint))

	vertexAttrLocation := getAttributeLocation(shaderProgram, "vertex")
	textureAttrLocation := getAttributeLocation(shaderProgram, "texture")

	C.glEnableClientState(C.GL_VERTEX_ARRAY)
	C.glEnableClientState(C.GL_TEXTURE_COORD_ARRAY)
	C.glEnableVertexAttribArray(C.GLuint(vertexAttrLocation))
	C.glEnableVertexAttribArray(C.GLuint(textureAttrLocation))
	vertices := []float32{}
	texCoords := []float32{}
	indicies := []uint32{}
	// TODO: Check len(parts) and GL_MAX_ELEMENTS_INDICES
	for i, part := range parts {
		x1 := float32(part.LocationX)
		x2 := float32(part.LocationX + part.Source.Width)
		y1 := float32(part.LocationY)
		y2 := float32(part.LocationY + part.Source.Height)
		vertices = append(vertices,
			x1, y1,
			x2, y1,
			x1, y2,
			x2, y2,
		)
		src := part.Source
		tu1 := float32(texture.U(src.X))
		tu2 := float32(texture.U(src.X + src.Width))
		tv1 := float32(texture.V(src.Y))
		tv2 := float32(texture.V(src.Y + src.Height))
		texCoords = append(texCoords,
			tu1, tv1,
			tu2, tv1,
			tu1, tv2,
			tu2, tv2,
		)
		base := uint32(i*4)
		indicies = append(indicies,
			base, base+1, base+2,
			base+1, base+2, base+3,
		)
	}
	C.glVertexAttribPointer(C.GLuint(vertexAttrLocation), 2,
		C.GL_FLOAT, C.GL_FALSE,
		0, unsafe.Pointer(&vertices[0]))
	C.glVertexAttribPointer(C.GLuint(textureAttrLocation), 2,
		C.GL_FLOAT, C.GL_FALSE,
		0, unsafe.Pointer(&texCoords[0]))
	C.glDrawElements(C.GL_TRIANGLES, C.GLsizei(len(indicies)),
		C.GL_UNSIGNED_INT, unsafe.Pointer(&indicies[0]))
	C.glDisableVertexAttribArray(C.GLuint(textureAttrLocation))
	C.glDisableVertexAttribArray(C.GLuint(vertexAttrLocation))
	C.glDisableClientState(C.GL_TEXTURE_COORD_ARRAY)
	C.glDisableClientState(C.GL_VERTEX_ARRAY)
}

func (context *Context) ResetOffscreen() {
	context.SetOffscreen(context.screenId)
}

func (context *Context) SetOffscreen(renderTargetId graphics.RenderTargetId) {
	renderTarget := context.renderTargets[renderTargetId]
	context.setOffscreen(renderTarget)
}

func (context *Context) setOffscreen(renderTarget *RenderTarget) {
	context.currentOffscreen = renderTarget

	C.glFlush()

	C.glBindFramebuffer(C.GL_FRAMEBUFFER, renderTarget.framebuffer)
	err := C.glCheckFramebufferStatus(C.GL_FRAMEBUFFER)
	if err != C.GL_FRAMEBUFFER_COMPLETE {
		panic(fmt.Sprintf("glBindFramebuffer failed: %d", err))
	}

	C.glEnable(C.GL_BLEND)
	C.glBlendFuncSeparate(C.GL_SRC_ALPHA, C.GL_ONE_MINUS_SRC_ALPHA,
		C.GL_ZERO, C.GL_ONE)

	context.currentOffscreen.SetAsViewport(context)
}

func (context *Context) SetViewport(x, y, width, height int) {
	C.glViewport(C.GLint(x), C.GLint(y), C.GLsizei(width), C.GLsizei(height))

	matrix := graphics.OrthoProjectionMatrix(x, width, y, height)
	if context.currentOffscreen == context.mainFramebufferTexture {
		// Flip Y and move to fit with the top of the window.
		matrix[1][1] *= -1
		actualHeight := context.screenHeight * context.screenScale
		matrix[1][3] += float64(actualHeight) / float64(height) * 2
	}

	for j := 0; j < 4; j++ {
		for i := 0; i < 4; i++ {
			context.projectionMatrix[i+j*4] = float32(matrix[i][j])
		}
	}

	// TODO: call 'setShaderProgram' here?
}

func (context *Context) setMainFramebufferOffscreen() {
	context.setOffscreen(context.mainFramebufferTexture)
}

func (context *Context) flush() {
	C.glFlush()
}

func (context *Context) setShaderProgram(
	geometryMatrix matrix.Geometry, colorMatrix matrix.Color) (program C.GLuint) {
	if colorMatrix.IsIdentity() {
		program = regularShaderProgram
	} else {
		program = colorMatrixShaderProgram
	}
	// TODO: cache and skip?
	C.glUseProgram(program)

	C.glUniformMatrix4fv(getUniformLocation(program, "projection_matrix"),
		1, C.GL_FALSE,
		(*C.GLfloat)(&context.projectionMatrix[0]))

	a := float32(geometryMatrix.Elements[0][0])
	b := float32(geometryMatrix.Elements[0][1])
	c := float32(geometryMatrix.Elements[1][0])
	d := float32(geometryMatrix.Elements[1][1])
	tx := float32(geometryMatrix.Elements[0][2])
	ty := float32(geometryMatrix.Elements[1][2])
	glModelviewMatrix := [...]float32{
		a, c, 0, 0,
		b, d, 0, 0,
		0, 0, 1, 0,
		tx, ty, 0, 1,
	}
	C.glUniformMatrix4fv(getUniformLocation(program, "modelview_matrix"),
		1, C.GL_FALSE,
		(*C.GLfloat)(&glModelviewMatrix[0]))

	C.glUniform1i(getUniformLocation(program, "texture"), 0)

	if program != colorMatrixShaderProgram {
		return
	}

	e := [4][5]float32{}
	for i := 0; i < 4; i++ {
		for j := 0; j < 5; j++ {
			e[i][j] = float32(colorMatrix.Elements[i][j])
		}
	}

	glColorMatrix := [...]float32{
		e[0][0], e[1][0], e[2][0], e[3][0],
		e[0][1], e[1][1], e[2][1], e[3][1],
		e[0][2], e[1][2], e[2][2], e[3][2],
		e[0][3], e[1][3], e[2][3], e[3][3],
	}
	C.glUniformMatrix4fv(getUniformLocation(program, "color_matrix"),
		1, C.GL_FALSE, (*C.GLfloat)(&glColorMatrix[0]))

	glColorMatrixTranslation := [...]float32{
		e[0][4], e[1][4], e[2][4], e[3][4],
	}
	C.glUniform4fv(getUniformLocation(program, "color_matrix_translation"),
		1, (*C.GLfloat)(&glColorMatrixTranslation[0]))

	return
}

func (context *Context) NewRenderTarget(width, height int) (
	graphics.RenderTargetId, error) {
	renderTarget, err := newRenderTarget(width, height)
	if err != nil {
		return 0, nil
	}
	renderTargetId := graphics.RenderTargetId(<-newId)
	textureId := graphics.TextureId(<-newId)
	context.renderTargets[renderTargetId] = renderTarget
	context.textures[textureId] = renderTarget.texture
	context.renderTargetToTexture[renderTargetId] = textureId

	context.setOffscreen(renderTarget)
	context.Clear()
	// TODO: Is it OK to revert he main framebuffer?
	context.setMainFramebufferOffscreen()

	return renderTargetId, nil
}

func (context *Context) NewTextureFromImage(img image.Image) (
	graphics.TextureId, error) {
	texture, err := texture.NewFromImage(img, &NativeTextureCreator{})
	if err != nil {
		return 0, err
	}
	textureId := graphics.TextureId(<-newId)
	context.textures[textureId] = texture
	return textureId, nil
}

var newId chan int

func init() {
	newId = make(chan int)
	go func() {
		for i := 0; ; i++ {
			newId <- i
		}
	}()
}
<|endoftext|>


<commit_before>package client

import (
	"golang.org/x/net/context"
)

// Req sends a synchronous request to a service using a new client, and unmarshals the response into the supplied
// protobuf
func Req(ctx context.Context, service, endpoint string, req, res interface{}) error {
	return NewClient().
		Add(Call{
			Uid:      "1",
			Service:  service,
			Endpoint: endpoint,
			Body:     req,
			Response: res,
			Context:  ctx,
		}).
		Execute().
		Errors().
		Combined()
}
<commit_msg>Switch default uid used by sugar to 'default' so this is obviously different from 1 which could be misread as an index position.<commit_after>package client

import (
	"golang.org/x/net/context"
)

// Req sends a synchronous request to a service using a new client, and unmarshals the response into the supplied
// protobuf
func Req(ctx context.Context, service, endpoint string, req, res interface{}) error {
	return NewClient().
		Add(Call{
			Uid:      "default",
			Service:  service,
			Endpoint: endpoint,
			Body:     req,
			Response: res,
			Context:  ctx,
		}).
		Execute().
		Errors().
		Combined()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
)

const Version = "0.1.0"

var (
	bucket       string
	region       string
	fileSuffix   string
	outputDir    string
	printVersion bool
	keyPrefix    string
)

func init() {
	flag.StringVar(&bucket, "bucket", "", "s3 bucket name with optional path")
	flag.StringVar(&region, "region", "", "aws region")
	flag.StringVar(&fileSuffix, "file-suffix", ".encrypted", "encrypted file suffix")
	flag.StringVar(&outputDir, "output-dir", "/run/secrets", "output directory")
	flag.BoolVar(&printVersion, "version", false, "print version and exit")
}

func main() {
	flag.Parse()
	if printVersion {
		fmt.Printf("s3secrets %s\n", Version)
		os.Exit(0)
	}
	if bucket == "" {
		fmt.Println("Please specify S3 bucket name. Exiting..")
		os.Exit(1)
	}
	if !dirExists(outputDir) {
		fmt.Println("Output dir does not exist. Exiting..")
		os.Exit(1)
	}
	var cfg *aws.Config
	if region != "" {
		cfg = &aws.Config{Region: region}
	}

	sp := splitPath(bucket)
	bucketName := sp[0]
	if len(sp) == 2 {
		keyPrefix = sp[1]
	} else {
		keyPrefix = ""
	}

	kmsClient := getKmsClient(cfg)
	s3Client := getS3Client(cfg)
	list, err := listObjects(s3Client, bucketName, keyPrefix)
	if err != nil {
		log.Fatalln(err)
	}
	// TODO(vaijab): extract below into a goroutine
	for _, key := range list {
		blob, err := getBlob(s3Client, bucketName, key)
		if err != nil {
			fmt.Println(err)
			continue
		}
		data, err := decrypt(kmsClient, &blob)
		if err != nil {
			fmt.Println(err)
			continue
		}
		fileName := path.Base(key)
		file := path.Join(outputDir, strings.TrimSuffix(fileName, fileSuffix))
		if err = ioutil.WriteFile(file, data, 0600); err != nil {
			fmt.Printf("Error writing to %s\n", fileName)
		} else {
			fmt.Printf("Successfully decrypted %s to %s\n", path.Join(bucket, key), file)
		}
	}
}

func dirExists(f string) bool {
	if _, err := os.Stat(f); err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			return true
		}
	}
	return true
}

func splitPath(b string) []string {
	p := strings.SplitN(b, "/", 2)
	return p
}
<commit_msg>logging and verbose mode added<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
)

const Version = "0.1.0"

var (
	bucket       string
	region       string
	fileSuffix   string
	outputDir    string
	printVersion bool
	keyPrefix    string
	verbose      bool
)

func init() {
	flag.StringVar(&bucket, "bucket", "", "s3 bucket name with optional path")
	flag.StringVar(&region, "region", "", "aws region")
	flag.StringVar(&fileSuffix, "file-suffix", ".encrypted", "encrypted file suffix")
	flag.StringVar(&outputDir, "output-dir", "/run/secrets", "output directory")
	flag.BoolVar(&printVersion, "version", false, "print version and exit")
	flag.BoolVar(&verbose, "verbose", false, "enable verbose mode")
}

func main() {
	flag.Parse()
	if printVersion {
		log.Printf("s3secrets %s\n", Version)
		os.Exit(0)
	}

	if bucket == "" || !dirExists(outputDir) {
		flag.Usage()
		os.Exit(1)
	}

	var cfg *aws.Config
	if region != "" {
		cfg = &aws.Config{Region: &region}
	}

	sp := splitPath(bucket)
	bucketName := sp[0]
	if len(sp) == 2 {
		keyPrefix = sp[1]
	} else {
		keyPrefix = ""
	}

	kmsClient := getKmsClient(cfg)
	if verbose {
		log.Println("Initialising KMS client")
	}

	s3Client := getS3Client(cfg)
	if verbose {
		log.Println("Connecting to S3...")
	}

	list, err := listObjects(s3Client, bucketName, keyPrefix)
	if err != nil {
		log.Fatalln(err)
	}
	// TODO(vaijab): extract below into a goroutine

	for _, key := range list {
		if verbose {
			log.Printf("Fetching file: %s/%s\n", bucketName, key)
		}

		blob, err := getBlob(s3Client, bucketName, key)
		if err != nil {
			fmt.Println(err)
			continue
		}
		if verbose {
			log.Printf("Decrypting %s/%s\n", bucketName, key)
		}
		data, err := decrypt(kmsClient, &blob)
		if err != nil {
			fmt.Println(err)
			continue
		}
		fileName := path.Base(key)
		file := path.Join(outputDir, strings.TrimSuffix(fileName, fileSuffix))
		if verbose {
			log.Printf("Writing file: %s\n", file)
		}
		if err = ioutil.WriteFile(file, data, 0600); err != nil {
			log.Printf("Error writing to %s\n", fileName)
		} else {
			log.Printf("Successfully decrypted %s to %s\n", path.Join(bucket, key), file)
		}
	}
}

func dirExists(f string) bool {
	if _, err := os.Stat(f); err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			return true
		}
	}
	return true
}

func splitPath(b string) []string {
	p := strings.SplitN(b, "/", 2)
	return p
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package msgpack

import "github.com/m3db/m3x/pool"

type bufferedEncoderPool struct {
	maxCapacity int
	pool        pool.ObjectPool
}

// NewBufferedEncoderPool creates a new pool for buffered encoders.
func NewBufferedEncoderPool(opts BufferedEncoderPoolOptions) BufferedEncoderPool {
	return &bufferedEncoderPool{
		maxCapacity: opts.MaxCapacity(),
		pool:        pool.NewObjectPool(opts.ObjectPoolOptions()),
	}
}

func (p *bufferedEncoderPool) Init(alloc BufferedEncoderAlloc) {
	p.pool.Init(func() interface{} {
		return alloc()
	})
}

func (p *bufferedEncoderPool) Get() BufferedEncoder {
	return p.pool.Get().(BufferedEncoder)
}

func (p *bufferedEncoderPool) Put(encoder BufferedEncoder) {
	if encoder.Buffer().Cap() > p.maxCapacity {
		return
	}
	p.pool.Put(encoder)
}
<commit_msg>[buffered encoder pool] Check if options are nil (#77)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package msgpack

import "github.com/m3db/m3x/pool"

type bufferedEncoderPool struct {
	maxCapacity int
	pool        pool.ObjectPool
}

// NewBufferedEncoderPool creates a new pool for buffered encoders.
func NewBufferedEncoderPool(opts BufferedEncoderPoolOptions) BufferedEncoderPool {
	if opts == nil {
		opts = NewBufferedEncoderPoolOptions()
	}
	return &bufferedEncoderPool{
		maxCapacity: opts.MaxCapacity(),
		pool:        pool.NewObjectPool(opts.ObjectPoolOptions()),
	}
}

func (p *bufferedEncoderPool) Init(alloc BufferedEncoderAlloc) {
	p.pool.Init(func() interface{} {
		return alloc()
	})
}

func (p *bufferedEncoderPool) Get() BufferedEncoder {
	return p.pool.Get().(BufferedEncoder)
}

func (p *bufferedEncoderPool) Put(encoder BufferedEncoder) {
	if encoder.Buffer().Cap() > p.maxCapacity {
		return
	}
	p.pool.Put(encoder)
}
<|endoftext|>


<commit_before>package apns

import (
	"crypto/tls"
	"log"
	"time"

	"code.google.com/p/go.net/context"
	"github.com/cenkalti/backoff"
)

// Sender sends notifications
type Sender struct {
	addr       string
	cert       *tls.Certificate
	conn       conn
	notifc     chan *Notification
	prioNotifc *priochan
	errorc     chan *ErrorFeedback
	readc      chan *readEvent
	newConn    func(addr string, cert *tls.Certificate) (conn, error)
}

// ErrorFeedback represents an error feedback
type ErrorFeedback struct {
	Notification  *Notification
	ErrorResponse *ErrorResponse
}

type readEvent struct {
	resp *ErrorResponse
	conn conn
}

// NewSender creates a new Sender
func NewSender(ctx context.Context, addr string, cert *tls.Certificate) *Sender {
	s := &Sender{
		addr:       addr,
		cert:       cert,
		notifc:     make(chan *Notification),
		prioNotifc: newPriochan(),
		errorc:     make(chan *ErrorFeedback),
		readc:      make(chan *readEvent),
		newConn:    newConn,
	}

	s.prioNotifc.Add(s.notifc)

	go s.senderJob(ctx)

	return s
}

// Notifications returns the channel to which to send notifications
func (s *Sender) Notifications() chan *Notification {
	return s.notifc
}

// ErrorFeedbacks returns the channel from which to receive ErrorFeedbacks
func (s *Sender) ErrorFeedbacks() <-chan *ErrorFeedback {
	return s.errorc
}

func (s *Sender) senderJob(ctx context.Context) {

	ticker := time.Tick(time.Second)

	for {
		select {
		case <-ctx.Done():
			if s.conn != nil {
				s.conn.Close()
			}
			s.prioNotifc.Close()
			return
		case ev := <-s.readc:
			s.handleRead(ev)
		case n := <-s.prioNotifc.Receive():
			log.Printf("Sending notification %v", n.Identifier)
			s.doSend(n)
		case <-ticker:
			if s.conn != nil {
				s.conn.Expire()
			}
		}
	}
}

func (s *Sender) handleRead(ev *readEvent) {

	var n *Notification
	var sent []*Notification
	conn := ev.conn

	conn.Close()
	if conn == s.conn {
		s.conn = nil
	}

	if resp := ev.resp; resp != nil {
		n = conn.GetSentNotification(resp.Identifier)

		if n == nil {
			log.Printf("Got a response for unknown notification %v", resp.Identifier)
		} else {
			log.Printf("Got a response for notification %v", resp.Identifier)

			// for ShutdownErrorStatus, the Identifier indicates the last
			// notification that was successfully sent
			if resp.Status != ShutdownErrorStatus {
				s.errorc <- &ErrorFeedback{
					Notification:  n,
					ErrorResponse: resp,
				}
			}
		}
	}

	if n != nil {
		sent = conn.GetSentNotificationsAfter(n.Identifier())
	} else {
		sent = conn.GetSentNotifications()
	}

	// requeue notifications before anything sent to s.notifc
	c := make(chan *Notification)
	s.prioNotifc.Add(c)

	go func() {
		for _, n := range sent {
			log.Printf("Requeuing notification %v", n.Identifier)
			c <- n
		}
		close(c)
	}()
}

func (s *Sender) doSend(n *Notification) {

	for {
		s.connect()

		if connError, err := s.conn.Write(n); err != nil {
			if connError {
				s.conn.Close()
				s.conn = nil
				log.Printf("%v; will retry", err)
			} else {
				log.Println(err)
			}
		} else {
			break
		}
	}
}

func (s *Sender) connect() {

	for s.conn == nil {
		var conn conn
		var err error

		connect := func() error {
			log.Printf("Connecting to %v", s.addr)
			conn, err = s.newConn(s.addr, s.cert)
			if err != nil {
				log.Printf("Failed connecting to %v: %v; will retry", s.addr, err)
				return err
			}
			return nil
		}

		if backoff.Retry(connect, backoff.NewExponentialBackOff()) != nil {
			continue
		}

		log.Printf("Connected to %v", s.addr)

		go s.read(conn)

		s.conn = conn
	}
}

func (s *Sender) read(c conn) {
	for {
		select {
		case <-c.Done():
			return
		case pnr := <-c.Read():
			s.readc <- &readEvent{pnr, c}
		}
	}
}
<commit_msg>added Done()<commit_after>package apns

import (
	"crypto/tls"
	"log"
	"time"

	"code.google.com/p/go.net/context"
	"github.com/cenkalti/backoff"
)

// Sender sends notifications
type Sender struct {
	addr       string
	cert       *tls.Certificate
	conn       conn
	notifc     chan *Notification
	prioNotifc *priochan
	errorc     chan *ErrorFeedback
	readc      chan *readEvent
	newConn    func(addr string, cert *tls.Certificate) (conn, error)
	donec      chan struct{}
}

// ErrorFeedback represents an error feedback
type ErrorFeedback struct {
	Notification  *Notification
	ErrorResponse *ErrorResponse
}

type readEvent struct {
	resp *ErrorResponse
	conn conn
}

// NewSender creates a new Sender
func NewSender(ctx context.Context, addr string, cert *tls.Certificate) *Sender {
	s := &Sender{
		addr:       addr,
		cert:       cert,
		notifc:     make(chan *Notification),
		prioNotifc: newPriochan(),
		errorc:     make(chan *ErrorFeedback),
		readc:      make(chan *readEvent),
		newConn:    newConn,
		donec:      make(chan struct{}),
	}

	s.prioNotifc.Add(s.notifc)

	go s.senderJob(ctx)

	return s
}

// Notifications returns the channel to which to send notifications
func (s *Sender) Notifications() chan *Notification {
	return s.notifc
}

// ErrorFeedbacks returns the channel from which to receive ErrorFeedbacks
func (s *Sender) ErrorFeedbacks() <-chan *ErrorFeedback {
	return s.errorc
}

// Done returns a channel that's closed when this Sender has terminated
func (s *Sender) Done() <-chan struct{} {
	return s.donec
}

func (s *Sender) senderJob(ctx context.Context) {

	ticker := time.Tick(time.Second)

for_loop:
	for {
		select {
		case <-ctx.Done():
			if s.conn != nil {
				s.conn.Close()
			}
			s.prioNotifc.Close()
			break for_loop
		case ev := <-s.readc:
			s.handleRead(ev)
		case n := <-s.prioNotifc.Receive():
			log.Printf("Sending notification %v", n.Identifier)
			s.doSend(n)
		case <-ticker:
			if s.conn != nil {
				s.conn.Expire()
			}
		}
	}

	close(s.donec)
}

func (s *Sender) handleRead(ev *readEvent) {

	var n *Notification
	var sent []*Notification
	conn := ev.conn

	conn.Close()
	if conn == s.conn {
		s.conn = nil
	}

	if resp := ev.resp; resp != nil {
		n = conn.GetSentNotification(resp.Identifier)

		if n == nil {
			log.Printf("Got a response for unknown notification %v", resp.Identifier)
		} else {
			log.Printf("Got a response for notification %v", resp.Identifier)

			// for ShutdownErrorStatus, the Identifier indicates the last
			// notification that was successfully sent
			if resp.Status != ShutdownErrorStatus {
				s.errorc <- &ErrorFeedback{
					Notification:  n,
					ErrorResponse: resp,
				}
			}
		}
	}

	if n != nil {
		sent = conn.GetSentNotificationsAfter(n.Identifier())
	} else {
		sent = conn.GetSentNotifications()
	}

	// requeue notifications before anything sent to s.notifc
	c := make(chan *Notification)
	s.prioNotifc.Add(c)

	go func() {
		for _, n := range sent {
			log.Printf("Requeuing notification %v", n.Identifier)
			c <- n
		}
		close(c)
	}()
}

func (s *Sender) doSend(n *Notification) {

	for {
		s.connect()

		if connError, err := s.conn.Write(n); err != nil {
			if connError {
				s.conn.Close()
				s.conn = nil
				log.Printf("%v; will retry", err)
			} else {
				log.Println(err)
			}
		} else {
			break
		}
	}
}

func (s *Sender) connect() {

	for s.conn == nil {
		var conn conn
		var err error

		connect := func() error {
			log.Printf("Connecting to %v", s.addr)
			conn, err = s.newConn(s.addr, s.cert)
			if err != nil {
				log.Printf("Failed connecting to %v: %v; will retry", s.addr, err)
				return err
			}
			return nil
		}

		if backoff.Retry(connect, backoff.NewExponentialBackOff()) != nil {
			continue
		}

		log.Printf("Connected to %v", s.addr)

		go s.read(conn)

		s.conn = conn
	}
}

func (s *Sender) read(c conn) {
	for {
		select {
		case <-c.Done():
			return
		case pnr := <-c.Read():
			s.readc <- &readEvent{pnr, c}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"encoding/json"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// Update updates a document based on a script provided. The operation gets the document
// (collocated with the shard) from the index, runs the script (with optional script language and parameters),
// and index back the result (also allows to delete, or ignore the operation). It uses versioning to make sure
// no updates have happened during the “get” and “reindex”. (available from 0.19 onwards).
// Note, this operation still means full reindex of the document, it just removes some network roundtrips
// and reduces chances of version conflicts between the get and the index. The _source field need to be enabled
// for this feature to work.
//
// http://www.elasticsearch.org/guide/reference/api/update.html
// TODO: finish this, it's fairly complex
func Update(pretty bool, index string, _type string, id string) (api.BaseResponse, error) {
	var url string
	var retval api.BaseResponse
	url = fmt.Sprintf("/%s/%s/%s/_update?%s", index, _type, id, api.Pretty(pretty))
	body, err := api.DoCommand("POST", url, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	fmt.Println(body)
	return retval, err
}
<commit_msg>- added support for Update API<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"encoding/json"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// Update updates a document based on a script provided. The operation gets the document
// (collocated with the shard) from the index, runs the script (with optional script language and parameters),
// and index back the result (also allows to delete, or ignore the operation). It uses versioning to make sure
// no updates have happened during the “get” and “reindex”. (available from 0.19 onwards).
// Note, this operation still means full reindex of the document, it just removes some network roundtrips
// and reduces chances of version conflicts between the get and the index. The _source field need to be enabled
// for this feature to work.
//
// http://www.elasticsearch.org/guide/reference/api/update.html
// TODO: finish this, it's fairly complex
func Update(pretty bool, index string, _type string, id string, data interface{}) (api.BaseResponse, error) {
	var url string
	var retval api.BaseResponse
	url = fmt.Sprintf("/%s/%s/%s/_update?%s", index, _type, id, api.Pretty(pretty))
	body, err := api.DoCommand("POST", url, data)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	fmt.Println(body)
	return retval, err
}

// UpdateWithPartialDoc updates a document based on partial document provided. The update API also
// support passing a partial document (since 0.20), which will be merged into the existing
// document (simple recursive merge, inner merging of objects, replacing core "keys/values" and arrays).
// If both doc and script is specified, then doc is ignored. Best is to put your field pairs of the partial
// document in the script itself.
//
// http://www.elasticsearch.org/guide/reference/api/update.html
func UpdateWithPartialDoc(pretty bool, index string, _type string, id string, doc interface{}, upsert bool) (api.BaseResponse, error) {
	switch v := doc.(type) {
	case string:
		upsertStr := ""
		if upsert {
			upsertStr = ", \"doc_as_upsert\":true"
		}
		content := fmt.Sprintf("{\"doc\":%s %s}", v, upsertStr)
		return Update(pretty, index, _type, id, content)
	default:
		var data map[string]interface{} = make(map[string]interface{})
		data["doc"] = doc
		if upsert {
			data["doc_as_upsert"] = true
		}
		return Update(pretty, index, _type, id, data)
	}
}

// UpdateWithScript updates a document based on a script provided.
// The operation gets the document (collocated with the shard) from the index, runs the script
// (with optional script language and parameters), and index back the result (also allows to
// delete, or ignore the operation). It uses versioning to make sure no updates have happened
// during the "get" and "reindex". (available from 0.19 onwards).
//
// Note, this operation still means full reindex of the document, it just removes some network
// roundtrips and reduces chances of version conflicts between the get and the index. The _source
// field need to be enabled for this feature to work.
// http://www.elasticsearch.org/guide/reference/api/update.html
func UpdateWithScript(pretty bool, index string, _type string, id string, script string, params interface{}) (api.BaseResponse, error) {
	switch v := params.(type) {
	case string:
		paramsPart := fmt.Sprintf("{\"params\":%s}", v)
		data := fmt.Sprintf("{\"script\":\"%s\", \"params\":%s}", script, paramsPart)
		return Update(pretty, index, _type, id, data)
	default:
		var data map[string]interface{} = make(map[string]interface{})
		data["params"] = params
		data["script"] = script
		return Update(pretty, index, _type, id, data)
	}
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"io"
	"os"
	"regexp"
	"unicode/utf8"

	"github.com/mibk/syd/ui/term"
)

const EOF = utf8.MaxRune + 1

type Window struct {
	col      *Column
	filename string
	win      *term.Window
	con      Content

	buf  *UndoBuffer
	tag  *Text
	body *Text

	// used by Read and flush methods
	insertbuf bytes.Buffer
}

func (win *Window) SetFilename(filename string) {
	win.filename = filename
	win.tag.buf.Insert(0, filename)
	win.col.ed.wins[filename] = win
}

func (win *Window) Frame() *term.Frame { return win.body.text.Frame() } // TODO: delete

func (win *Window) redraw() {
	win.win.SetDirty(win.buf.Dirty())
	win.tag.redraw()
	win.body.redraw()
}

func (win *Window) Close() error {
	win.win.Delete()
	win.col.deleteWindow(win)
	if ed := win.col.ed; ed.errWin == win {
		ed.errWin = nil
	}
	if win.filename != "" {
		delete(win.col.ed.wins, win.filename)
	}
	return win.con.Close()
}

func (win *Window) Write(b []byte) (n int, err error) {
	return win.insertbuf.Write(b)
}

func (win *Window) flush() {
	s := win.insertbuf.String()
	win.insertbuf.Reset()
	q := win.body.q0
	win.body.Insert(s)
	win.body.Select(q, q+int64(utf8.RuneCountInString(s)))

	// TODO: Come up with a better solution?
	win.buf.Commit()
}

func (win *Window) saveFile() {
	if win.filename == "" {
		win.readFilename()
	}

	// TODO: Don't use '~' suffix, make saving safer.
	f, err := os.Create(win.filename + "~")
	if err != nil {
		panic(err)
	}
	r := io.NewSectionReader(win.buf, 0, win.buf.Size())
	if _, err := io.Copy(f, r); err != nil {
		panic(err)
	}
	f.Close()

	if err := os.Rename(win.filename+"~", win.filename); err != nil {
		panic(err)
	}
}

func (win *Window) readFilename() {
	var runes []rune
	var p int64
	for {
		r := win.tag.readRuneAt(p)
		if r == 0 || r == EOF {
			break
		}
		runes = append(runes, r)
		p++
	}
	if len(runes) == 0 {
		return
	}
	win.filename = string(runes)
}

func (win *Window) findNextExactMatch(s string) {
	rx := regexp.MustCompile(regexp.QuoteMeta(s))

	body := win.body
	buf := win.buf
	for _, q := range []int64{body.q1, 0} {
		r, off := buf.RuneReaderFrom(q)
		if loc := rx.FindReaderIndex(r); loc != nil {
			q0, q1 := buf.FindRange(off+int64(loc[0]), int64(loc[1]-loc[0]))
			body.Select(q0, q1)
			return
		}
	}
}

func (win *Window) editor() (ed *Editor)           { return win.col.ed }
func (win *Window) column() (col *Column, ok bool) { return win.col, true }
func (win *Window) window() (w *Window, ok bool)   { return win, true }
<commit_msg>core: Remove (*Window).Frame<commit_after>package core

import (
	"bytes"
	"io"
	"os"
	"regexp"
	"unicode/utf8"

	"github.com/mibk/syd/ui/term"
)

const EOF = utf8.MaxRune + 1

type Window struct {
	col      *Column
	filename string
	win      *term.Window
	con      Content

	buf  *UndoBuffer
	tag  *Text
	body *Text

	// used by Read and flush methods
	insertbuf bytes.Buffer
}

func (win *Window) SetFilename(filename string) {
	win.filename = filename
	win.tag.buf.Insert(0, filename)
	win.col.ed.wins[filename] = win
}

func (win *Window) redraw() {
	win.win.SetDirty(win.buf.Dirty())
	win.tag.redraw()
	win.body.redraw()
}

func (win *Window) Close() error {
	win.win.Delete()
	win.col.deleteWindow(win)
	if ed := win.col.ed; ed.errWin == win {
		ed.errWin = nil
	}
	if win.filename != "" {
		delete(win.col.ed.wins, win.filename)
	}
	return win.con.Close()
}

func (win *Window) Write(b []byte) (n int, err error) {
	return win.insertbuf.Write(b)
}

func (win *Window) flush() {
	s := win.insertbuf.String()
	win.insertbuf.Reset()
	q := win.body.q0
	win.body.Insert(s)
	win.body.Select(q, q+int64(utf8.RuneCountInString(s)))

	// TODO: Come up with a better solution?
	win.buf.Commit()
}

func (win *Window) saveFile() {
	if win.filename == "" {
		win.readFilename()
	}

	// TODO: Don't use '~' suffix, make saving safer.
	f, err := os.Create(win.filename + "~")
	if err != nil {
		panic(err)
	}
	r := io.NewSectionReader(win.buf, 0, win.buf.Size())
	if _, err := io.Copy(f, r); err != nil {
		panic(err)
	}
	f.Close()

	if err := os.Rename(win.filename+"~", win.filename); err != nil {
		panic(err)
	}
}

func (win *Window) readFilename() {
	var runes []rune
	var p int64
	for {
		r := win.tag.readRuneAt(p)
		if r == 0 || r == EOF {
			break
		}
		runes = append(runes, r)
		p++
	}
	if len(runes) == 0 {
		return
	}
	win.filename = string(runes)
}

func (win *Window) findNextExactMatch(s string) {
	rx := regexp.MustCompile(regexp.QuoteMeta(s))

	body := win.body
	buf := win.buf
	for _, q := range []int64{body.q1, 0} {
		r, off := buf.RuneReaderFrom(q)
		if loc := rx.FindReaderIndex(r); loc != nil {
			q0, q1 := buf.FindRange(off+int64(loc[0]), int64(loc[1]-loc[0]))
			body.Select(q0, q1)
			return
		}
	}
}

func (win *Window) editor() (ed *Editor)           { return win.col.ed }
func (win *Window) column() (col *Column, ok bool) { return win.col, true }
func (win *Window) window() (w *Window, ok bool)   { return win, true }
<|endoftext|>


<commit_before>package git

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// Finds things that aren't tracked, and creates fake IndexEntrys for them to be merged into
// the output if --others is passed.
func findUntrackedFilesFromDir(c *Client, opts LsFilesOptions, root, parent, dir File, tracked map[IndexPath]bool, recursedir bool, ignorePatterns []IgnorePattern) (untracked []*IndexEntry) {
	files, err := ioutil.ReadDir(dir.String())
	if err != nil {
		return nil
	}
	for _, ignorefile := range opts.ExcludePerDirectory {
		ignoreInDir := ignorefile
		if dir != "" {
			ignoreInDir = dir + "/" + ignorefile
		}

		if ignoreInDir.Exists() {
			log.Println("Adding excludes from", ignoreInDir)

			patterns, err := ParseIgnorePatterns(c, ignoreInDir, dir)
			if err != nil {
				panic(err)
			}
			ignorePatterns = append(ignorePatterns, patterns...)
		}
	}
files:
	for _, fi := range files {
		fname := File(fi.Name())
		if fi.Name() == ".git" {
			continue
		}
		for _, pattern := range ignorePatterns {
			var name File
			if parent == "" {
				name = fname
			} else {
				name = parent + "/" + fname
			}
			if pattern.Matches(name.String(), fi.IsDir()) {
				continue files
			}
		}
		if fi.IsDir() {
			if !recursedir {
				// This isn't very efficient, but lets us implement git ls-files --directory
				// without too many changes.
				indexPath, err := (parent + "/" + fname).IndexPath(c)
				if err != nil {
					panic(err)
				}
				dirHasTracked := false
				for path := range tracked {
					if strings.HasPrefix(path.String(), indexPath.String()) {
						dirHasTracked = true
						break
					}
				}
				if !dirHasTracked {
					untracked = append(untracked, &IndexEntry{PathName: indexPath})
					continue
				}
			}
			var newparent, newdir File
			if parent == "" {
				newparent = fname
			} else {
				newparent = parent + "/" + fname
			}
			if dir == "" {
				newdir = fname
			} else {
				newdir = dir + "/" + fname
			}

			recurseFiles := findUntrackedFilesFromDir(c, opts, root, newparent, newdir, tracked, recursedir, ignorePatterns)
			untracked = append(untracked, recurseFiles...)
		} else {
			var filePath File
			if parent == "" {
				filePath = File(strings.TrimPrefix(fname.String(), root.String()))

			} else {
				filePath = File(strings.TrimPrefix((parent + "/" + fname).String(), root.String()))
			}
			indexPath, err := filePath.IndexPath(c)
			if err != nil {
				panic(err)
			}
			indexPath = IndexPath(filePath)

			if _, ok := tracked[indexPath]; !ok {
				untracked = append(untracked, &IndexEntry{PathName: indexPath})
			}
		}
	}
	return
}

// Describes the options that may be specified on the command line for
// "git diff-index". Note that only raw mode is currently supported, even
// though all the other options are parsed/set in this struct.
type LsFilesOptions struct {
	// Types of files to show
	Cached, Deleted, Modified, Others bool

	// Invert exclusion logic
	Ignored bool

	// Show stage status instead of just file name
	Stage bool

	// Show files which are unmerged. Implies Stage.
	Unmerged bool

	// If a directory is classified as "other", show only its name, not
	// its contents
	Directory bool

	// Do not show empty directories with --others
	NoEmptyDirectory bool

	// Exclude standard patterns (ie. .gitignore and .git/info/exclude)
	ExcludeStandard bool

	// Exclude using the provided patterns
	ExcludePatterns []string

	// Exclude using the provided file with the patterns
	ExcludeFiles []File

	// Exclude using additional patterns from each directory
	ExcludePerDirectory []File

	ErrorUnmatch bool
}

// LsFiles implements the git ls-files command. It returns an array of files
// that match the options passed.
func LsFiles(c *Client, opt LsFilesOptions, files []File) ([]*IndexEntry, error) {
	var fs []*IndexEntry
	index, err := c.GitDir.ReadIndex()
	if err != nil {
		return nil, err
	}

	// We need to keep track of what's in the index if --others is passed.
	// Keep a map instead of doing an O(n) search every time.
	var filesInIndex map[IndexPath]bool
	if opt.Others || opt.ErrorUnmatch {
		filesInIndex = make(map[IndexPath]bool)
	}

	for _, entry := range index.Objects {
		f, err := entry.PathName.FilePath(c)
		if err != nil {
			return nil, err
		}

		if opt.Others || opt.ErrorUnmatch {
			filesInIndex[entry.PathName] = true
		}

		if strings.HasPrefix(f.String(), "../") || len(files) > 0 {
			skip := true
			for _, explicit := range files {
				eAbs, err := filepath.Abs(explicit.String())
				if err != nil {
					return nil, err
				}
				fAbs, err := filepath.Abs(f.String())
				if err != nil {
					return nil, err
				}
				if fAbs == eAbs || strings.HasPrefix(fAbs, eAbs+"/") {
					skip = false
					break
				}
				if f.MatchGlob(explicit.String()) {
					skip = false
					break
				}
			}
			if skip {
				continue
			}
		}

		if opt.Cached {
			fs = append(fs, entry)
			continue
		}
		if opt.Deleted {
			if !f.Exists() {
				fs = append(fs, entry)
				continue
			}
		}

		if opt.Unmerged && entry.Stage() != Stage0 {
			fs = append(fs, entry)
			continue
		}

		if opt.Modified {
			if f.IsDir() {
				fs = append(fs, entry)
				continue
			}
			_, err := f.Stat()
			// The file being deleted means it was modified
			if os.IsNotExist(err) {
				fs = append(fs, entry)
				continue
			} else if err != nil {
				return nil, err
			}

			// We've done everything we can to avoid hashing the file, but now
			// we need to to avoid the case where someone changes a file, then
			// changes it back to the original contents
			hash, _, err := HashFile("blob", f.String())
			if err != nil {
				return nil, err
			}
			if hash != entry.Sha1 {
				fs = append(fs, entry)
			}
		}
	}

	if opt.ErrorUnmatch {
		for _, file := range files {
			indexPath, err := file.IndexPath(c)
			if err != nil {
				return nil, err
			}
			if _, ok := filesInIndex[indexPath]; !ok {
				fmt.Printf("%v", filesInIndex)
				return nil, fmt.Errorf("error: pathspec '%v' did not match any file(s) known to git", file)
			}
		}
	}

	if opt.Others {
		wd := File(c.WorkDir)

		ignorePatterns := []IgnorePattern{}

		for _, file := range opt.ExcludeFiles {
			patterns, err := ParseIgnorePatterns(c, file, "")
			if err != nil {
				return nil, err
			}
			ignorePatterns = append(ignorePatterns, patterns...)
		}

		for _, pattern := range opt.ExcludePatterns {
			ignorePatterns = append(ignorePatterns, IgnorePattern{Pattern: pattern, Source: "", LineNum: 1, Scope: ""})
		}

		others := findUntrackedFilesFromDir(c, opt, wd+"/", wd, wd, filesInIndex, !opt.Directory, ignorePatterns)
		otherFiles := make([]File, 0, len(others))

		for _, file := range others {
			f, err := file.PathName.FilePath(c)
			if err != nil {
				return nil, err
			}

			if strings.HasPrefix(f.String(), "../") || len(files) > 0 {
				skip := true
				for _, explicit := range files {
					eAbs, err := filepath.Abs(explicit.String())
					if err != nil {
						return nil, err
					}
					fAbs, err := filepath.Abs(f.String())
					if err != nil {
						return nil, err
					}
					if fAbs == eAbs || strings.HasPrefix(fAbs, eAbs+"/") {
						skip = false
						break
					}
				}
				if skip {
					continue
				}
			}

			if f.IsDir() && opt.Directory {
				if opt.NoEmptyDirectory {
					if files, err := ioutil.ReadDir(f.String()); len(files) == 0 && err == nil {
						continue
					}
				}
				f += "/"
			}

			otherFiles = append(otherFiles, f)
		}

		if opt.ExcludeStandard {
			standardPatterns, err := StandardIgnorePatterns(c, otherFiles)
			if err != nil {
				return nil, err
			}
			ignorePatterns = append(ignorePatterns, standardPatterns...)
		}

		matches, err := MatchIgnores(c, ignorePatterns, otherFiles)
		if err != nil {
			return nil, err
		}

		for _, match := range matches {
			if match.Pattern == "" { // TODO add ignore here
				indexPath, err := match.PathName.IndexPath(c)
				if err != nil {
					return nil, err
				}
				// Add a "/" if --directory is set so that it sorts properly in some
				// edge cases.
				if match.PathName.IsDir() && opt.Directory {
					indexPath += "/"

				}
				fs = append(fs, &IndexEntry{PathName: indexPath})
			}
		}
	}

	sort.Sort(ByPath(fs))
	return fs, nil
}
<commit_msg>Make exclude-standard an alias for --exclude-per-directory and --exclude-file<commit_after>package git

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// Finds things that aren't tracked, and creates fake IndexEntrys for them to be merged into
// the output if --others is passed.
func findUntrackedFilesFromDir(c *Client, opts LsFilesOptions, root, parent, dir File, tracked map[IndexPath]bool, recursedir bool, ignorePatterns []IgnorePattern) (untracked []*IndexEntry) {
	files, err := ioutil.ReadDir(dir.String())
	if err != nil {
		return nil
	}
	for _, ignorefile := range opts.ExcludePerDirectory {
		ignoreInDir := ignorefile
		if dir != "" {
			ignoreInDir = dir + "/" + ignorefile
		}

		if ignoreInDir.Exists() {
			log.Println("Adding excludes from", ignoreInDir)

			patterns, err := ParseIgnorePatterns(c, ignoreInDir, dir)
			if err != nil {
				panic(err)
			}
			ignorePatterns = append(ignorePatterns, patterns...)
		}
	}
files:
	for _, fi := range files {
		fname := File(fi.Name())
		if fi.Name() == ".git" {
			continue
		}
		for _, pattern := range ignorePatterns {
			var name File
			if parent == "" {
				name = fname
			} else {
				name = parent + "/" + fname
			}
			if pattern.Matches(name.String(), fi.IsDir()) {
				continue files
			}
		}
		if fi.IsDir() {
			if !recursedir {
				// This isn't very efficient, but lets us implement git ls-files --directory
				// without too many changes.
				indexPath, err := (parent + "/" + fname).IndexPath(c)
				if err != nil {
					panic(err)
				}
				dirHasTracked := false
				for path := range tracked {
					if strings.HasPrefix(path.String(), indexPath.String()) {
						dirHasTracked = true
						break
					}
				}
				if !dirHasTracked {
					untracked = append(untracked, &IndexEntry{PathName: indexPath})
					continue
				}
			}
			var newparent, newdir File
			if parent == "" {
				newparent = fname
			} else {
				newparent = parent + "/" + fname
			}
			if dir == "" {
				newdir = fname
			} else {
				newdir = dir + "/" + fname
			}

			recurseFiles := findUntrackedFilesFromDir(c, opts, root, newparent, newdir, tracked, recursedir, ignorePatterns)
			untracked = append(untracked, recurseFiles...)
		} else {
			var filePath File
			if parent == "" {
				filePath = File(strings.TrimPrefix(fname.String(), root.String()))

			} else {
				filePath = File(strings.TrimPrefix((parent + "/" + fname).String(), root.String()))
			}
			indexPath, err := filePath.IndexPath(c)
			if err != nil {
				panic(err)
			}
			indexPath = IndexPath(filePath)

			if _, ok := tracked[indexPath]; !ok {
				untracked = append(untracked, &IndexEntry{PathName: indexPath})
			}
		}
	}
	return
}

// Describes the options that may be specified on the command line for
// "git diff-index". Note that only raw mode is currently supported, even
// though all the other options are parsed/set in this struct.
type LsFilesOptions struct {
	// Types of files to show
	Cached, Deleted, Modified, Others bool

	// Invert exclusion logic
	Ignored bool

	// Show stage status instead of just file name
	Stage bool

	// Show files which are unmerged. Implies Stage.
	Unmerged bool

	// If a directory is classified as "other", show only its name, not
	// its contents
	Directory bool

	// Do not show empty directories with --others
	NoEmptyDirectory bool

	// Exclude standard patterns (ie. .gitignore and .git/info/exclude)
	ExcludeStandard bool

	// Exclude using the provided patterns
	ExcludePatterns []string

	// Exclude using the provided file with the patterns
	ExcludeFiles []File

	// Exclude using additional patterns from each directory
	ExcludePerDirectory []File

	ErrorUnmatch bool
}

// LsFiles implements the git ls-files command. It returns an array of files
// that match the options passed.
func LsFiles(c *Client, opt LsFilesOptions, files []File) ([]*IndexEntry, error) {
	var fs []*IndexEntry
	index, err := c.GitDir.ReadIndex()
	if err != nil {
		return nil, err
	}

	// We need to keep track of what's in the index if --others is passed.
	// Keep a map instead of doing an O(n) search every time.
	var filesInIndex map[IndexPath]bool
	if opt.Others || opt.ErrorUnmatch {
		filesInIndex = make(map[IndexPath]bool)
	}

	for _, entry := range index.Objects {
		f, err := entry.PathName.FilePath(c)
		if err != nil {
			return nil, err
		}

		if opt.Others || opt.ErrorUnmatch {
			filesInIndex[entry.PathName] = true
		}

		if strings.HasPrefix(f.String(), "../") || len(files) > 0 {
			skip := true
			for _, explicit := range files {
				eAbs, err := filepath.Abs(explicit.String())
				if err != nil {
					return nil, err
				}
				fAbs, err := filepath.Abs(f.String())
				if err != nil {
					return nil, err
				}
				if fAbs == eAbs || strings.HasPrefix(fAbs, eAbs+"/") {
					skip = false
					break
				}
				if f.MatchGlob(explicit.String()) {
					skip = false
					break
				}
			}
			if skip {
				continue
			}
		}

		if opt.Cached {
			fs = append(fs, entry)
			continue
		}
		if opt.Deleted {
			if !f.Exists() {
				fs = append(fs, entry)
				continue
			}
		}

		if opt.Unmerged && entry.Stage() != Stage0 {
			fs = append(fs, entry)
			continue
		}

		if opt.Modified {
			if f.IsDir() {
				fs = append(fs, entry)
				continue
			}
			_, err := f.Stat()
			// The file being deleted means it was modified
			if os.IsNotExist(err) {
				fs = append(fs, entry)
				continue
			} else if err != nil {
				return nil, err
			}

			// We've done everything we can to avoid hashing the file, but now
			// we need to to avoid the case where someone changes a file, then
			// changes it back to the original contents
			hash, _, err := HashFile("blob", f.String())
			if err != nil {
				return nil, err
			}
			if hash != entry.Sha1 {
				fs = append(fs, entry)
			}
		}
	}

	if opt.ErrorUnmatch {
		for _, file := range files {
			indexPath, err := file.IndexPath(c)
			if err != nil {
				return nil, err
			}
			if _, ok := filesInIndex[indexPath]; !ok {
				fmt.Printf("%v", filesInIndex)
				return nil, fmt.Errorf("error: pathspec '%v' did not match any file(s) known to git", file)
			}
		}
	}

	if opt.Others {
		wd := File(c.WorkDir)

		ignorePatterns := []IgnorePattern{}

		if opt.ExcludeStandard {
			opt.ExcludeFiles = append(opt.ExcludeFiles, File(filepath.Join(c.GitDir.String(), "info/exclude")))
			opt.ExcludePerDirectory = append(opt.ExcludePerDirectory, ".gitignore")
		}

		for _, file := range opt.ExcludeFiles {
			patterns, err := ParseIgnorePatterns(c, file, "")
			if err != nil {
				return nil, err
			}
			ignorePatterns = append(ignorePatterns, patterns...)
		}

		for _, pattern := range opt.ExcludePatterns {
			ignorePatterns = append(ignorePatterns, IgnorePattern{Pattern: pattern, Source: "", LineNum: 1, Scope: ""})
		}

		others := findUntrackedFilesFromDir(c, opt, wd+"/", wd, wd, filesInIndex, !opt.Directory, ignorePatterns)
		for _, file := range others {
			f, err := file.PathName.FilePath(c)
			if err != nil {
				return nil, err
			}

			if strings.HasPrefix(f.String(), "../") || len(files) > 0 {
				skip := true
				for _, explicit := range files {
					eAbs, err := filepath.Abs(explicit.String())
					if err != nil {
						return nil, err
					}
					fAbs, err := filepath.Abs(f.String())
					if err != nil {
						return nil, err
					}
					if fAbs == eAbs || strings.HasPrefix(fAbs, eAbs+"/") {
						skip = false
						break
					}
				}
				if skip {
					continue
				}
			}

			if f.IsDir() && opt.Directory {
				if opt.NoEmptyDirectory {
					if files, err := ioutil.ReadDir(f.String()); len(files) == 0 && err == nil {
						continue
					}
				}
				f += "/"
			}

			fs = append(fs, file)
		}
	}

	sort.Sort(ByPath(fs))
	return fs, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clientv3 implements the official Go etcd client for v3.
//
// Create client using `clientv3.New`:
//
//	// expect dial time-out on ipv4 blackhole
//	_, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"http://254.0.0.1:12345"},
//		DialTimeout: 2 * time.Second
//	})
//
//	// etcd clientv3 >= v3.2.10, grpc/grpc-go >= v1.7.3
//	if err == context.DeadlineExceeded {
//		// handle errors
//	}
//
//	// etcd clientv3 <= v3.2.9, grpc/grpc-go <= v1.2.1
//	if err == grpc.ErrClientConnTimeout {
//		// handle errors
//	}
//
//	cli, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
//		DialTimeout: 5 * time.Second,
//	})
//	if err != nil {
//		// handle error!
//	}
//	defer cli.Close()
//
// Make sure to close the client after using it. If the client is not closed, the
// connection will have leaky goroutines.
//
// To specify a client request timeout, wrap the context with context.WithTimeout:
//
//	ctx, cancel := context.WithTimeout(context.Background(), timeout)
//	resp, err := kvc.Put(ctx, "sample_key", "sample_value")
//	cancel()
//	if err != nil {
//	    // handle error!
//	}
//	// use the response
//
// The Client has internal state (watchers and leases), so Clients should be reused instead of created as needed.
// Clients are safe for concurrent use by multiple goroutines.
//
// etcd client returns 3 types of errors:
//
//  1. context error: canceled or deadline exceeded.
//  2. gRPC status error: e.g. when clock drifts in server-side before client's context deadline exceeded.
//  3. gRPC error: see https://go.etcd.io/etcd/blob/master/etcdserver/api/v3rpc/rpctypes/error.go
//
// Here is the example code to handle client errors:
//
//	resp, err := kvc.Put(ctx, "", "")
//	if err != nil {
//		if err == context.Canceled {
//			// ctx is canceled by another routine
//		} else if err == context.DeadlineExceeded {
//			// ctx is attached with a deadline and it exceeded
//		} else if err == rpctypes.ErrEmptyKey {
//			// client-side error: key is not provided
//		} else if ev, ok := status.FromError(err); ok {
//			code := ev.Code()
//			if code == codes.DeadlineExceeded {
//				// server-side context might have timed-out first (due to clock skew)
//				// while original client-side context is not timed-out yet
//			}
//		} else {
//			// bad cluster endpoints, which are not etcd servers
//		}
//	}
//
//	go func() { cli.Close() }()
//	_, err := kvc.Get(ctx, "a")
//	if err != nil {
//		// with etcd clientv3 <= v3.3
//		if err == context.Canceled {
//			// grpc balancer calls 'Get' with an inflight client.Close
//		} else if err == grpc.ErrClientConnClosing {
//			// grpc balancer calls 'Get' after client.Close.
//		}
//		// with etcd clientv3 >= v3.4
//		if clientv3.IsConnCanceled(err) {
//			// gRPC client connection is closed
//		}
//	}
//
// The grpc load balancer is registered statically and is shared across etcd clients.
// To enable detailed load balancer logging, set the ETCD_CLIENT_DEBUG environment
// variable.  E.g. "ETCD_CLIENT_DEBUG=1".
//
package clientv3
<commit_msg>clientv3/doc: Fix code example<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clientv3 implements the official Go etcd client for v3.
//
// Create client using `clientv3.New`:
//
//	// expect dial time-out on ipv4 blackhole
//	_, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"http://254.0.0.1:12345"},
//		DialTimeout: 2 * time.Second,
//	})
//
//	// etcd clientv3 >= v3.2.10, grpc/grpc-go >= v1.7.3
//	if err == context.DeadlineExceeded {
//		// handle errors
//	}
//
//	// etcd clientv3 <= v3.2.9, grpc/grpc-go <= v1.2.1
//	if err == grpc.ErrClientConnTimeout {
//		// handle errors
//	}
//
//	cli, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
//		DialTimeout: 5 * time.Second,
//	})
//	if err != nil {
//		// handle error!
//	}
//	defer cli.Close()
//
// Make sure to close the client after using it. If the client is not closed, the
// connection will have leaky goroutines.
//
// To specify a client request timeout, wrap the context with context.WithTimeout:
//
//	ctx, cancel := context.WithTimeout(context.Background(), timeout)
//	resp, err := kvc.Put(ctx, "sample_key", "sample_value")
//	cancel()
//	if err != nil {
//	    // handle error!
//	}
//	// use the response
//
// The Client has internal state (watchers and leases), so Clients should be reused instead of created as needed.
// Clients are safe for concurrent use by multiple goroutines.
//
// etcd client returns 3 types of errors:
//
//  1. context error: canceled or deadline exceeded.
//  2. gRPC status error: e.g. when clock drifts in server-side before client's context deadline exceeded.
//  3. gRPC error: see https://go.etcd.io/etcd/blob/master/etcdserver/api/v3rpc/rpctypes/error.go
//
// Here is the example code to handle client errors:
//
//	resp, err := kvc.Put(ctx, "", "")
//	if err != nil {
//		if err == context.Canceled {
//			// ctx is canceled by another routine
//		} else if err == context.DeadlineExceeded {
//			// ctx is attached with a deadline and it exceeded
//		} else if err == rpctypes.ErrEmptyKey {
//			// client-side error: key is not provided
//		} else if ev, ok := status.FromError(err); ok {
//			code := ev.Code()
//			if code == codes.DeadlineExceeded {
//				// server-side context might have timed-out first (due to clock skew)
//				// while original client-side context is not timed-out yet
//			}
//		} else {
//			// bad cluster endpoints, which are not etcd servers
//		}
//	}
//
//	go func() { cli.Close() }()
//	_, err := kvc.Get(ctx, "a")
//	if err != nil {
//		// with etcd clientv3 <= v3.3
//		if err == context.Canceled {
//			// grpc balancer calls 'Get' with an inflight client.Close
//		} else if err == grpc.ErrClientConnClosing {
//			// grpc balancer calls 'Get' after client.Close.
//		}
//		// with etcd clientv3 >= v3.4
//		if clientv3.IsConnCanceled(err) {
//			// gRPC client connection is closed
//		}
//	}
//
// The grpc load balancer is registered statically and is shared across etcd clients.
// To enable detailed load balancer logging, set the ETCD_CLIENT_DEBUG environment
// variable.  E.g. "ETCD_CLIENT_DEBUG=1".
//
package clientv3
<|endoftext|>


<commit_before>package mapreduce

import (
	"encoding/json"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/pachyderm-io/pfs/lib/btrfs"
	"github.com/samalba/dockerclient"
)

var retries int = 5

// StartContainer pulls image and starts a container from it with command. It
// returns the container id or an error.
func spinupContainer(image string, command []string) (string, error) {
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return "", err
	}
	if err := docker.PullImage(image, nil); err != nil {
		return "", err
	}

	containerConfig := &dockerclient.ContainerConfig{Image: image, Cmd: command}

	containerId, err := docker.CreateContainer(containerConfig, "")
	if err != nil {
		return "", nil
	}

	if err := docker.StartContainer(containerId, &dockerclient.HostConfig{}); err != nil {
		return "", err
	}

	return containerId, nil
}

func ipAddr(containerId string) (string, error) {
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return "", err
	}
	containerInfo, err := docker.InspectContainer(containerId)
	if err != nil {
		return "", err
	}

	return containerInfo.NetworkSettings.IpAddress, nil
}

func retry(f func() error, retries int, pause time.Duration) error {
	var err error
	for i := 0; i < retries; i++ {
		err = f()
		if err == nil {
			break
		} else {
			time.Sleep(pause)
		}
	}
	return err
}

type Job struct {
	Input     string   `json:"input"`
	Container string   `json:"container"`
	Command   []string `json:"command"`
}

// contains checks if set contains val. It assums that set has already been
// sorted.
func contains(set []string, val string) bool {
	index := sort.SearchStrings(set, val)
	return index < len(set) && set[index] == val
}

// filterPrefix returns the strings in set which are prefixed by prefix
func filterPrefix(set []string, prefix string) []string {
	rightBoundSearcher := func(i int) bool {
		return strings.HasPrefix(set[i], prefix) || set[i] < prefix
	}
	return set[sort.SearchStrings(set, prefix):sort.Search(len(set), rightBoundSearcher)]
}

// Materialize parses the jobs found in `in_repo`/`commit`/`jobDir` runs them
// with `in_repo/commit` as input, outputs the results to `out_repo`/`branch`
// and commits them as `out_repo`/`commit`
func Materialize(in_repo, branch, commit, out_repo, jobDir string) error {
	log.Printf("Materialize: %s %s %s %s %s.", in_repo, branch, commit, out_repo, jobDir)
	// We make sure that this function always commits so that we know the comp
	// repo stays in sync with the data repo.
	defer func() {
		if err := btrfs.Commit(out_repo, commit, branch); err != nil {
			log.Print("btrfs.Commit error in Materliaze: ", err)
		}
	}()
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return err
	}
	exists, err := btrfs.FileExists(path.Join(in_repo, commit, jobDir))
	if err != nil {
		return err
	}
	if !exists {
		// Perfectly valid to have no jobs dir, it just means we have no work
		// to do.
		return nil
	}
	newFiles, err := btrfs.NewFiles(in_repo, commit)
	if err != nil {
		return err
	}
	sort.Strings(newFiles)

	jobsPath := path.Join(in_repo, commit, jobDir)
	jobs, err := btrfs.ReadDir(jobsPath)
	if err != nil {
		return err
	}
	for _, jobInfo := range jobs {
		jobFile, err := btrfs.Open(path.Join(jobsPath, jobInfo.Name()))
		if err != nil {
			return err
		}
		defer jobFile.Close()
		decoder := json.NewDecoder(jobFile)
		j := &Job{}
		if err = decoder.Decode(j); err != nil {
			return err
		}

		var inFiles []os.FileInfo

		log.Print("newFiles: ", newFiles)
		if contains(newFiles, path.Join(jobDir, jobInfo.Name())) {
			// This is a brand new job. We need to run every single file in `input`
			log.Printf("Brand new job %s, running it on everything.", jobInfo.Name())
			inFiles, err = btrfs.ReadDir(path.Join(in_repo, commit, j.Input))
			if err != nil {
				return err
			}
		} else {
			// This isn't a new job, only new files need to be run through it
			log.Printf("Old job %s, running it on new stuff.", jobInfo.Name())
			allInFiles, err := btrfs.ReadDir(path.Join(in_repo, commit, j.Input))
			if err != nil {
				return err
			}
			for _, f := range allInFiles {
				if contains(newFiles, f.Name()) {
					inFiles = append(inFiles, f)
				}
			}
		}

		log.Print("inFiles is: ", inFiles)

		if len(inFiles) == 0 {
			continue
		}

		containerId, err := spinupContainer(j.Container, j.Command)
		if err != nil {
			return err
		}
		defer docker.StopContainer(containerId, 5)

		containerAddr, err := ipAddr(containerId)
		if err != nil {
			return err
		}

		var wg sync.WaitGroup
		defer wg.Wait()
		for _, inF := range inFiles {
			wg.Add(1)
			go func() {
				defer wg.Done()
				inFile, err := btrfs.Open(path.Join(in_repo, commit, j.Input, inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer inFile.Close()

				var resp *http.Response
				err = retry(func() error {
					resp, err = http.Post("http://"+path.Join(containerAddr, inF.Name()), "application/text", inFile)
					return err
				}, 5, 200*time.Millisecond)
				if err != nil {
					log.Print(err)
					return
				}
				defer resp.Body.Close()

				exists, err := btrfs.FileExists(path.Join(out_repo, branch))
				if err != nil {
					log.Print(err)
					return
				}
				if !exists {
					log.Printf("Invalid state. %s should already exists.", path.Join(out_repo, branch))
					return
				} else {
					if err := btrfs.MkdirAll(path.Join(out_repo, branch, jobInfo.Name())); err != nil {
						log.Print(err)
						return
					}
				}

				outFile, err := btrfs.Create(path.Join(out_repo, branch, jobInfo.Name(), inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer outFile.Close()
				if _, err := io.Copy(outFile, resp.Body); err != nil {
					log.Print(err)
					return
				}
			}()
		}
	}
	return nil
}
<commit_msg>Whoops I made an unsafe closure. :-(<commit_after>package mapreduce

import (
	"encoding/json"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/pachyderm-io/pfs/lib/btrfs"
	"github.com/samalba/dockerclient"
)

var retries int = 5

// StartContainer pulls image and starts a container from it with command. It
// returns the container id or an error.
func spinupContainer(image string, command []string) (string, error) {
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return "", err
	}
	if err := docker.PullImage(image, nil); err != nil {
		return "", err
	}

	containerConfig := &dockerclient.ContainerConfig{Image: image, Cmd: command}

	containerId, err := docker.CreateContainer(containerConfig, "")
	if err != nil {
		return "", nil
	}

	if err := docker.StartContainer(containerId, &dockerclient.HostConfig{}); err != nil {
		return "", err
	}

	return containerId, nil
}

func ipAddr(containerId string) (string, error) {
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return "", err
	}
	containerInfo, err := docker.InspectContainer(containerId)
	if err != nil {
		return "", err
	}

	return containerInfo.NetworkSettings.IpAddress, nil
}

func retry(f func() error, retries int, pause time.Duration) error {
	var err error
	for i := 0; i < retries; i++ {
		err = f()
		if err == nil {
			break
		} else {
			time.Sleep(pause)
		}
	}
	return err
}

type Job struct {
	Input     string   `json:"input"`
	Container string   `json:"container"`
	Command   []string `json:"command"`
}

// contains checks if set contains val. It assums that set has already been
// sorted.
func contains(set []string, val string) bool {
	index := sort.SearchStrings(set, val)
	return index < len(set) && set[index] == val
}

// filterPrefix returns the strings in set which are prefixed by prefix
func filterPrefix(set []string, prefix string) []string {
	rightBoundSearcher := func(i int) bool {
		return strings.HasPrefix(set[i], prefix) || set[i] < prefix
	}
	return set[sort.SearchStrings(set, prefix):sort.Search(len(set), rightBoundSearcher)]
}

// Materialize parses the jobs found in `in_repo`/`commit`/`jobDir` runs them
// with `in_repo/commit` as input, outputs the results to `out_repo`/`branch`
// and commits them as `out_repo`/`commit`
func Materialize(in_repo, branch, commit, out_repo, jobDir string) error {
	log.Printf("Materialize: %s %s %s %s %s.", in_repo, branch, commit, out_repo, jobDir)
	// We make sure that this function always commits so that we know the comp
	// repo stays in sync with the data repo.
	defer func() {
		if err := btrfs.Commit(out_repo, commit, branch); err != nil {
			log.Print("btrfs.Commit error in Materliaze: ", err)
		}
	}()
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return err
	}
	exists, err := btrfs.FileExists(path.Join(in_repo, commit, jobDir))
	if err != nil {
		return err
	}
	if !exists {
		// Perfectly valid to have no jobs dir, it just means we have no work
		// to do.
		return nil
	}
	newFiles, err := btrfs.NewFiles(in_repo, commit)
	if err != nil {
		return err
	}
	sort.Strings(newFiles)

	jobsPath := path.Join(in_repo, commit, jobDir)
	jobs, err := btrfs.ReadDir(jobsPath)
	if err != nil {
		return err
	}
	for _, jobInfo := range jobs {
		jobFile, err := btrfs.Open(path.Join(jobsPath, jobInfo.Name()))
		if err != nil {
			return err
		}
		defer jobFile.Close()
		decoder := json.NewDecoder(jobFile)
		j := &Job{}
		if err = decoder.Decode(j); err != nil {
			return err
		}

		var inFiles []os.FileInfo

		log.Print("newFiles: ", newFiles)
		if contains(newFiles, path.Join(jobDir, jobInfo.Name())) {
			// This is a brand new job. We need to run every single file in `input`
			log.Printf("Brand new job %s, running it on everything.", jobInfo.Name())
			inFiles, err = btrfs.ReadDir(path.Join(in_repo, commit, j.Input))
			if err != nil {
				return err
			}
		} else {
			// This isn't a new job, only new files need to be run through it
			log.Printf("Old job %s, running it on new stuff.", jobInfo.Name())
			allInFiles, err := btrfs.ReadDir(path.Join(in_repo, commit, j.Input))
			if err != nil {
				return err
			}
			for _, f := range allInFiles {
				if contains(newFiles, f.Name()) {
					inFiles = append(inFiles, f)
				}
			}
		}

		log.Print("inFiles is: ", inFiles)

		if len(inFiles) == 0 {
			continue
		}

		containerId, err := spinupContainer(j.Container, j.Command)
		if err != nil {
			return err
		}
		defer docker.StopContainer(containerId, 5)

		containerAddr, err := ipAddr(containerId)
		if err != nil {
			return err
		}

		var wg sync.WaitGroup
		defer wg.Wait()
		for _, inF := range inFiles {
			wg.Add(1)
			go func(inF os.FileInfo) {
				defer wg.Done()
				inFile, err := btrfs.Open(path.Join(in_repo, commit, j.Input, inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer inFile.Close()

				var resp *http.Response
				err = retry(func() error {
					log.Print("Posting: ", inF.Name())
					resp, err = http.Post("http://"+path.Join(containerAddr, inF.Name()), "application/text", inFile)
					return err
				}, 5, 200*time.Millisecond)
				if err != nil {
					log.Print(err)
					return
				}
				defer resp.Body.Close()

				exists, err := btrfs.FileExists(path.Join(out_repo, branch))
				if err != nil {
					log.Print(err)
					return
				}
				if !exists {
					log.Printf("Invalid state. %s should already exists.", path.Join(out_repo, branch))
					return
				} else {
					if err := btrfs.MkdirAll(path.Join(out_repo, branch, jobInfo.Name())); err != nil {
						log.Print(err)
						return
					}
				}

				outFile, err := btrfs.Create(path.Join(out_repo, branch, jobInfo.Name(), inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer outFile.Close()
				if _, err := io.Copy(outFile, resp.Body); err != nil {
					log.Print(err)
					return
				}
			}(inF)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package slack

import (
	"encoding/json"
	"fmt"
	"github.com/alinpopa/barvin/data"
	"golang.org/x/net/websocket"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"time"
)

func startRtm(origin string) data.RtmResponse {
	resp, err := http.Get(origin)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	var data data.RtmResponse
	json.NewDecoder(resp.Body).Decode(&data)
	return data
}

func connectWs(url string, origin string) (*websocket.Conn, error) {
	return websocket.Dial(url, "", origin)
}

func currentIpMessage(prefix string) data.WsMessage {
	ipResp, err := http.Get("https://api.ipify.org?format=json")
	if err != nil {
		fmt.Println("Error", err)
		return data.WsMessage{Msg: fmt.Sprintf("Error: %s", err)}
	}
	defer ipResp.Body.Close()
	var ipInfo data.IpInfo
	json.NewDecoder(ipResp.Body).Decode(&ipInfo)
	if len(prefix) > 0 {
		return data.WsMessage{Msg: prefix + ":" + ipInfo.Ip}
	}
	return data.WsMessage{Msg: ipInfo.Ip}
}

func replyMessage(ws *websocket.Conn, event data.WsEvent, msg string) error {
	return websocket.JSON.Send(ws, &data.WsEvent{
		Id:      event.Id,
		Type:    "message",
		Channel: event.Channel,
		Text:    msg,
		User:    event.User,
	})
}

func sendPrvMessage(to string, msg string, token string) error {
	rsp, err := http.PostForm("https://slack.com/api/chat.postMessage?token="+token, url.Values{"channel": {to}, "as_user": {"true"}, "text": {msg}})
	fmt.Println("Got resp:", rsp)
	fmt.Println("Got err:", err)
	if rsp != nil {
		rsp.Body.Close()
	}
	return err
}

func restart(msg string, err error, c chan<- string) {
	var m string
	if err != nil {
		m = fmt.Sprintf("%s[%s]", msg, err)
	} else {
		m = msg
	}
	fmt.Println(m)
	go func() {
		c <- m
	}()
}

func SlackHandler(initMessage string, restartChannel chan<- string, userId string, token string) {
	origin := "https://slack.com/api/rtm.start?token=" + token
	rtm := startRtm(origin)
	fmt.Println(rtm.Url)
	ws, err := connectWs(rtm.Url, origin)
	for err != nil {
		fmt.Println(">>> Got error; trying to reconnect to WS", err)
		time.Sleep(35 * time.Second)
		rtm := startRtm(origin)
		ws, err = connectWs(rtm.Url, origin)
	}
	sendPrvMessage(userId, currentIpMessage(initMessage).Msg, token)
	for {
		var msg string
		var event data.WsEvent
		err := websocket.Message.Receive(ws, &msg)
		if err != nil {
			restart("Error while receiving message", err, restartChannel)
			break
		}
		unmarshallErr := json.Unmarshal([]byte(msg), &event)
		if unmarshallErr != nil {
			fmt.Println("Error while unmarshaling message:", msg)
			restart("Error unmarshalling message", unmarshallErr, restartChannel)
			break
		}
		fmt.Println("Raw message:", msg)
		fmt.Printf("Got event %+v\n", event)
		if strings.ToLower(event.Text) == "ip" && event.User == userId {
			replyMessage(ws, event, currentIpMessage("").Msg)
		}
	}
}
<commit_msg>Remove unused runtime import.<commit_after>package slack

import (
	"encoding/json"
	"fmt"
	"github.com/alinpopa/barvin/data"
	"golang.org/x/net/websocket"
	"net/http"
	"net/url"
	"strings"
	"time"
)

func startRtm(origin string) data.RtmResponse {
	resp, err := http.Get(origin)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	var data data.RtmResponse
	json.NewDecoder(resp.Body).Decode(&data)
	return data
}

func connectWs(url string, origin string) (*websocket.Conn, error) {
	return websocket.Dial(url, "", origin)
}

func currentIpMessage(prefix string) data.WsMessage {
	ipResp, err := http.Get("https://api.ipify.org?format=json")
	if err != nil {
		fmt.Println("Error", err)
		return data.WsMessage{Msg: fmt.Sprintf("Error: %s", err)}
	}
	defer ipResp.Body.Close()
	var ipInfo data.IpInfo
	json.NewDecoder(ipResp.Body).Decode(&ipInfo)
	if len(prefix) > 0 {
		return data.WsMessage{Msg: prefix + ":" + ipInfo.Ip}
	}
	return data.WsMessage{Msg: ipInfo.Ip}
}

func replyMessage(ws *websocket.Conn, event data.WsEvent, msg string) error {
	return websocket.JSON.Send(ws, &data.WsEvent{
		Id:      event.Id,
		Type:    "message",
		Channel: event.Channel,
		Text:    msg,
		User:    event.User,
	})
}

func sendPrvMessage(to string, msg string, token string) error {
	rsp, err := http.PostForm("https://slack.com/api/chat.postMessage?token="+token, url.Values{"channel": {to}, "as_user": {"true"}, "text": {msg}})
	fmt.Println("Got resp:", rsp)
	fmt.Println("Got err:", err)
	if rsp != nil {
		rsp.Body.Close()
	}
	return err
}

func restart(msg string, err error, c chan<- string) {
	var m string
	if err != nil {
		m = fmt.Sprintf("%s[%s]", msg, err)
	} else {
		m = msg
	}
	fmt.Println(m)
	go func() {
		c <- m
	}()
}

func SlackHandler(initMessage string, restartChannel chan<- string, userId string, token string) {
	origin := "https://slack.com/api/rtm.start?token=" + token
	rtm := startRtm(origin)
	fmt.Println(rtm.Url)
	ws, err := connectWs(rtm.Url, origin)
	for err != nil {
		fmt.Println(">>> Got error; trying to reconnect to WS", err)
		time.Sleep(35 * time.Second)
		rtm := startRtm(origin)
		ws, err = connectWs(rtm.Url, origin)
	}
	sendPrvMessage(userId, currentIpMessage(initMessage).Msg, token)
	for {
		var msg string
		var event data.WsEvent
		err := websocket.Message.Receive(ws, &msg)
		if err != nil {
			restart("Error while receiving message", err, restartChannel)
			break
		}
		unmarshallErr := json.Unmarshal([]byte(msg), &event)
		if unmarshallErr != nil {
			fmt.Println("Error while unmarshaling message:", msg)
			restart("Error unmarshalling message", unmarshallErr, restartChannel)
			break
		}
		fmt.Println("Raw message:", msg)
		fmt.Printf("Got event %+v\n", event)
		if strings.ToLower(event.Text) == "ip" && event.User == userId {
			replyMessage(ws, event, currentIpMessage("").Msg)
		}
	}
}
<|endoftext|>