<commit_before>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
)

func init() {
	Commands = append(Commands, &commandRemoteUnmount{})
}

type commandRemoteUnmount struct {
}

func (c *commandRemoteUnmount) Name() string {
	return "remote.unmount"
}

func (c *commandRemoteUnmount) Help() string {
	return `unmount remote storage

	# assume a remote storage is configured to name "s3_1"
	remote.configure -name=s3_1 -type=s3 -access_key=xxx -secret_key=yyy
	# mount and pull one bucket
	remote.mount -dir=/xxx -remote=s3_1/bucket

	# unmount the mounted directory and remove its cache
	remote.unmount -dir=/xxx

`
}

func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteMountCommand.String("dir", "", "a directory in filer")

	if err = remoteMountCommand.Parse(args); err != nil {
		return nil
	}

	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
	if listErr != nil {
		return listErr
	}
	if *dir == "" {
		return jsonPrintln(writer, mappings)
	}

	_, found := mappings.Mappings[*dir]
	if !found {
		return fmt.Errorf("directory %s is not mounted", *dir)
	}

	// purge mounted data
	if err = c.purgeMountedData(commandEnv, *dir); err != nil {
		return fmt.Errorf("purge mounted data: %v", err)
	}

	// store a mount configuration in filer
	if err = c.deleteMountMapping(commandEnv, *dir); err != nil {
		return fmt.Errorf("delete mount mapping: %v", err)
	}

	return nil
}

func (c *commandRemoteUnmount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *remote_pb.RemoteStorageLocation) (conf *remote_pb.RemoteConf, err error) {

	return filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote.Name)

}

func (c *commandRemoteUnmount) purgeMountedData(commandEnv *CommandEnv, dir string) error {

	// find existing directory, and ensure the directory is empty
	err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		parent, name := util.FullPath(dir).DirAndName()
		lookupResp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
			Directory: parent,
			Name:      name,
		})
		if lookupErr != nil {
			return fmt.Errorf("lookup %s: %v", dir, lookupErr)
		}

		oldEntry := lookupResp.Entry

		deleteError := filer_pb.DoRemove(client, parent, name, true, true, true, false, nil)
		if deleteError != nil {
			return fmt.Errorf("delete %s: %v", dir, deleteError)
		}

		mkdirErr := filer_pb.DoMkdir(client, parent, name, func(entry *filer_pb.Entry) {
			entry.Attributes = oldEntry.Attributes
			entry.Extended = oldEntry.Extended
		})
		if mkdirErr != nil {
			return fmt.Errorf("mkdir %s: %v", dir, mkdirErr)
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

func (c *commandRemoteUnmount) deleteMountMapping(commandEnv *CommandEnv, dir string) (err error) {

	// read current mapping
	var oldContent, newContent []byte
	err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE)
		return err
	})
	if err != nil {
		if err != filer_pb.ErrNotFound {
			return fmt.Errorf("read existing mapping: %v", err)
		}
	}

	// add new mapping
	newContent, err = filer.RemoveRemoteStorageMapping(oldContent, dir)
	if err != nil {
		return fmt.Errorf("delete mount %s: %v", dir, err)
	}

	// save back
	err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE, newContent)
	})
	if err != nil {
		return fmt.Errorf("save mapping: %v", err)
	}

	return nil
}
<commit_msg>add warning on unmount a folder<commit_after>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
)

func init() {
	Commands = append(Commands, &commandRemoteUnmount{})
}

type commandRemoteUnmount struct {
}

func (c *commandRemoteUnmount) Name() string {
	return "remote.unmount"
}

func (c *commandRemoteUnmount) Help() string {
	return `unmount remote storage

	# assume a remote storage is configured to name "s3_1"
	remote.configure -name=s3_1 -type=s3 -access_key=xxx -secret_key=yyy
	# mount and pull one bucket
	remote.mount -dir=/xxx -remote=s3_1/bucket

	# unmount the mounted directory and remove its cache
	# Make sure you have stopped "weed filer.remote.sync" first!
	# Otherwise, the deletion will also be propagated to the remote storage!!!
	remote.unmount -dir=/xxx -iHaveStoppedRemoteSync

`
}

func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteMountCommand.String("dir", "", "a directory in filer")
	hasStoppedRemoteSync := remoteMountCommand.Bool("iHaveStoppedRemoteSync", false, "confirm to stop weed filer.remote.sync first")

	if err = remoteMountCommand.Parse(args); err != nil {
		return nil
	}

	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
	if listErr != nil {
		return listErr
	}
	if *dir == "" {
		return jsonPrintln(writer, mappings)
	}

	_, found := mappings.Mappings[*dir]
	if !found {
		return fmt.Errorf("directory %s is not mounted", *dir)
	}

	if !*hasStoppedRemoteSync {
		return fmt.Errorf("make sure \"weed filer.remote.sync\" is stopped to avoid data loss")
	}
	// purge mounted data
	if err = c.purgeMountedData(commandEnv, *dir); err != nil {
		return fmt.Errorf("purge mounted data: %v", err)
	}

	// store a mount configuration in filer
	if err = c.deleteMountMapping(commandEnv, *dir); err != nil {
		return fmt.Errorf("delete mount mapping: %v", err)
	}

	return nil
}

func (c *commandRemoteUnmount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *remote_pb.RemoteStorageLocation) (conf *remote_pb.RemoteConf, err error) {

	return filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote.Name)

}

func (c *commandRemoteUnmount) purgeMountedData(commandEnv *CommandEnv, dir string) error {

	// find existing directory, and ensure the directory is empty
	err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		parent, name := util.FullPath(dir).DirAndName()
		lookupResp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
			Directory: parent,
			Name:      name,
		})
		if lookupErr != nil {
			return fmt.Errorf("lookup %s: %v", dir, lookupErr)
		}

		oldEntry := lookupResp.Entry

		deleteError := filer_pb.DoRemove(client, parent, name, true, true, true, false, nil)
		if deleteError != nil {
			return fmt.Errorf("delete %s: %v", dir, deleteError)
		}

		mkdirErr := filer_pb.DoMkdir(client, parent, name, func(entry *filer_pb.Entry) {
			entry.Attributes = oldEntry.Attributes
			entry.Extended = oldEntry.Extended
		})
		if mkdirErr != nil {
			return fmt.Errorf("mkdir %s: %v", dir, mkdirErr)
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

func (c *commandRemoteUnmount) deleteMountMapping(commandEnv *CommandEnv, dir string) (err error) {

	// read current mapping
	var oldContent, newContent []byte
	err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE)
		return err
	})
	if err != nil {
		if err != filer_pb.ErrNotFound {
			return fmt.Errorf("read existing mapping: %v", err)
		}
	}

	// add new mapping
	newContent, err = filer.RemoveRemoteStorageMapping(oldContent, dir)
	if err != nil {
		return fmt.Errorf("delete mount %s: %v", dir, err)
	}

	// save back
	err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE, newContent)
	})
	if err != nil {
		return fmt.Errorf("save mapping: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build example jsgo

package main

import (
	"bytes"
	"fmt"
	"image"
	"image/color"
	_ "image/png"
	"log"
	"math"
	"math/rand"
	"time"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/audio"
	"github.com/hajimehoshi/ebiten/audio/vorbis"
	"github.com/hajimehoshi/ebiten/audio/wav"
	"github.com/hajimehoshi/ebiten/ebitenutil"
	raudio "github.com/hajimehoshi/ebiten/examples/resources/audio"
	"github.com/hajimehoshi/ebiten/examples/resources/fonts"
	resources "github.com/hajimehoshi/ebiten/examples/resources/images/flappy"
	"github.com/hajimehoshi/ebiten/inpututil"
	"github.com/hajimehoshi/ebiten/text"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func floorDiv(x, y int) int {
	d := x / y
	if d*y == x || x >= 0 {
		return d
	}
	return d - 1
}

func floorMod(x, y int) int {
	return x - floorDiv(x, y)*y
}

const (
	screenWidth      = 640
	screenHeight     = 480
	tileSize         = 32
	fontSize         = 32
	pipeWidth        = tileSize * 2
	pipeStartOffsetX = 8
	pipeIntervalX    = 8
	pipeGapY         = 5
)

var (
	gopherImage *ebiten.Image
	tilesImage  *ebiten.Image
	arcadeFont  font.Face
)

func init() {
	img, _, err := image.Decode(bytes.NewReader(resources.Gopher_png))
	if err != nil {
		log.Fatal(err)
	}
	gopherImage, _ = ebiten.NewImageFromImage(img, ebiten.FilterDefault)

	img, _, err = image.Decode(bytes.NewReader(resources.Tiles_png))
	if err != nil {
		log.Fatal(err)
	}
	tilesImage, _ = ebiten.NewImageFromImage(img, ebiten.FilterDefault)
}

func init() {
	tt, err := truetype.Parse(fonts.ArcadeN_ttf)
	if err != nil {
		log.Fatal(err)
	}
	const dpi = 72
	arcadeFont = truetype.NewFace(tt, &truetype.Options{
		Size:    fontSize,
		DPI:     dpi,
		Hinting: font.HintingFull,
	})
}

var (
	audioContext *audio.Context
	jumpPlayer   *audio.Player
	hitPlayer    *audio.Player
)

func init() {
	audioContext, _ = audio.NewContext(44100)

	jumpD, err := vorbis.Decode(audioContext, audio.BytesReadSeekCloser(raudio.Jump_ogg))
	if err != nil {
		println("!?")
		log.Fatal(err)
	}
	jumpPlayer, err = audio.NewPlayer(audioContext, jumpD)
	if err != nil {
		log.Fatal(err)
	}

	jabD, err := wav.Decode(audioContext, audio.BytesReadSeekCloser(raudio.Jab_wav))
	if err != nil {
		log.Fatal(err)
	}
	hitPlayer, err = audio.NewPlayer(audioContext, jabD)
	if err != nil {
		log.Fatal(err)
	}
}

type Mode int

const (
	ModeTitle Mode = iota
	ModeGame
	ModeGameOver
)

type Game struct {
	mode Mode

	// The gopher's position
	x16  int
	y16  int
	vy16 int

	// Camera
	cameraX int
	cameraY int

	// Pipes
	pipeTileYs []int

	gameoverCount int
}

func NewGame() *Game {
	g := &Game{}
	g.init()
	return g
}

func (g *Game) init() {
	g.x16 = 0
	g.y16 = 100 * 16
	g.cameraX = -240
	g.cameraY = 0
	g.pipeTileYs = make([]int, 256)
	for i := range g.pipeTileYs {
		g.pipeTileYs[i] = rand.Intn(6) + 2
	}
}

func jump() bool {
	if inpututil.IsKeyJustPressed(ebiten.KeySpace) {
		return true
	}
	if inpututil.IsMouseButtonJustPressed(ebiten.MouseButtonLeft) {
		return true
	}
	// TODO: Is it correct to use ID '0' here?
	if inpututil.IsJustTouched(0) {
		return true
	}
	return false
}

func (g *Game) Update(screen *ebiten.Image) error {
	switch g.mode {
	case ModeTitle:
		if jump() {
			g.mode = ModeGame
		}
	case ModeGame:
		g.x16 += 32
		g.cameraX += 2
		if jump() {
			g.vy16 = -96
			jumpPlayer.Rewind()
			jumpPlayer.Play()
		}
		g.y16 += g.vy16

		// Gravity
		g.vy16 += 4
		if g.vy16 > 96 {
			g.vy16 = 96
		}

		if g.hit() {
			hitPlayer.Rewind()
			hitPlayer.Play()
			g.mode = ModeGameOver
			g.gameoverCount = 30
		}
	case ModeGameOver:
		if g.gameoverCount > 0 {
			g.gameoverCount--
		}
		if g.gameoverCount == 0 && jump() {
			g.init()
			g.mode = ModeTitle
		}
	}

	if ebiten.IsRunningSlowly() {
		return nil
	}

	screen.Fill(color.RGBA{0x80, 0xa0, 0xc0, 0xff})
	g.drawTiles(screen)
	if g.mode != ModeTitle {
		g.drawGopher(screen)
	}
	var texts []string
	switch g.mode {
	case ModeTitle:
		texts = []string{"FLAPPY GOPHER", "", "", "", "", "PRESS SPACE"}
	case ModeGameOver:
		texts = []string{"", "GAMEOVER!"}
	}
	for i, l := range texts {
		x := (screenWidth - len(l)*fontSize) / 2
		text.Draw(screen, l, arcadeFont, x, (i+4)*fontSize, color.White)
	}

	scoreStr := fmt.Sprintf("%04d", g.score())
	text.Draw(screen, scoreStr, arcadeFont, screenWidth-len(scoreStr)*fontSize, fontSize, color.White)
	ebitenutil.DebugPrint(screen, fmt.Sprintf("FPS: %0.2f", ebiten.CurrentFPS()))
	return nil
}

func (g *Game) pipeAt(tileX int) (tileY int, ok bool) {
	if (tileX - pipeStartOffsetX) <= 0 {
		return 0, false
	}
	if floorMod(tileX-pipeStartOffsetX, pipeIntervalX) != 0 {
		return 0, false
	}
	idx := floorDiv(tileX-pipeStartOffsetX, pipeIntervalX)
	return g.pipeTileYs[idx%len(g.pipeTileYs)], true
}

func (g *Game) score() int {
	x := floorDiv(g.x16, 16) / tileSize
	if (x - pipeStartOffsetX) <= 0 {
		return 0
	}
	return floorDiv(x-pipeStartOffsetX, pipeIntervalX)
}

func (g *Game) hit() bool {
	if g.mode != ModeGame {
		return false
	}
	const (
		gopherWidth  = 30
		gopherHeight = 60
	)
	w, h := gopherImage.Size()
	x0 := floorDiv(g.x16, 16) + (w-gopherWidth)/2
	y0 := floorDiv(g.y16, 16) + (h-gopherHeight)/2
	x1 := x0 + gopherWidth
	y1 := y0 + gopherHeight
	if y0 < -tileSize*4 {
		return true
	}
	if y1 >= screenHeight-tileSize {
		return true
	}
	xMin := floorDiv(x0-pipeWidth, tileSize)
	xMax := floorDiv(x0+gopherWidth, tileSize)
	for x := xMin; x <= xMax; x++ {
		y, ok := g.pipeAt(x)
		if !ok {
			continue
		}
		if x0 >= x*tileSize+pipeWidth {
			continue
		}
		if x1 < x*tileSize {
			continue
		}
		if y0 < y*tileSize {
			return true
		}
		if y1 >= (y+pipeGapY)*tileSize {
			return true
		}
	}
	return false
}

func (g *Game) drawTiles(screen *ebiten.Image) {
	const (
		nx           = screenWidth / tileSize
		ny           = screenHeight / tileSize
		pipeTileSrcX = 128
		pipeTileSrcY = 192
	)

	op := &ebiten.DrawImageOptions{}
	for i := -2; i < nx+1; i++ {
		// ground
		op.GeoM.Reset()
		op.GeoM.Translate(float64(i*tileSize-floorMod(g.cameraX, tileSize)),
			float64((ny-1)*tileSize-floorMod(g.cameraY, tileSize)))
		r := image.Rect(0, 0, tileSize, tileSize)
		op.SourceRect = &r
		screen.DrawImage(tilesImage, op)

		// pipe
		if tileY, ok := g.pipeAt(floorDiv(g.cameraX, tileSize) + i); ok {
			for j := 0; j < tileY; j++ {
				op.GeoM.Reset()
				op.GeoM.Scale(1, -1)
				op.GeoM.Translate(float64(i*tileSize-floorMod(g.cameraX, tileSize)),
					float64(j*tileSize-floorMod(g.cameraY, tileSize)))
				op.GeoM.Translate(0, tileSize)
				if j == tileY-1 {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY, pipeTileSrcX+tileSize*2, pipeTileSrcY+tileSize)
					op.SourceRect = &r
				} else {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY+tileSize, pipeTileSrcX+tileSize*2, pipeTileSrcY+tileSize*2)
					op.SourceRect = &r
				}
				screen.DrawImage(tilesImage, op)
			}
			for j := tileY + pipeGapY; j < screenHeight/tileSize-1; j++ {
				op.GeoM.Reset()
				op.GeoM.Translate(float64(i*tileSize-floorMod(g.cameraX, tileSize)),
					float64(j*tileSize-floorMod(g.cameraY, tileSize)))
				if j == tileY+pipeGapY {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY, pipeTileSrcX+pipeWidth, pipeTileSrcY+tileSize)
					op.SourceRect = &r
				} else {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY+tileSize, pipeTileSrcX+pipeWidth, pipeTileSrcY+tileSize+tileSize)
					op.SourceRect = &r
				}
				screen.DrawImage(tilesImage, op)
			}
		}
	}
}

func (g *Game) drawGopher(screen *ebiten.Image) {
	op := &ebiten.DrawImageOptions{}
	_, h := gopherImage.Size()
	op.GeoM.Translate(-float64(h)/2.0, -float64(h)/2.0)
	op.GeoM.Rotate(float64(g.vy16) / 96.0 * math.Pi / 6)
	op.GeoM.Translate(float64(h)/2.0, float64(h)/2.0)
	op.GeoM.Translate(float64(g.x16/16.0)-float64(g.cameraX), float64(g.y16/16.0)-float64(g.cameraY))
	op.Filter = ebiten.FilterLinear
	screen.DrawImage(gopherImage, op)
}

func main() {
	g := NewGame()
	if err := ebiten.Run(g.Update, screenWidth, screenHeight, 1, "Flappy Gopher (Ebiten Demo)"); err != nil {
		panic(err)
	}
}
<commit_msg>examples/flappy: Fullscreen on browsers<commit_after>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build example jsgo

package main

import (
	"bytes"
	"fmt"
	"image"
	"image/color"
	_ "image/png"
	"log"
	"math"
	"math/rand"
	"runtime"
	"time"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/audio"
	"github.com/hajimehoshi/ebiten/audio/vorbis"
	"github.com/hajimehoshi/ebiten/audio/wav"
	"github.com/hajimehoshi/ebiten/ebitenutil"
	raudio "github.com/hajimehoshi/ebiten/examples/resources/audio"
	"github.com/hajimehoshi/ebiten/examples/resources/fonts"
	resources "github.com/hajimehoshi/ebiten/examples/resources/images/flappy"
	"github.com/hajimehoshi/ebiten/inpututil"
	"github.com/hajimehoshi/ebiten/text"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func floorDiv(x, y int) int {
	d := x / y
	if d*y == x || x >= 0 {
		return d
	}
	return d - 1
}

func floorMod(x, y int) int {
	return x - floorDiv(x, y)*y
}

const (
	screenWidth      = 640
	screenHeight     = 480
	tileSize         = 32
	fontSize         = 32
	pipeWidth        = tileSize * 2
	pipeStartOffsetX = 8
	pipeIntervalX    = 8
	pipeGapY         = 5
)

var (
	gopherImage *ebiten.Image
	tilesImage  *ebiten.Image
	arcadeFont  font.Face
)

func init() {
	img, _, err := image.Decode(bytes.NewReader(resources.Gopher_png))
	if err != nil {
		log.Fatal(err)
	}
	gopherImage, _ = ebiten.NewImageFromImage(img, ebiten.FilterDefault)

	img, _, err = image.Decode(bytes.NewReader(resources.Tiles_png))
	if err != nil {
		log.Fatal(err)
	}
	tilesImage, _ = ebiten.NewImageFromImage(img, ebiten.FilterDefault)
}

func init() {
	tt, err := truetype.Parse(fonts.ArcadeN_ttf)
	if err != nil {
		log.Fatal(err)
	}
	const dpi = 72
	arcadeFont = truetype.NewFace(tt, &truetype.Options{
		Size:    fontSize,
		DPI:     dpi,
		Hinting: font.HintingFull,
	})
}

var (
	audioContext *audio.Context
	jumpPlayer   *audio.Player
	hitPlayer    *audio.Player
)

func init() {
	audioContext, _ = audio.NewContext(44100)

	jumpD, err := vorbis.Decode(audioContext, audio.BytesReadSeekCloser(raudio.Jump_ogg))
	if err != nil {
		println("!?")
		log.Fatal(err)
	}
	jumpPlayer, err = audio.NewPlayer(audioContext, jumpD)
	if err != nil {
		log.Fatal(err)
	}

	jabD, err := wav.Decode(audioContext, audio.BytesReadSeekCloser(raudio.Jab_wav))
	if err != nil {
		log.Fatal(err)
	}
	hitPlayer, err = audio.NewPlayer(audioContext, jabD)
	if err != nil {
		log.Fatal(err)
	}
}

type Mode int

const (
	ModeTitle Mode = iota
	ModeGame
	ModeGameOver
)

type Game struct {
	mode Mode

	// The gopher's position
	x16  int
	y16  int
	vy16 int

	// Camera
	cameraX int
	cameraY int

	// Pipes
	pipeTileYs []int

	gameoverCount int
}

func NewGame() *Game {
	g := &Game{}
	g.init()
	return g
}

func (g *Game) init() {
	g.x16 = 0
	g.y16 = 100 * 16
	g.cameraX = -240
	g.cameraY = 0
	g.pipeTileYs = make([]int, 256)
	for i := range g.pipeTileYs {
		g.pipeTileYs[i] = rand.Intn(6) + 2
	}
}

func jump() bool {
	if inpututil.IsKeyJustPressed(ebiten.KeySpace) {
		return true
	}
	if inpututil.IsMouseButtonJustPressed(ebiten.MouseButtonLeft) {
		return true
	}
	// TODO: Is it correct to use ID '0' here?
	if inpututil.IsJustTouched(0) {
		return true
	}
	return false
}

func (g *Game) Update(screen *ebiten.Image) error {
	switch g.mode {
	case ModeTitle:
		if jump() {
			g.mode = ModeGame
		}
	case ModeGame:
		g.x16 += 32
		g.cameraX += 2
		if jump() {
			g.vy16 = -96
			jumpPlayer.Rewind()
			jumpPlayer.Play()
		}
		g.y16 += g.vy16

		// Gravity
		g.vy16 += 4
		if g.vy16 > 96 {
			g.vy16 = 96
		}

		if g.hit() {
			hitPlayer.Rewind()
			hitPlayer.Play()
			g.mode = ModeGameOver
			g.gameoverCount = 30
		}
	case ModeGameOver:
		if g.gameoverCount > 0 {
			g.gameoverCount--
		}
		if g.gameoverCount == 0 && jump() {
			g.init()
			g.mode = ModeTitle
		}
	}

	if ebiten.IsRunningSlowly() {
		return nil
	}

	screen.Fill(color.RGBA{0x80, 0xa0, 0xc0, 0xff})
	g.drawTiles(screen)
	if g.mode != ModeTitle {
		g.drawGopher(screen)
	}
	var texts []string
	switch g.mode {
	case ModeTitle:
		texts = []string{"FLAPPY GOPHER", "", "", "", "", "PRESS SPACE"}
	case ModeGameOver:
		texts = []string{"", "GAMEOVER!"}
	}
	for i, l := range texts {
		x := (screenWidth - len(l)*fontSize) / 2
		text.Draw(screen, l, arcadeFont, x, (i+4)*fontSize, color.White)
	}

	scoreStr := fmt.Sprintf("%04d", g.score())
	text.Draw(screen, scoreStr, arcadeFont, screenWidth-len(scoreStr)*fontSize, fontSize, color.White)
	ebitenutil.DebugPrint(screen, fmt.Sprintf("FPS: %0.2f", ebiten.CurrentFPS()))
	return nil
}

func (g *Game) pipeAt(tileX int) (tileY int, ok bool) {
	if (tileX - pipeStartOffsetX) <= 0 {
		return 0, false
	}
	if floorMod(tileX-pipeStartOffsetX, pipeIntervalX) != 0 {
		return 0, false
	}
	idx := floorDiv(tileX-pipeStartOffsetX, pipeIntervalX)
	return g.pipeTileYs[idx%len(g.pipeTileYs)], true
}

func (g *Game) score() int {
	x := floorDiv(g.x16, 16) / tileSize
	if (x - pipeStartOffsetX) <= 0 {
		return 0
	}
	return floorDiv(x-pipeStartOffsetX, pipeIntervalX)
}

func (g *Game) hit() bool {
	if g.mode != ModeGame {
		return false
	}
	const (
		gopherWidth  = 30
		gopherHeight = 60
	)
	w, h := gopherImage.Size()
	x0 := floorDiv(g.x16, 16) + (w-gopherWidth)/2
	y0 := floorDiv(g.y16, 16) + (h-gopherHeight)/2
	x1 := x0 + gopherWidth
	y1 := y0 + gopherHeight
	if y0 < -tileSize*4 {
		return true
	}
	if y1 >= screenHeight-tileSize {
		return true
	}
	xMin := floorDiv(x0-pipeWidth, tileSize)
	xMax := floorDiv(x0+gopherWidth, tileSize)
	for x := xMin; x <= xMax; x++ {
		y, ok := g.pipeAt(x)
		if !ok {
			continue
		}
		if x0 >= x*tileSize+pipeWidth {
			continue
		}
		if x1 < x*tileSize {
			continue
		}
		if y0 < y*tileSize {
			return true
		}
		if y1 >= (y+pipeGapY)*tileSize {
			return true
		}
	}
	return false
}

func (g *Game) drawTiles(screen *ebiten.Image) {
	const (
		nx           = screenWidth / tileSize
		ny           = screenHeight / tileSize
		pipeTileSrcX = 128
		pipeTileSrcY = 192
	)

	op := &ebiten.DrawImageOptions{}
	for i := -2; i < nx+1; i++ {
		// ground
		op.GeoM.Reset()
		op.GeoM.Translate(float64(i*tileSize-floorMod(g.cameraX, tileSize)),
			float64((ny-1)*tileSize-floorMod(g.cameraY, tileSize)))
		r := image.Rect(0, 0, tileSize, tileSize)
		op.SourceRect = &r
		screen.DrawImage(tilesImage, op)

		// pipe
		if tileY, ok := g.pipeAt(floorDiv(g.cameraX, tileSize) + i); ok {
			for j := 0; j < tileY; j++ {
				op.GeoM.Reset()
				op.GeoM.Scale(1, -1)
				op.GeoM.Translate(float64(i*tileSize-floorMod(g.cameraX, tileSize)),
					float64(j*tileSize-floorMod(g.cameraY, tileSize)))
				op.GeoM.Translate(0, tileSize)
				if j == tileY-1 {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY, pipeTileSrcX+tileSize*2, pipeTileSrcY+tileSize)
					op.SourceRect = &r
				} else {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY+tileSize, pipeTileSrcX+tileSize*2, pipeTileSrcY+tileSize*2)
					op.SourceRect = &r
				}
				screen.DrawImage(tilesImage, op)
			}
			for j := tileY + pipeGapY; j < screenHeight/tileSize-1; j++ {
				op.GeoM.Reset()
				op.GeoM.Translate(float64(i*tileSize-floorMod(g.cameraX, tileSize)),
					float64(j*tileSize-floorMod(g.cameraY, tileSize)))
				if j == tileY+pipeGapY {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY, pipeTileSrcX+pipeWidth, pipeTileSrcY+tileSize)
					op.SourceRect = &r
				} else {
					r := image.Rect(pipeTileSrcX, pipeTileSrcY+tileSize, pipeTileSrcX+pipeWidth, pipeTileSrcY+tileSize+tileSize)
					op.SourceRect = &r
				}
				screen.DrawImage(tilesImage, op)
			}
		}
	}
}

func (g *Game) drawGopher(screen *ebiten.Image) {
	op := &ebiten.DrawImageOptions{}
	_, h := gopherImage.Size()
	op.GeoM.Translate(-float64(h)/2.0, -float64(h)/2.0)
	op.GeoM.Rotate(float64(g.vy16) / 96.0 * math.Pi / 6)
	op.GeoM.Translate(float64(h)/2.0, float64(h)/2.0)
	op.GeoM.Translate(float64(g.x16/16.0)-float64(g.cameraX), float64(g.y16/16.0)-float64(g.cameraY))
	op.Filter = ebiten.FilterLinear
	screen.DrawImage(gopherImage, op)
}

func main() {
	g := NewGame()
	// On browsers, let's use fullscreen so that this is playable on any browsers.
	// It is planned to ignore the given 'scale' apply fullscreen automatically on browsers (#571).
	if runtime.GOARCH == "js" {
		ebiten.SetFullscreen(true)
	}
	if err := ebiten.Run(g.Update, screenWidth, screenHeight, 1, "Flappy Gopher (Ebiten Demo)"); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package wrappa_test

import (
	"net/http"
	"net/http/httptest"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/wrappa"
	"github.com/concourse/concourse/atc/wrappa/wrappafakes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"
)

var _ = Describe("Concurrent Request Limits Wrappa", func() {
	var (
		fakeHandler *wrappafakes.FakeHandler
		fakePolicy  *wrappafakes.FakeConcurrentRequestPolicy
		fakePool    *wrappafakes.FakePool
		testLogger  *lagertest.TestLogger
		handler     http.Handler
		request     *http.Request
	)

	BeforeEach(func() {
		fakeHandler = new(wrappafakes.FakeHandler)
		testLogger = lagertest.NewTestLogger("test")
		request, _ = http.NewRequest("GET", "localhost:8080", nil)
	})

	AfterEach(func() {
		metric.ConcurrentRequests = map[string]*metric.Gauge{}
	})

	givenConcurrentRequestLimit := func(limit int) {
		fakePolicy = new(wrappafakes.FakeConcurrentRequestPolicy)
		fakePool = new(wrappafakes.FakePool)
		fakePolicy.HandlerPoolReturns(fakePool, true)
		fakePool.SizeReturns(limit)

		handler = wrappa.NewConcurrentRequestLimitsWrappa(testLogger, fakePolicy).
			Wrap(map[string]http.Handler{
				atc.ListAllJobs: fakeHandler,
			})[atc.ListAllJobs]
	}

	It("records the number of requests in-flight", func() {
		givenConcurrentRequestLimit(1)

		handler.ServeHTTP(httptest.NewRecorder(), request)
		handler.ServeHTTP(httptest.NewRecorder(), request)

		Expect(metric.ConcurrentRequests[atc.ListAllJobs].Max()).To(Equal(float64(1)))
	})

	It("records when the concurrent request limit is hit", func() {
		givenConcurrentRequestLimit(0)

		handler.ServeHTTP(httptest.NewRecorder(), request)
		handler.ServeHTTP(httptest.NewRecorder(), request)

		Expect(metric.ConcurrentRequestsLimitHit[atc.ListAllJobs].Delta()).To(Equal(float64(2)))
	})

	It("logs when the concurrent request limit is hit", func() {
		givenConcurrentRequestLimit(0)

		It("responds with a 503", func() {
			recorder := httptest.NewRecorder()
			handler.ServeHTTP(recorder, request)

			Expect(recorder.Code).To(Equal(http.StatusServiceUnavailable))
		})

		It("logs an INFO message", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(testLogger.Logs()).To(ConsistOf(
				MatchFields(IgnoreExtras, Fields{
					"Message":  Equal("test.concurrent-request-limit-reached"),
					"LogLevel": Equal(lager.INFO),
				}),
			))
		})

		It("increments the 'limitHit' counter", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(metric.ConcurrentRequestsLimitHit[atc.ListAllJobs].Delta()).To(Equal(float64(2)))
		})
	})

	Context("when the limit is not reached", func() {
		BeforeEach(func() {
			givenConcurrentRequestLimit(1)
			fakePool.TryAcquireReturns(true)
		})

		It("invokes the wrapped handler", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(fakeHandler.ServeHTTPCallCount()).To(Equal(1), "wrapped handler not invoked")
		})

		It("releases the pool", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(fakePool.ReleaseCallCount()).To(Equal(1))
		})

		It("records the number of requests in-flight", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(metric.ConcurrentRequests[atc.ListAllJobs].Max()).To(Equal(float64(1)))
		})
	})

	Context("when the endpoint is disabled", func() {
		BeforeEach(func() {
			givenConcurrentRequestLimit(0)
		})

		It("responds with a 501", func() {
			recorder := httptest.NewRecorder()
			handler.ServeHTTP(recorder, request)

			Expect(recorder.Code).To(Equal(http.StatusNotImplemented))
		})

		It("logs a DEBUG message", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(testLogger.Logs()).To(ConsistOf(
				MatchFields(IgnoreExtras, Fields{
					"Message":  Equal("test.endpoint-disabled"),
					"LogLevel": Equal(lager.DEBUG),
				}),
			))
		})

		It("increments the 'limitHit' counter", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(metric.ConcurrentRequestsLimitHit[atc.ListAllJobs].Delta()).To(Equal(float64(2)))
		})
	})
})
<commit_msg>atc: fix rebase issue in tests<commit_after>package wrappa_test

import (
	"net/http"
	"net/http/httptest"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/wrappa"
	"github.com/concourse/concourse/atc/wrappa/wrappafakes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"
)

var _ = Describe("Concurrent Request Limits Wrappa", func() {
	var (
		fakeHandler *wrappafakes.FakeHandler
		fakePolicy  *wrappafakes.FakeConcurrentRequestPolicy
		fakePool    *wrappafakes.FakePool
		testLogger  *lagertest.TestLogger
		handler     http.Handler
		request     *http.Request
	)

	BeforeEach(func() {
		fakeHandler = new(wrappafakes.FakeHandler)
		testLogger = lagertest.NewTestLogger("test")
		request, _ = http.NewRequest("GET", "localhost:8080", nil)
	})

	AfterEach(func() {
		metric.ConcurrentRequests = map[string]*metric.Gauge{}
	})

	givenConcurrentRequestLimit := func(limit int) {
		fakePolicy = new(wrappafakes.FakeConcurrentRequestPolicy)
		fakePool = new(wrappafakes.FakePool)
		fakePolicy.HandlerPoolReturns(fakePool, true)
		fakePool.SizeReturns(limit)

		handler = wrappa.NewConcurrentRequestLimitsWrappa(testLogger, fakePolicy).
			Wrap(map[string]http.Handler{
				atc.ListAllJobs: fakeHandler,
			})[atc.ListAllJobs]
	}

	Context("when the limit is reached", func() {
		BeforeEach(func() {
			givenConcurrentRequestLimit(1)
			fakePool.TryAcquireReturns(false)
		})

		It("responds with a 503", func() {
			recorder := httptest.NewRecorder()
			handler.ServeHTTP(recorder, request)

			Expect(recorder.Code).To(Equal(http.StatusServiceUnavailable))
		})

		It("logs an INFO message", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(testLogger.Logs()).To(ConsistOf(
				MatchFields(IgnoreExtras, Fields{
					"Message":  Equal("test.concurrent-request-limit-reached"),
					"LogLevel": Equal(lager.INFO),
				}),
			))
		})

		It("increments the 'limitHit' counter", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(metric.ConcurrentRequestsLimitHit[atc.ListAllJobs].Delta()).To(Equal(float64(2)))
		})
	})

	Context("when the limit is not reached", func() {
		BeforeEach(func() {
			givenConcurrentRequestLimit(1)
			fakePool.TryAcquireReturns(true)
		})

		It("invokes the wrapped handler", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(fakeHandler.ServeHTTPCallCount()).To(Equal(1), "wrapped handler not invoked")
		})

		It("releases the pool", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(fakePool.ReleaseCallCount()).To(Equal(1))
		})

		It("records the number of requests in-flight", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(metric.ConcurrentRequests[atc.ListAllJobs].Max()).To(Equal(float64(1)))
		})
	})

	Context("when the endpoint is disabled", func() {
		BeforeEach(func() {
			givenConcurrentRequestLimit(0)
		})

		It("responds with a 501", func() {
			recorder := httptest.NewRecorder()
			handler.ServeHTTP(recorder, request)

			Expect(recorder.Code).To(Equal(http.StatusNotImplemented))
		})

		It("logs a DEBUG message", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(testLogger.Logs()).To(ConsistOf(
				MatchFields(IgnoreExtras, Fields{
					"Message":  Equal("test.endpoint-disabled"),
					"LogLevel": Equal(lager.DEBUG),
				}),
			))
		})

		It("increments the 'limitHit' counter", func() {
			handler.ServeHTTP(httptest.NewRecorder(), request)
			handler.ServeHTTP(httptest.NewRecorder(), request)

			Expect(metric.ConcurrentRequestsLimitHit[atc.ListAllJobs].Delta()).To(Equal(float64(2)))
		})
	})
})
<|endoftext|>


<commit_before>package sqliteStorage

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	"crawshaw.io/sqlite"
	"crawshaw.io/sqlite/sqlitex"
	"github.com/anacrolix/missinggo/iter"
	"github.com/anacrolix/missinggo/v2/resource"
)

type conn = *sqlite.Conn

func initConn(conn conn, wal bool) error {
	err := sqlitex.ExecTransient(conn, `pragma synchronous=off`, nil)
	if err != nil {
		return err
	}
	if !wal {
		err = sqlitex.ExecTransient(conn, `pragma journal_mode=off`, nil)
		if err != nil {
			return err
		}
	}
	err = sqlitex.ExecTransient(conn, `pragma mmap_size=1000000000`, nil)
	if err != nil {
		return err
	}
	return nil
}

func initSchema(conn conn) error {
	return sqlitex.ExecScript(conn, `
pragma auto_vacuum=incremental;

create table if not exists blob(
	name text,
	last_used timestamp default (datetime('now')),
	data blob,
	primary key (name)
);

create table if not exists setting(
	name primary key on conflict replace,
	value
);

create view if not exists deletable_blob as
with recursive excess(
	usage_with,
	last_used,
	blob_rowid,
	data_length
) as (
	select * from (select (select sum(length(data)) from blob) as usage_with, last_used, rowid, length(data) from blob order by last_used, rowid limit 1)
		where usage_with >= (select value from setting where name='capacity')
	union all
	select usage_with-data_length, blob.last_used, blob.rowid, length(data) from excess join blob
		on blob.rowid=(select rowid from blob where (last_used, rowid) > (excess.last_used, blob_rowid))
	where usage_with >= (select value from setting where name='capacity')
) select * from excess;

CREATE TRIGGER if not exists trim_blobs_to_capacity_after_update after update on blob begin
	delete from blob where rowid in (select blob_rowid from deletable_blob);
end;
CREATE TRIGGER if not exists trim_blobs_to_capacity_after_insert after insert on blob begin
	delete from blob where rowid in (select blob_rowid from deletable_blob);
end;
`)
}

// Emulates a pool from a single Conn.
type poolFromConn struct {
	mu   sync.Mutex
	conn conn
}

func (me *poolFromConn) Get(ctx context.Context) conn {
	me.mu.Lock()
	return me.conn
}

func (me *poolFromConn) Put(conn conn) {
	if conn != me.conn {
		panic("expected to same conn")
	}
	me.mu.Unlock()
}

func NewProvider(conn *sqlite.Conn) (_ *provider, err error) {
	err = initConn(conn, false)
	if err != nil {
		return
	}
	err = initSchema(conn)
	return &provider{&poolFromConn{conn: conn}}, err
}

// Needs the pool size so it can initialize all the connections with pragmas.
func NewProviderPool(pool *sqlitex.Pool, numConns int, wal bool) (_ *provider, err error) {
	_, err = initPoolConns(context.TODO(), pool, numConns, wal)
	if err != nil {
		return
	}
	conn := pool.Get(context.TODO())
	defer pool.Put(conn)
	err = initSchema(conn)
	return &provider{pool: pool}, err
}

func initPoolConns(ctx context.Context, pool *sqlitex.Pool, numConn int, wal bool) (numInited int, err error) {
	var conns []conn
	defer func() {
		for _, c := range conns {
			pool.Put(c)
		}
	}()
	for range iter.N(numConn) {
		conn := pool.Get(ctx)
		if conn == nil {
			break
		}
		conns = append(conns, conn)
		err = initConn(conn, wal)
		if err != nil {
			err = fmt.Errorf("initing conn %v: %w", len(conns), err)
			return
		}
		numInited++
	}
	return
}

type pool interface {
	Get(context.Context) conn
	Put(conn)
}

type provider struct {
	pool pool
}

func (p *provider) NewInstance(s string) (resource.Instance, error) {
	return instance{s, p}, nil
}

type instance struct {
	location string
	p        *provider
}

func (i instance) withConn(with func(conn conn)) {
	conn := i.p.pool.Get(context.TODO())
	//err := sqlitex.Exec(conn, "pragma synchronous", func(stmt *sqlite.Stmt) error {
	//	log.Print(stmt.ColumnText(0))
	//	return nil
	//})
	//if err != nil {
	//	log.Print(err)
	//}
	defer i.p.pool.Put(conn)
	with(conn)
}

func (i instance) getConn() *sqlite.Conn {
	return i.p.pool.Get(context.TODO())
}

func (i instance) putConn(conn *sqlite.Conn) {
	i.p.pool.Put(conn)
}

func (i instance) Readdirnames() (names []string, err error) {
	prefix := i.location + "/"
	i.withConn(func(conn conn) {
		err = sqlitex.Exec(conn, "select name from blob where name like ?", func(stmt *sqlite.Stmt) error {
			names = append(names, stmt.ColumnText(0)[len(prefix):])
			return nil
		}, prefix+"%")
	})
	//log.Printf("readdir %q gave %q", i.location, names)
	return
}

func (i instance) getBlobRowid(conn conn) (rowid int64, err error) {
	rows := 0
	err = sqlitex.Exec(conn, "select rowid from blob where name=?", func(stmt *sqlite.Stmt) error {
		rowid = stmt.ColumnInt64(0)
		rows++
		return nil
	}, i.location)
	if err != nil {
		return
	}
	if rows == 1 {
		return
	}
	if rows == 0 {
		err = errors.New("blob not found")
		return
	}
	panic(rows)
}

type connBlob struct {
	*sqlite.Blob
	onClose func()
}

func (me connBlob) Close() error {
	err := me.Blob.Close()
	me.onClose()
	return err
}

func (i instance) Get() (ret io.ReadCloser, err error) {
	conn := i.getConn()
	blob, err := i.openBlob(conn, false, true)
	if err != nil {
		i.putConn(conn)
		return
	}
	var once sync.Once
	return connBlob{blob, func() {
		once.Do(func() { i.putConn(conn) })
	}}, nil
}

func (i instance) openBlob(conn conn, write, updateAccess bool) (*sqlite.Blob, error) {
	rowid, err := i.getBlobRowid(conn)
	if err != nil {
		return nil, err
	}
	// This seems to cause locking issues with in-memory databases. Is it something to do with not
	// having WAL?
	if updateAccess {
		err = sqlitex.Exec(conn, "update blob set last_used=datetime('now') where rowid=?", nil, rowid)
		if err != nil {
			err = fmt.Errorf("updating last_used: %w", err)
			return nil, err
		}
		if conn.Changes() != 1 {
			panic(conn.Changes())
		}
	}
	return conn.OpenBlob("main", "blob", "data", rowid, write)
}

func (i instance) Put(reader io.Reader) (err error) {
	var buf bytes.Buffer
	_, err = io.Copy(&buf, reader)
	if err != nil {
		return err
	}
	i.withConn(func(conn conn) {
		for range iter.N(10) {
			err = sqlitex.Exec(conn, "insert or replace into blob(name, data) values(?, ?)", nil, i.location, buf.Bytes())
			if err, ok := err.(sqlite.Error); ok && err.Code == sqlite.SQLITE_BUSY {
				log.Print("sqlite busy")
				time.Sleep(time.Second)
				continue
			}
			break
		}
	})
	return
}

type fileInfo struct {
	size int64
}

func (f fileInfo) Name() string {
	panic("implement me")
}

func (f fileInfo) Size() int64 {
	return f.size
}

func (f fileInfo) Mode() os.FileMode {
	panic("implement me")
}

func (f fileInfo) ModTime() time.Time {
	panic("implement me")
}

func (f fileInfo) IsDir() bool {
	panic("implement me")
}

func (f fileInfo) Sys() interface{} {
	panic("implement me")
}

func (i instance) Stat() (ret os.FileInfo, err error) {
	i.withConn(func(conn conn) {
		var blob *sqlite.Blob
		blob, err = i.openBlob(conn, false, false)
		if err != nil {
			return
		}
		defer blob.Close()
		ret = fileInfo{blob.Size()}
	})
	return
}

func (i instance) ReadAt(p []byte, off int64) (n int, err error) {
	i.withConn(func(conn conn) {
		if false {
			var blob *sqlite.Blob
			blob, err = i.openBlob(conn, false, true)
			if err != nil {
				return
			}
			defer blob.Close()
			if off >= blob.Size() {
				err = io.EOF
				return
			}
			if off+int64(len(p)) > blob.Size() {
				p = p[:blob.Size()-off]
			}
			n, err = blob.ReadAt(p, off)
		} else {
			gotRow := false
			err = sqlitex.Exec(
				conn,
				"select substr(data, ?, ?) from blob where name=?",
				func(stmt *sqlite.Stmt) error {
					if gotRow {
						panic("found multiple matching blobs")
					} else {
						gotRow = true
					}
					n = stmt.ColumnBytes(0, p)
					return nil
				},
				off+1, len(p), i.location,
			)
			if err != nil {
				return
			}
			if !gotRow {
				err = errors.New("blob not found")
				return
			}
			if n < len(p) {
				err = io.EOF
			}
		}
	})
	return
}

func (i instance) WriteAt(bytes []byte, i2 int64) (int, error) {
	panic("implement me")
}

func (i instance) Delete() (err error) {
	i.withConn(func(conn conn) {
		err = sqlitex.Exec(conn, "delete from blob where name=?", nil, i.location)
	})
	return
}
<commit_msg>sqlite storage: Force data to be used as a blob<commit_after>package sqliteStorage

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	"crawshaw.io/sqlite"
	"crawshaw.io/sqlite/sqlitex"
	"github.com/anacrolix/missinggo/iter"
	"github.com/anacrolix/missinggo/v2/resource"
)

type conn = *sqlite.Conn

func initConn(conn conn, wal bool) error {
	err := sqlitex.ExecTransient(conn, `pragma synchronous=off`, nil)
	if err != nil {
		return err
	}
	if !wal {
		err = sqlitex.ExecTransient(conn, `pragma journal_mode=off`, nil)
		if err != nil {
			return err
		}
	}
	err = sqlitex.ExecTransient(conn, `pragma mmap_size=1000000000`, nil)
	if err != nil {
		return err
	}
	return nil
}

func initSchema(conn conn) error {
	return sqlitex.ExecScript(conn, `
pragma auto_vacuum=incremental;

create table if not exists blob(
	name text,
	last_used timestamp default (datetime('now')),
	data blob,
	primary key (name)
);

create table if not exists setting(
	name primary key on conflict replace,
	value
);

create view if not exists deletable_blob as
with recursive excess(
	usage_with,
	last_used,
	blob_rowid,
	data_length
) as (
	select * from (select (select sum(length(cast(data as blob))) from blob) as usage_with, last_used, rowid, length(cast(data as blob)) from blob order by last_used, rowid limit 1)
		where usage_with >= (select value from setting where name='capacity')
	union all
	select usage_with-data_length, blob.last_used, blob.rowid, length(cast(data as blob)) from excess join blob
		on blob.rowid=(select rowid from blob where (last_used, rowid) > (excess.last_used, blob_rowid))
	where usage_with >= (select value from setting where name='capacity')
) select * from excess;

CREATE TRIGGER if not exists trim_blobs_to_capacity_after_update after update on blob begin
	delete from blob where rowid in (select blob_rowid from deletable_blob);
end;
CREATE TRIGGER if not exists trim_blobs_to_capacity_after_insert after insert on blob begin
	delete from blob where rowid in (select blob_rowid from deletable_blob);
end;
`)
}

// Emulates a pool from a single Conn.
type poolFromConn struct {
	mu   sync.Mutex
	conn conn
}

func (me *poolFromConn) Get(ctx context.Context) conn {
	me.mu.Lock()
	return me.conn
}

func (me *poolFromConn) Put(conn conn) {
	if conn != me.conn {
		panic("expected to same conn")
	}
	me.mu.Unlock()
}

func NewProvider(conn *sqlite.Conn) (_ *provider, err error) {
	err = initConn(conn, false)
	if err != nil {
		return
	}
	err = initSchema(conn)
	return &provider{&poolFromConn{conn: conn}}, err
}

// Needs the pool size so it can initialize all the connections with pragmas.
func NewProviderPool(pool *sqlitex.Pool, numConns int, wal bool) (_ *provider, err error) {
	_, err = initPoolConns(context.TODO(), pool, numConns, wal)
	if err != nil {
		return
	}
	conn := pool.Get(context.TODO())
	defer pool.Put(conn)
	err = initSchema(conn)
	return &provider{pool: pool}, err
}

func initPoolConns(ctx context.Context, pool *sqlitex.Pool, numConn int, wal bool) (numInited int, err error) {
	var conns []conn
	defer func() {
		for _, c := range conns {
			pool.Put(c)
		}
	}()
	for range iter.N(numConn) {
		conn := pool.Get(ctx)
		if conn == nil {
			break
		}
		conns = append(conns, conn)
		err = initConn(conn, wal)
		if err != nil {
			err = fmt.Errorf("initing conn %v: %w", len(conns), err)
			return
		}
		numInited++
	}
	return
}

type pool interface {
	Get(context.Context) conn
	Put(conn)
}

type provider struct {
	pool pool
}

func (p *provider) NewInstance(s string) (resource.Instance, error) {
	return instance{s, p}, nil
}

type instance struct {
	location string
	p        *provider
}

func (i instance) withConn(with func(conn conn)) {
	conn := i.p.pool.Get(context.TODO())
	//err := sqlitex.Exec(conn, "pragma synchronous", func(stmt *sqlite.Stmt) error {
	//	log.Print(stmt.ColumnText(0))
	//	return nil
	//})
	//if err != nil {
	//	log.Print(err)
	//}
	defer i.p.pool.Put(conn)
	with(conn)
}

func (i instance) getConn() *sqlite.Conn {
	return i.p.pool.Get(context.TODO())
}

func (i instance) putConn(conn *sqlite.Conn) {
	i.p.pool.Put(conn)
}

func (i instance) Readdirnames() (names []string, err error) {
	prefix := i.location + "/"
	i.withConn(func(conn conn) {
		err = sqlitex.Exec(conn, "select name from blob where name like ?", func(stmt *sqlite.Stmt) error {
			names = append(names, stmt.ColumnText(0)[len(prefix):])
			return nil
		}, prefix+"%")
	})
	//log.Printf("readdir %q gave %q", i.location, names)
	return
}

func (i instance) getBlobRowid(conn conn) (rowid int64, err error) {
	rows := 0
	err = sqlitex.Exec(conn, "select rowid from blob where name=?", func(stmt *sqlite.Stmt) error {
		rowid = stmt.ColumnInt64(0)
		rows++
		return nil
	}, i.location)
	if err != nil {
		return
	}
	if rows == 1 {
		return
	}
	if rows == 0 {
		err = errors.New("blob not found")
		return
	}
	panic(rows)
}

type connBlob struct {
	*sqlite.Blob
	onClose func()
}

func (me connBlob) Close() error {
	err := me.Blob.Close()
	me.onClose()
	return err
}

func (i instance) Get() (ret io.ReadCloser, err error) {
	conn := i.getConn()
	blob, err := i.openBlob(conn, false, true)
	if err != nil {
		i.putConn(conn)
		return
	}
	var once sync.Once
	return connBlob{blob, func() {
		once.Do(func() { i.putConn(conn) })
	}}, nil
}

func (i instance) openBlob(conn conn, write, updateAccess bool) (*sqlite.Blob, error) {
	rowid, err := i.getBlobRowid(conn)
	if err != nil {
		return nil, err
	}
	// This seems to cause locking issues with in-memory databases. Is it something to do with not
	// having WAL?
	if updateAccess {
		err = sqlitex.Exec(conn, "update blob set last_used=datetime('now') where rowid=?", nil, rowid)
		if err != nil {
			err = fmt.Errorf("updating last_used: %w", err)
			return nil, err
		}
		if conn.Changes() != 1 {
			panic(conn.Changes())
		}
	}
	return conn.OpenBlob("main", "blob", "data", rowid, write)
}

func (i instance) Put(reader io.Reader) (err error) {
	var buf bytes.Buffer
	_, err = io.Copy(&buf, reader)
	if err != nil {
		return err
	}
	i.withConn(func(conn conn) {
		for range iter.N(10) {
			err = sqlitex.Exec(conn,
				"insert or replace into blob(name, data) values(?, cast(? as blob))",
				nil,
				i.location, buf.Bytes())
			if err, ok := err.(sqlite.Error); ok && err.Code == sqlite.SQLITE_BUSY {
				log.Print("sqlite busy")
				time.Sleep(time.Second)
				continue
			}
			break
		}
	})
	return
}

type fileInfo struct {
	size int64
}

func (f fileInfo) Name() string {
	panic("implement me")
}

func (f fileInfo) Size() int64 {
	return f.size
}

func (f fileInfo) Mode() os.FileMode {
	panic("implement me")
}

func (f fileInfo) ModTime() time.Time {
	panic("implement me")
}

func (f fileInfo) IsDir() bool {
	panic("implement me")
}

func (f fileInfo) Sys() interface{} {
	panic("implement me")
}

func (i instance) Stat() (ret os.FileInfo, err error) {
	i.withConn(func(conn conn) {
		var blob *sqlite.Blob
		blob, err = i.openBlob(conn, false, false)
		if err != nil {
			return
		}
		defer blob.Close()
		ret = fileInfo{blob.Size()}
	})
	return
}

func (i instance) ReadAt(p []byte, off int64) (n int, err error) {
	i.withConn(func(conn conn) {
		if false {
			var blob *sqlite.Blob
			blob, err = i.openBlob(conn, false, true)
			if err != nil {
				return
			}
			defer blob.Close()
			if off >= blob.Size() {
				err = io.EOF
				return
			}
			if off+int64(len(p)) > blob.Size() {
				p = p[:blob.Size()-off]
			}
			n, err = blob.ReadAt(p, off)
		} else {
			gotRow := false
			err = sqlitex.Exec(
				conn,
				"select substr(cast(data as blob), ?, ?) from blob where name=?",
				func(stmt *sqlite.Stmt) error {
					if gotRow {
						panic("found multiple matching blobs")
					} else {
						gotRow = true
					}
					n = stmt.ColumnBytes(0, p)
					return nil
				},
				off+1, len(p), i.location,
			)
			if err != nil {
				return
			}
			if !gotRow {
				err = errors.New("blob not found")
				return
			}
			if n < len(p) {
				err = io.EOF
			}
		}
	})
	return
}

func (i instance) WriteAt(bytes []byte, i2 int64) (int, error) {
	panic("implement me")
}

func (i instance) Delete() (err error) {
	i.withConn(func(conn conn) {
		err = sqlitex.Exec(conn, "delete from blob where name=?", nil, i.location)
	})
	return
}
<|endoftext|>


<commit_before>// The MIT License (MIT)

// Copyright (c) 2017 Fabian Wenzelmann

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package goauth

import (
	"errors"
	"math"

	"golang.org/x/crypto/bcrypt"
)

/*
PasswordHandler is an interface that knows three methods:
Create a hash from a given (plaintext) password
Compare a previously by this method generated hash and compare it to
plaintext password.
A function PasswordHashLength that returns the length of the
password hashes.
There is an implementation BcryptHandler, so you don't have to write one
on your own, but you could!
*/
type PasswordHandler interface {
	// GenerateHash generates a hash from the given password.
	GenerateHash(password []byte) ([]byte, error)

	// CheckPassword tests if the passwords are equal, can return an error
	// (something is wrong with the data, random engine...). This should still
	// be handled as failure but you may wish to preceed differently.
	CheckPassword(hashedPW, password []byte) (bool, error)

	// PasswordHashLength returns the length of the password hash.
	// The hashes must be of the same length, so this method
	// must return the length of the elements created with
	// GenerateHash.
	// For bcrypt the length is 60.
	PasswordHashLength() int
}

// DefaultCost is the default cost parameter for bcrypt.
const DefaultCost = 10

// DefaultPWLength is he default length of encrypted passwords.
// This is 60 for bcrypt.
const DefaultPWLength = 60

// BcryptHandler is a PasswordHandler that uses bcrypt.
type BcryptHandler struct {
	cost int
}

// NewBcryptHandler creates a new PasswordHandler that uses bcrypt.
// cost is the cost parameter for the algorithm, use -1 for the default value
// (which should be fine in most cases). The default value is 10.
// Note that bcrypt has some further restrictions on the cost parameter:
// Currently it must be between 4 and 31.
func NewBcryptHandler(cost int) *BcryptHandler {
	if cost <= 0 {
		cost = DefaultCost
	}
	return &BcryptHandler{cost: cost}
}

// GenerateHash generates the password hash using bcrypt.
func (handler *BcryptHandler) GenerateHash(password []byte) ([]byte, error) {
	return bcrypt.GenerateFromPassword(password, handler.cost)
}

// CheckPassword checks if the plaintext password was used to create the
// hashedPW.
func (handler *BcryptHandler) CheckPassword(hashedPW, password []byte) (bool, error) {
	// get error from bcrypt
	err := bcrypt.CompareHashAndPassword(hashedPW, password)
	// Check what the error was, if it is nil everything is ok
	if err == nil {
		return true, nil
	}
	// if it is ErrMismatchedHashAndPassword no real error occurred, pws simply
	// didn't match
	if err == bcrypt.ErrMismatchedHashAndPassword {
		return false, nil
	}
	// otherwise something really went wrong
	return false, err
}

// PasswordHashLength returns the default length for bcrypt,
// that is 60.
func (handler *BcryptHandler) PasswordHashLength() int {
	return DefaultPWLength
}

// NoUserID is an user id that is returned if the user was
// not found or some error occurred.
const NoUserID = math.MaxUint64

// ErrUserNotFound is an error that is used in the Validate
// function to signal that the user with the given username
// was not found.
var ErrUserNotFound = errors.New("Username not found")

// UserHandler is an interface to deal with the management of
// users.
type UserHandler interface {
	// Init initializes the underlying storage.
	// Use this function every time you start your app, this
	// function must take sure that no error is produced if
	// invoked several times.
	// In SQL for example "CREATE TABLE IF NOT EXISTS"
	Init() error

	// Insert inserts a new user into the default scheme.
	// This function must return NoUserID and an error != nil
	// if any error occurred.
	// If the insert took place it always returns an error == nil.
	// However it can return nil as an error and NoUserID, in this case the
	// database doesn't support an immediate lookup for the newly inserted id
	// (sqlite3 and MySQL seem to support this though, postgre not).
	// Note that an error is also raised if the username is already in use
	// (must be unique).
	Insert(userName, firstName, lastName, email string, plainPW []byte) (uint64, error)

	// Validate validates the given plaintext password with the hashed password
	// of the user in the storage.
	// If err != nil you should always consider the lookup as a failure.
	// The function returns NoUserID and ErrUserNotFound if the user was not
	// found.
	// If no error occurred you can check if the login was successful by checking
	// the returned user id:
	// On failure it returns NoUserID and on success the id of the user with
	// username.
	Validate(userName string, CleartextPwCheck []byte) (uint64, error)
}
<commit_msg>rearranged const definitions<commit_after>// The MIT License (MIT)

// Copyright (c) 2017 Fabian Wenzelmann

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package goauth

import (
	"errors"
	"math"

	"golang.org/x/crypto/bcrypt"
)

/*
PasswordHandler is an interface that knows three methods:
Create a hash from a given (plaintext) password
Compare a previously by this method generated hash and compare it to
plaintext password.
A function PasswordHashLength that returns the length of the
password hashes.
There is an implementation BcryptHandler, so you don't have to write one
on your own, but you could!
*/
type PasswordHandler interface {
	// GenerateHash generates a hash from the given password.
	GenerateHash(password []byte) ([]byte, error)

	// CheckPassword tests if the passwords are equal, can return an error
	// (something is wrong with the data, random engine...). This should still
	// be handled as failure but you may wish to preceed differently.
	CheckPassword(hashedPW, password []byte) (bool, error)

	// PasswordHashLength returns the length of the password hash.
	// The hashes must be of the same length, so this method
	// must return the length of the elements created with
	// GenerateHash.
	// For bcrypt the length is 60.
	PasswordHashLength() int
}

const (
	// DefaultCost is the default cost parameter for bcrypt.
	DefaultCost = 10

	// DefaultPWLength is he default length of encrypted passwords.
	// This is 60 for bcrypt.
	DefaultPWLength = 60

	// NoUserID is an user id that is returned if the user was
	// not found or some error occurred.
	NoUserID = math.MaxUint64
)

// BcryptHandler is a PasswordHandler that uses bcrypt.
type BcryptHandler struct {
	cost int
}

// NewBcryptHandler creates a new PasswordHandler that uses bcrypt.
// cost is the cost parameter for the algorithm, use -1 for the default value
// (which should be fine in most cases). The default value is 10.
// Note that bcrypt has some further restrictions on the cost parameter:
// Currently it must be between 4 and 31.
func NewBcryptHandler(cost int) *BcryptHandler {
	if cost <= 0 {
		cost = DefaultCost
	}
	return &BcryptHandler{cost: cost}
}

// GenerateHash generates the password hash using bcrypt.
func (handler *BcryptHandler) GenerateHash(password []byte) ([]byte, error) {
	return bcrypt.GenerateFromPassword(password, handler.cost)
}

// CheckPassword checks if the plaintext password was used to create the
// hashedPW.
func (handler *BcryptHandler) CheckPassword(hashedPW, password []byte) (bool, error) {
	// get error from bcrypt
	err := bcrypt.CompareHashAndPassword(hashedPW, password)
	// Check what the error was, if it is nil everything is ok
	if err == nil {
		return true, nil
	}
	// if it is ErrMismatchedHashAndPassword no real error occurred, pws simply
	// didn't match
	if err == bcrypt.ErrMismatchedHashAndPassword {
		return false, nil
	}
	// otherwise something really went wrong
	return false, err
}

// PasswordHashLength returns the default length for bcrypt,
// that is 60.
func (handler *BcryptHandler) PasswordHashLength() int {
	return DefaultPWLength
}

// ErrUserNotFound is an error that is used in the Validate
// function to signal that the user with the given username
// was not found.
var ErrUserNotFound = errors.New("Username not found.")

// UserHandler is an interface to deal with the management of
// users.
type UserHandler interface {
	// Init initializes the underlying storage.
	// Use this function every time you start your app, this
	// function must take sure that no error is produced if
	// invoked several times.
	// In SQL for example "CREATE TABLE IF NOT EXISTS"
	Init() error

	// Insert inserts a new user into the default scheme.
	// This function must return NoUserID and an error != nil
	// if any error occurred.
	// If the insert took place it always returns an error == nil.
	// However it can return nil as an error and NoUserID, in this case the
	// database doesn't support an immediate lookup for the newly inserted id
	// (sqlite3 and MySQL seem to support this though, postgre not).
	// Note that an error is also raised if the username is already in use
	// (must be unique).
	Insert(userName, firstName, lastName, email string, plainPW []byte) (uint64, error)

	// Validate validates the given plaintext password with the hashed password
	// of the user in the storage.
	// If err != nil you should always consider the lookup as a failure.
	// The function returns NoUserID and ErrUserNotFound if the user was not
	// found.
	// If no error occurred you can check if the login was successful by checking
	// the returned user id:
	// On failure it returns NoUserID and on success the id of the user with
	// username.
	Validate(userName string, CleartextPwCheck []byte) (uint64, error)
}
<|endoftext|>


<commit_before>// This file is subject to a 1-clause BSD license.
// Its contents can be found in the enclosed LICENSE file.

package url

import (
	"bytes"
	"github.com/jteeuwen/ircb/plugin"
	"github.com/jteeuwen/ircb/proto"
	"html"
    "os"
    "strconv"
	"io/ioutil"
	"net/http"
	"regexp"
    "github.com/ChimeraCoder/anaconda"
)

//This regex will check if a URL points to a Twitter status
var twitterUrlRegex = regexp.MustCompile(`https?:\/\/(www\.)?twitter.com\/[A-Za-z0-9]*\/status\/([0-9]+)`)

var api anaconda.TwitterApi

func init() { plugin.Register(New) }

type Plugin struct {
	*plugin.Base

	// Each entry holds a regex pattern which should be excluded
	// from the url-title-lookup.
	exclude []*regexp.Regexp

	// Pattern which recognizes urls.
	url *regexp.Regexp
}

func New(profile string) plugin.Plugin {
	p := new(Plugin)
	p.Base = plugin.New(profile, "url")
	p.url = regexp.MustCompile(`\bhttps?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]+(\:[0-9]+)?(/\S*)?\b`)
	return p
}

// Init initializes the plugin. it loads configuration data and binds
// commands and protocol handlers.
func (p *Plugin) Load(c *proto.Client) (err error) {
	err = p.Base.Load(c)
	if err != nil {
		return
	}

	c.Bind(proto.CmdPrivMsg, func(c *proto.Client, m *proto.Message) {
		p.parseURL(c, m)
	})

	ini := p.LoadConfig()
	if ini == nil {
		return
	}

	s := ini.Section("exclude")
	list := s.List("url")
	p.exclude = make([]*regexp.Regexp, len(list))

	for i := range list {
		p.exclude[i], err = regexp.Compile(list[i])

		if err != nil {
			return
		}
	}

	return
}

// parseURL looks for URL's embedded in incoming messages.
// If they are valid http[s] url's and not in the exclude list,
// we use them to fetch page titles from the internet.
func (p *Plugin) parseURL(c *proto.Client, m *proto.Message) {
	list := p.url.FindAllString(m.Data, -1)
	if len(list) == 0 {
		return
	}

	for _, url := range list {
        //TODO make this less hackny
        if twitterUrlRegex.MatchString(url) {
            go fetchTweet(c, m, url)

        } else if !p.excluded(url) {
			go fetchTitle(c, m, url)
		}
	}
}

// excluded returns true if the given url is part of the exclusion list.
func (p *Plugin) excluded(url string) bool {
	for _, excl := range p.exclude {
		if excl.MatchString(url) {
			return true
		}
	}

	return false
}

// fetchTitle attempts to retrieve the title element for a given url.
func fetchTitle(c *proto.Client, m *proto.Message, url string) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}

	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	if err != nil {
		return
	}

	body = bytes.ToLower(body)
	s := bytes.Index(body, []byte("<title>"))
	if s == -1 {
		return
	}

	body = body[s+7:]

	e := bytes.Index(body, []byte("</title>"))
	if e == -1 {
		e = len(body) - 1
	}

	body = bytes.TrimSpace(body[:e])

	c.PrivMsg(m.Receiver, "%s's link shows: %s",
		m.SenderName, html.UnescapeString(string(body)))
}

// fetchTweet attempts to retrieve the tweet associated with a given url.
func fetchTweet(c *proto.Client, m *proto.Message, url string) {
    id, _ := strconv.ParseInt(twitterUrlRegex.FindStringSubmatch(url)[2], 10, 64)
    tweet, err := api.GetTweet(id, nil)
    if err != nil{
        c.PrivMsg(m.Receiver, "error parsing tweet :(")
    }

	c.PrivMsg(m.Receiver, "%s's tweet shows: %s",
		m.SenderName, html.UnescapeString(tweet.Text))
}

func init(){
    anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
    anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))
    api = anaconda.NewTwitterApi(os.Getenv("TWITTER_ACCESS_TOKEN"), os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))
}

<commit_msg>Failover gracefully to using <title> attr if tweet extraction fails<commit_after>// This file is subject to a 1-clause BSD license.
// Its contents can be found in the enclosed LICENSE file.

package url

import (
	"bytes"
	"github.com/ChimeraCoder/anaconda"
	"github.com/jteeuwen/ircb/plugin"
	"github.com/jteeuwen/ircb/proto"
	"html"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

//This regex will check if a URL points to a Twitter status
var twitterUrlRegex = regexp.MustCompile(`https?:\/\/(www\.)?twitter.com\/[A-Za-z0-9]*\/status\/([0-9]+)`)

var api anaconda.TwitterApi

func init() { plugin.Register(New) }

type Plugin struct {
	*plugin.Base

	// Each entry holds a regex pattern which should be excluded
	// from the url-title-lookup.
	exclude []*regexp.Regexp

	// Pattern which recognizes urls.
	url *regexp.Regexp
}

func New(profile string) plugin.Plugin {
	p := new(Plugin)
	p.Base = plugin.New(profile, "url")
	p.url = regexp.MustCompile(`\bhttps?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]+(\:[0-9]+)?(/\S*)?\b`)
	return p
}

// Init initializes the plugin. it loads configuration data and binds
// commands and protocol handlers.
func (p *Plugin) Load(c *proto.Client) (err error) {
	err = p.Base.Load(c)
	if err != nil {
		return
	}

	c.Bind(proto.CmdPrivMsg, func(c *proto.Client, m *proto.Message) {
		p.parseURL(c, m)
	})

	ini := p.LoadConfig()
	if ini == nil {
		return
	}

	s := ini.Section("exclude")
	list := s.List("url")
	p.exclude = make([]*regexp.Regexp, len(list))

	for i := range list {
		p.exclude[i], err = regexp.Compile(list[i])

		if err != nil {
			return
		}
	}

	return
}

// parseURL looks for URL's embedded in incoming messages.
// If they are valid http[s] url's and not in the exclude list,
// we use them to fetch page titles from the internet.
func (p *Plugin) parseURL(c *proto.Client, m *proto.Message) {
	list := p.url.FindAllString(m.Data, -1)
	if len(list) == 0 {
		return
	}

	for _, url := range list {
		//TODO make this less hackny
		if twitterUrlRegex.MatchString(url) {
			go fetchTweet(c, m, url)

		} else if !p.excluded(url) {
			go fetchTitle(c, m, url)
		}
	}
}

// excluded returns true if the given url is part of the exclusion list.
func (p *Plugin) excluded(url string) bool {
	for _, excl := range p.exclude {
		if excl.MatchString(url) {
			return true
		}
	}

	return false
}

// fetchTitle attempts to retrieve the title element for a given url.
func fetchTitle(c *proto.Client, m *proto.Message, url string) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}

	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	if err != nil {
		return
	}

	body = bytes.ToLower(body)
	s := bytes.Index(body, []byte("<title>"))
	if s == -1 {
		return
	}

	body = body[s+7:]

	e := bytes.Index(body, []byte("</title>"))
	if e == -1 {
		e = len(body) - 1
	}

	body = bytes.TrimSpace(body[:e])

	c.PrivMsg(m.Receiver, "%s's link shows: %s",
		m.SenderName, html.UnescapeString(string(body)))
}

// fetchTweet attempts to retrieve the tweet associated with a given url.
func fetchTweet(c *proto.Client, m *proto.Message, url string) {
	id, err := strconv.ParseInt(twitterUrlRegex.FindStringSubmatch(url)[2], 10, 64)
	if err != nil {
		c.PrivMsg(m.Receiver, "error parsing tweet :(")
		log.Print("error parsing tweet for %s: %v", url, err)
		fetchTitle(c, m, url)
		return
	}
	tweet, err := api.GetTweet(id, nil)
	if err != nil {
		log.Print("error parsing tweet for %s: %v", url, err)
		fetchTitle(c, m, url)
		return
	}
	c.PrivMsg(m.Receiver, "%s's tweet shows: %s",
		m.SenderName, html.UnescapeString(tweet.Text))
}

func init() {
	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))
	api = anaconda.NewTwitterApi(os.Getenv("TWITTER_ACCESS_TOKEN"), os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package goleveldb

import (
	"fmt"

	"github.com/blevesearch/bleve/index/store"
	"github.com/blevesearch/bleve/registry"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
)

const Name = "goleveldb"

type Store struct {
	path string
	opts *opt.Options
	db   *leveldb.DB
	mo   store.MergeOperator

	defaultWriteOptions *opt.WriteOptions
	defaultReadOptions  *opt.ReadOptions
}

func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {

	path, ok := config["path"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify path")
	}

	opts, err := applyConfig(&opt.Options{}, config)
	if err != nil {
		return nil, err
	}

	db, err := leveldb.OpenFile(path, opts)
	if err != nil {
		return nil, err
	}

	rv := Store{
		path:                path,
		opts:                opts,
		db:                  db,
		mo:                  mo,
		defaultReadOptions:  &opt.ReadOptions{},
		defaultWriteOptions: &opt.WriteOptions{},
	}
	rv.defaultWriteOptions.Sync = true
	return &rv, nil
}

func (ldbs *Store) Close() error {
	return ldbs.db.Close()
}

func (ldbs *Store) Reader() (store.KVReader, error) {
	snapshot, _ := ldbs.db.GetSnapshot()
	return &Reader{
		store:    ldbs,
		snapshot: snapshot,
	}, nil
}

func (ldbs *Store) Writer() (store.KVWriter, error) {
	return &Writer{
		store: ldbs,
	}, nil
}

func init() {
	registry.RegisterKVStore(Name, New)
}
<commit_msg>Add compact method to goleveldb store<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package goleveldb

import (
	"fmt"

	"github.com/blevesearch/bleve/index/store"
	"github.com/blevesearch/bleve/registry"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

const Name = "goleveldb"

type Store struct {
	path string
	opts *opt.Options
	db   *leveldb.DB
	mo   store.MergeOperator

	defaultWriteOptions *opt.WriteOptions
	defaultReadOptions  *opt.ReadOptions
}

func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {

	path, ok := config["path"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify path")
	}

	opts, err := applyConfig(&opt.Options{}, config)
	if err != nil {
		return nil, err
	}

	db, err := leveldb.OpenFile(path, opts)
	if err != nil {
		return nil, err
	}

	rv := Store{
		path:                path,
		opts:                opts,
		db:                  db,
		mo:                  mo,
		defaultReadOptions:  &opt.ReadOptions{},
		defaultWriteOptions: &opt.WriteOptions{},
	}
	rv.defaultWriteOptions.Sync = true
	return &rv, nil
}

func (ldbs *Store) Close() error {
	return ldbs.db.Close()
}

func (ldbs *Store) Reader() (store.KVReader, error) {
	snapshot, _ := ldbs.db.GetSnapshot()
	return &Reader{
		store:    ldbs,
		snapshot: snapshot,
	}, nil
}

func (ldbs *Store) Writer() (store.KVWriter, error) {
	return &Writer{
		store: ldbs,
	}, nil
}

func (ldbs *Store) Compact() error {
	return ldbs.db.CompactRange(util.Range{nil, nil})
}

func init() {
	registry.RegisterKVStore(Name, New)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// isExists checks whether the path is exists
func isExists(path string) bool {
	_, err := os.Stat(path)

	return err == nil
}

// isDir checks whether the path is directory or not
func isDir(path string) bool {
	stat, err := os.Stat(path)

	return err == nil && stat.IsDir()
}

// mkClipDir makes .clip directory
func mkClipDir() {
	os.Mkdir(".clip", 0755)
	fmt.Println("Created .clip")
}

// pickValidCommits picks all valid commits corresponding to pictures by asc
func pickValidCommits() ([]string, error) {
	result, err := exec.Command("git", "rev-list", "--all").Output()
	if err != nil {
		return nil, err
	}

	tmp := strings.Split(string(result), "\n")

	hashes := make([]string, len(tmp))
	for _, hash := range tmp[:len(tmp)] {
		if strings.TrimSpace(hash) == "" {
			continue
		}

		fmt.Println("hash: ", hash)
		if isExists(filepath.Join(".clip", hash)) {
			hashes = append(hashes, hash)
		}
	}

	return reverse(hashes), nil
}
<commit_msg>[FIX] Assign length of hashes to capacity, not length<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// isExists checks whether the path is exists
func isExists(path string) bool {
	_, err := os.Stat(path)

	return err == nil
}

// isDir checks whether the path is directory or not
func isDir(path string) bool {
	stat, err := os.Stat(path)

	return err == nil && stat.IsDir()
}

// mkClipDir makes .clip directory
func mkClipDir() {
	os.Mkdir(".clip", 0755)
	fmt.Println("Created .clip")
}

// pickValidCommits picks all valid commits corresponding to pictures by asc
func pickValidCommits() ([]string, error) {
	result, err := exec.Command("git", "rev-list", "--all").Output()
	if err != nil {
		return nil, err
	}

	tmp := strings.Split(strings.TrimSpace(string(result)), "\n")

	hashes := make([]string, 0, len(tmp))
	for _, hash := range tmp {
		if isExists(filepath.Join(".clip", hash)) {
			hashes = append(hashes, hash)
		}
	}

	return reverse(hashes), nil
}
<|endoftext|>


<commit_before>package moskus

import (
	"errors"
	"fmt"
	"time"
)

// Checks if the given time is at the given month and day
func atMD(date time.Time, month, day int) bool {
	return (date.Month() == time.Month(month)) && (date.Day() == day)
}

// Checks if the two given times are at the same months and days
func atDate(t, when time.Time) bool {
	return (t.Month() == when.Month()) && (t.Day() == when.Day())
}

// Return the count of a given weekday from day t, +- a few days
func numberOfWeekdaysInPeriod(date time.Time, days int, whichWeekday time.Weekday) int {
	specialWeekdayCounter := 0
	when := date
	if days < 0 {
		for i := days; i <= 0; i++ {
			when = date.AddDate(0, 0, i)
			if when.Weekday() == whichWeekday {
				specialWeekdayCounter++
			}
		}
	} else {
		for i := 0; i <= days; i++ {
			when = date.AddDate(0, 0, i)
			if when.Weekday() == whichWeekday {
				specialWeekdayCounter++
			}
		}
	}
	return specialWeekdayCounter
}

// Return the number of sundays from day t, +- a few days
func sundaysInPeriod(date time.Time, days int) int {
	return numberOfWeekdaysInPeriod(date, days, time.Sunday)
}

// Find a preceeding sunday
func searchBackwardsForSunday(date time.Time) (time.Time, error) {
	// Start with the day before the given date
	current := date.AddDate(0, 0, -1)

	// Stay within the same year
	for current.Year() == date.Year() {
		// Check if it's a Sunday
		if current.Weekday() == time.Sunday {
			// Found one
			return current, nil
		}

		// Go the previous day
		current = current.AddDate(0, 0, -1)
	}

	return date, errors.New("Could not find an earlier Sunday the same year!")
}

// Find the Nth type of weekday of a given year and month
func nthWeekdayOfMonth(date time.Time, n int, whichWeekday time.Weekday) (time.Time, error) {

	specialWeekdayCounter := 0

	// Start at the first day in the given month
	current := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.UTC)

	// As long as we are in the same month
	for current.Month() == date.Month() {

		// Which weekday is it?
		if current.Weekday() == whichWeekday {
			specialWeekdayCounter++
		}

		// Is it the Nth occurance?
		if specialWeekdayCounter == n {
			return current, nil
		}

		// If it's the given weekday, advance almost one week forward
		if current.Weekday() == whichWeekday {
			current = current.AddDate(0, 0, 7)
			continue
		}

		// Advance to the next day
		current = current.AddDate(0, 0, 1)
	}

	return date, errors.New(fmt.Sprintf("Could not find the %dth %s in %s!", n, whichWeekday, date.Month()))
}

// Find the Nth sunday of a given year and month
func nthSundayOfMonth(date time.Time, n int) (time.Time, error) {
	return nthWeekdayOfMonth(date, n, time.Sunday)
}
<commit_msg>fmt.Errorf is more suitable<commit_after>package moskus

import (
	"errors"
	"fmt"
	"time"
)

// Checks if the given time is at the given month and day
func atMD(date time.Time, month, day int) bool {
	return (date.Month() == time.Month(month)) && (date.Day() == day)
}

// Checks if the two given times are at the same months and days
func atDate(t, when time.Time) bool {
	return (t.Month() == when.Month()) && (t.Day() == when.Day())
}

// Return the count of a given weekday from day t, +- a few days
func numberOfWeekdaysInPeriod(date time.Time, days int, whichWeekday time.Weekday) int {
	specialWeekdayCounter := 0
	when := date
	if days < 0 {
		for i := days; i <= 0; i++ {
			when = date.AddDate(0, 0, i)
			if when.Weekday() == whichWeekday {
				specialWeekdayCounter++
			}
		}
	} else {
		for i := 0; i <= days; i++ {
			when = date.AddDate(0, 0, i)
			if when.Weekday() == whichWeekday {
				specialWeekdayCounter++
			}
		}
	}
	return specialWeekdayCounter
}

// Return the number of sundays from day t, +- a few days
func sundaysInPeriod(date time.Time, days int) int {
	return numberOfWeekdaysInPeriod(date, days, time.Sunday)
}

// Find a preceeding sunday
func searchBackwardsForSunday(date time.Time) (time.Time, error) {
	// Start with the day before the given date
	current := date.AddDate(0, 0, -1)

	// Stay within the same year
	for current.Year() == date.Year() {
		// Check if it's a Sunday
		if current.Weekday() == time.Sunday {
			// Found one
			return current, nil
		}

		// Go the previous day
		current = current.AddDate(0, 0, -1)
	}

	return date, errors.New("Could not find an earlier Sunday the same year!")
}

// Find the Nth type of weekday of a given year and month
func nthWeekdayOfMonth(date time.Time, n int, whichWeekday time.Weekday) (time.Time, error) {

	specialWeekdayCounter := 0

	// Start at the first day in the given month
	current := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.UTC)

	// As long as we are in the same month
	for current.Month() == date.Month() {

		// Which weekday is it?
		if current.Weekday() == whichWeekday {
			specialWeekdayCounter++
		}

		// Is it the Nth occurance?
		if specialWeekdayCounter == n {
			return current, nil
		}

		// If it's the given weekday, advance almost one week forward
		if current.Weekday() == whichWeekday {
			current = current.AddDate(0, 0, 7)
			continue
		}

		// Advance to the next day
		current = current.AddDate(0, 0, 1)
	}

	return date, fmt.Errorf("Could not find the %dth %s in %s!", n, whichWeekday, date.Month())
}

// Find the Nth sunday of a given year and month
func nthSundayOfMonth(date time.Time, n int) (time.Time, error) {
	return nthWeekdayOfMonth(date, n, time.Sunday)
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/buildbucket/deprecated"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/buildbucket/protoutil"
	bbv1 "go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/milo/common/model"
	"go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/router"
	"google.golang.org/genproto/protobuf/field_mask"
)

var (
	buildCounter = metric.NewCounter(
		"luci/milo/buildbucket_pubsub/builds",
		"The number of buildbucket builds received by Milo from PubSub",
		nil,
		field.String("bucket"),
		// True for luci build, False for non-luci (ie buildbot) build.
		field.Bool("luci"),
		// Status can be "COMPLETED", "SCHEDULED", or "STARTED"
		field.String("status"),
		// Action can be one of 3 options.
		//   * "Created" - This is the first time Milo heard about this build
		//   * "Modified" - Milo updated some information about this build vs. what
		//     it knew before.
		//   * "Rejected" - Milo was unable to accept this build.
		field.String("action"))
)

// PubSubHandler is a webhook that stores the builds coming in from pubsub.
func PubSubHandler(ctx *router.Context) {
	err := pubSubHandlerImpl(ctx.Context, ctx.Request)
	if err != nil {
		logging.Errorf(ctx.Context, "error while handling pubsub event")
		errors.Log(ctx.Context, err)
	}
	if transient.Tag.In(err) {
		// Transient errors are 500 so that PubSub retries them.
		ctx.Writer.WriteHeader(http.StatusInternalServerError)
		return
	}
	// No errors or non-transient errors are 200s so that PubSub does not retry
	// them.
	ctx.Writer.WriteHeader(http.StatusOK)
}

func mustTimestamp(ts *timestamp.Timestamp) time.Time {
	if t, err := ptypes.Timestamp(ts); err == nil {
		return t
	}
	return time.Time{}
}

var summaryBuildMask = &field_mask.FieldMask{
	Paths: []string{
		"id",
		"builder",
		"number",
		"create_time",
		"start_time",
		"end_time",
		"update_time",
		"status",
		"summary_markdown",
		"tags",
		"infra.swarming",
		"input.experimental",
	},
}

// getSummary returns a model.BuildSummary representing a buildbucket build.
func getSummary(c context.Context, host string, project string, id int64) (*model.BuildSummary, error) {
	client, err := buildbucketClient(c, host, auth.AsProject, auth.WithProject(project))
	if err != nil {
		return nil, err
	}
	b, err := client.GetBuild(c, &buildbucketpb.GetBuildRequest{
		Id:     id,
		Fields: summaryBuildMask,
	})
	if err != nil {
		return nil, err
	}
	buildAddress := fmt.Sprintf("%d", b.Id)
	if b.Number != 0 {
		buildAddress = fmt.Sprintf("luci.%s.%s/%s/%d", b.Builder.Project, b.Builder.Bucket, b.Builder.Builder, b.Number)
	}

	// Note: The parent for buildbucket build summaries is currently a fake entity.
	// In the future, builds can be cached here, but we currently don't do that.
	buildKey := datastore.MakeKey(c, "buildbucket.Build", fmt.Sprintf("%s:%s", host, buildAddress))
	swarming := b.GetInfra().GetSwarming()

	bs := &model.BuildSummary{
		ProjectID:  b.Builder.Project,
		BuildKey:   buildKey,
		BuilderID:  BuilderID{*b.Builder}.String(),
		BuildID:    "buildbucket/" + buildAddress,
		BuildSet:   protoutil.BuildSets(b),
		ContextURI: []string{fmt.Sprintf("buildbucket://%s/build/%d", host, id)},
		Created:    mustTimestamp(b.CreateTime),
		Summary: model.Summary{
			Start:  mustTimestamp(b.StartTime),
			End:    mustTimestamp(b.EndTime),
			Status: statusMap[b.Status],
		},
		Version:      mustTimestamp(b.UpdateTime).UnixNano(),
		Experimental: b.GetInput().GetExperimental(),
		Critical:     b.GetCritical(),
	}
	if task := swarming.GetTaskId(); task != "" {
		bs.ContextURI = append(
			bs.ContextURI,
			fmt.Sprintf("swarming://%s/task/%s", swarming.GetHostname(), swarming.GetTaskId()))
	}
	return bs, nil
}

// pubSubHandlerImpl takes the http.Request, expects to find
// a common.PubSubSubscription JSON object in the Body, containing a bbPSEvent,
// and handles the contents with generateSummary.
func pubSubHandlerImpl(c context.Context, r *http.Request) error {
	// This is the default action. The code below will modify the values of some
	// or all of these parameters.
	isLUCI, bucket, status, action := false, "UNKNOWN", "UNKNOWN", "Rejected"

	defer func() {
		// closure for late binding
		buildCounter.Add(c, 1, bucket, isLUCI, status, action)
	}()

	msg := common.PubSubSubscription{}
	if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
		// This might be a transient error, e.g. when the json format changes
		// and Milo isn't updated yet.
		return errors.Annotate(err, "could not decode message").Tag(transient.Tag).Err()
	}
	if v, ok := msg.Message.Attributes["version"].(string); ok && v != "v1" {
		// TODO(nodir): switch to v2, crbug.com/826006
		logging.Debugf(c, "unsupported pubsub message version %q. Ignoring", v)
		return nil
	}
	bData, err := msg.GetData()
	if err != nil {
		return errors.Annotate(err, "could not parse pubsub message string").Err()
	}

	event := struct {
		Build    bbv1.LegacyApiCommonBuildMessage `json:"build"`
		Hostname string                           `json:"hostname"`
	}{}
	if err := json.Unmarshal(bData, &event); err != nil {
		return errors.Annotate(err, "could not parse pubsub message data").Err()
	}

	build := deprecated.Build{}
	if err := build.ParseMessage(&event.Build); err != nil {
		return errors.Annotate(err, "could not parse deprecated.Build").Err()
	}

	bucket = build.Bucket
	status = build.Status.String()
	isLUCI = strings.HasPrefix(bucket, "luci.")

	logging.Debugf(c, "Received from %s: build %s/%s (%s)\n%v",
		event.Hostname, bucket, build.Builder, status, build)

	if !isLUCI || build.Builder == "" {
		logging.Infof(c, "This is not an ingestable build, ignoring")
		return nil
	}

	// TODO(iannucci,nodir): get the bot context too
	// TODO(iannucci,nodir): support manifests/got_revision
	bs, err := getSummary(c, event.Hostname, build.Project, build.ID)
	if err != nil {
		return err
	}
	if err := bs.AddManifestKeysFromBuildSets(c); err != nil {
		return err
	}

	return transient.Tag.Apply(datastore.RunInTransaction(c, func(c context.Context) error {
		curBS := &model.BuildSummary{BuildKey: bs.BuildKey}
		switch err := datastore.Get(c, curBS); err {
		case datastore.ErrNoSuchEntity:
			action = "Created"
		case nil:
			action = "Modified"
		default:
			return errors.Annotate(err, "reading current BuildSummary").Err()
		}

		if bs.Version <= curBS.Version {
			logging.Warningf(c, "current BuildSummary is newer: %d <= %d",
				bs.Version, curBS.Version)
			return nil
		}

		if err := datastore.Put(c, bs); err != nil {
			return err
		}

		return model.UpdateBuilderForBuild(c, bs)
	}, &datastore.TransactionOptions{XG: true}))
}

// MakeBuildKey returns a new datastore Key for a buildbucket.Build.
//
// There's currently no model associated with this key, but it's used as
// a parent for a model.BuildSummary.
func MakeBuildKey(c context.Context, host, buildAddress string) *datastore.Key {
	return datastore.MakeKey(c,
		"buildbucket.Build", fmt.Sprintf("%s:%s", host, buildAddress))
}
<commit_msg>Add "critical" to field mask for Build -> BuildSummary data.<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/buildbucket/deprecated"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/buildbucket/protoutil"
	bbv1 "go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/milo/common/model"
	"go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/router"
	"google.golang.org/genproto/protobuf/field_mask"
)

var (
	buildCounter = metric.NewCounter(
		"luci/milo/buildbucket_pubsub/builds",
		"The number of buildbucket builds received by Milo from PubSub",
		nil,
		field.String("bucket"),
		// True for luci build, False for non-luci (ie buildbot) build.
		field.Bool("luci"),
		// Status can be "COMPLETED", "SCHEDULED", or "STARTED"
		field.String("status"),
		// Action can be one of 3 options.
		//   * "Created" - This is the first time Milo heard about this build
		//   * "Modified" - Milo updated some information about this build vs. what
		//     it knew before.
		//   * "Rejected" - Milo was unable to accept this build.
		field.String("action"))
)

// PubSubHandler is a webhook that stores the builds coming in from pubsub.
func PubSubHandler(ctx *router.Context) {
	err := pubSubHandlerImpl(ctx.Context, ctx.Request)
	if err != nil {
		logging.Errorf(ctx.Context, "error while handling pubsub event")
		errors.Log(ctx.Context, err)
	}
	if transient.Tag.In(err) {
		// Transient errors are 500 so that PubSub retries them.
		ctx.Writer.WriteHeader(http.StatusInternalServerError)
		return
	}
	// No errors or non-transient errors are 200s so that PubSub does not retry
	// them.
	ctx.Writer.WriteHeader(http.StatusOK)
}

func mustTimestamp(ts *timestamp.Timestamp) time.Time {
	if t, err := ptypes.Timestamp(ts); err == nil {
		return t
	}
	return time.Time{}
}

var summaryBuildMask = &field_mask.FieldMask{
	Paths: []string{
		"id",
		"builder",
		"number",
		"create_time",
		"start_time",
		"end_time",
		"update_time",
		"status",
		"summary_markdown",
		"tags",
		"infra.swarming",
		"input.experimental",
		"critical",
	},
}

// getSummary returns a model.BuildSummary representing a buildbucket build.
func getSummary(c context.Context, host string, project string, id int64) (*model.BuildSummary, error) {
	client, err := buildbucketClient(c, host, auth.AsProject, auth.WithProject(project))
	if err != nil {
		return nil, err
	}
	b, err := client.GetBuild(c, &buildbucketpb.GetBuildRequest{
		Id:     id,
		Fields: summaryBuildMask,
	})
	if err != nil {
		return nil, err
	}
	buildAddress := fmt.Sprintf("%d", b.Id)
	if b.Number != 0 {
		buildAddress = fmt.Sprintf("luci.%s.%s/%s/%d", b.Builder.Project, b.Builder.Bucket, b.Builder.Builder, b.Number)
	}

	// Note: The parent for buildbucket build summaries is currently a fake entity.
	// In the future, builds can be cached here, but we currently don't do that.
	buildKey := datastore.MakeKey(c, "buildbucket.Build", fmt.Sprintf("%s:%s", host, buildAddress))
	swarming := b.GetInfra().GetSwarming()

	bs := &model.BuildSummary{
		ProjectID:  b.Builder.Project,
		BuildKey:   buildKey,
		BuilderID:  BuilderID{*b.Builder}.String(),
		BuildID:    "buildbucket/" + buildAddress,
		BuildSet:   protoutil.BuildSets(b),
		ContextURI: []string{fmt.Sprintf("buildbucket://%s/build/%d", host, id)},
		Created:    mustTimestamp(b.CreateTime),
		Summary: model.Summary{
			Start:  mustTimestamp(b.StartTime),
			End:    mustTimestamp(b.EndTime),
			Status: statusMap[b.Status],
		},
		Version:      mustTimestamp(b.UpdateTime).UnixNano(),
		Experimental: b.GetInput().GetExperimental(),
		Critical:     b.GetCritical(),
	}
	if task := swarming.GetTaskId(); task != "" {
		bs.ContextURI = append(
			bs.ContextURI,
			fmt.Sprintf("swarming://%s/task/%s", swarming.GetHostname(), swarming.GetTaskId()))
	}
	return bs, nil
}

// pubSubHandlerImpl takes the http.Request, expects to find
// a common.PubSubSubscription JSON object in the Body, containing a bbPSEvent,
// and handles the contents with generateSummary.
func pubSubHandlerImpl(c context.Context, r *http.Request) error {
	// This is the default action. The code below will modify the values of some
	// or all of these parameters.
	isLUCI, bucket, status, action := false, "UNKNOWN", "UNKNOWN", "Rejected"

	defer func() {
		// closure for late binding
		buildCounter.Add(c, 1, bucket, isLUCI, status, action)
	}()

	msg := common.PubSubSubscription{}
	if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
		// This might be a transient error, e.g. when the json format changes
		// and Milo isn't updated yet.
		return errors.Annotate(err, "could not decode message").Tag(transient.Tag).Err()
	}
	if v, ok := msg.Message.Attributes["version"].(string); ok && v != "v1" {
		// TODO(nodir): switch to v2, crbug.com/826006
		logging.Debugf(c, "unsupported pubsub message version %q. Ignoring", v)
		return nil
	}
	bData, err := msg.GetData()
	if err != nil {
		return errors.Annotate(err, "could not parse pubsub message string").Err()
	}

	event := struct {
		Build    bbv1.LegacyApiCommonBuildMessage `json:"build"`
		Hostname string                           `json:"hostname"`
	}{}
	if err := json.Unmarshal(bData, &event); err != nil {
		return errors.Annotate(err, "could not parse pubsub message data").Err()
	}

	build := deprecated.Build{}
	if err := build.ParseMessage(&event.Build); err != nil {
		return errors.Annotate(err, "could not parse deprecated.Build").Err()
	}

	bucket = build.Bucket
	status = build.Status.String()
	isLUCI = strings.HasPrefix(bucket, "luci.")

	logging.Debugf(c, "Received from %s: build %s/%s (%s)\n%v",
		event.Hostname, bucket, build.Builder, status, build)

	if !isLUCI || build.Builder == "" {
		logging.Infof(c, "This is not an ingestable build, ignoring")
		return nil
	}

	// TODO(iannucci,nodir): get the bot context too
	// TODO(iannucci,nodir): support manifests/got_revision
	bs, err := getSummary(c, event.Hostname, build.Project, build.ID)
	if err != nil {
		return err
	}
	if err := bs.AddManifestKeysFromBuildSets(c); err != nil {
		return err
	}

	return transient.Tag.Apply(datastore.RunInTransaction(c, func(c context.Context) error {
		curBS := &model.BuildSummary{BuildKey: bs.BuildKey}
		switch err := datastore.Get(c, curBS); err {
		case datastore.ErrNoSuchEntity:
			action = "Created"
		case nil:
			action = "Modified"
		default:
			return errors.Annotate(err, "reading current BuildSummary").Err()
		}

		if bs.Version <= curBS.Version {
			logging.Warningf(c, "current BuildSummary is newer: %d <= %d",
				bs.Version, curBS.Version)
			return nil
		}

		if err := datastore.Put(c, bs); err != nil {
			return err
		}

		return model.UpdateBuilderForBuild(c, bs)
	}, &datastore.TransactionOptions{XG: true}))
}

// MakeBuildKey returns a new datastore Key for a buildbucket.Build.
//
// There's currently no model associated with this key, but it's used as
// a parent for a model.BuildSummary.
func MakeBuildKey(c context.Context, host, buildAddress string) *datastore.Key {
	return datastore.MakeKey(c,
		"buildbucket.Build", fmt.Sprintf("%s:%s", host, buildAddress))
}
<|endoftext|>