<commit_before>package corpse

import (
	"errors"
	"time"

	"github.com/ivan1993spb/snake-server/engine"
	"github.com/ivan1993spb/snake-server/world"
)

// Time for which corpse will be lie on playground
const corpseMaxExperience = time.Second * 15

// Snakes can eat corpses
type Corpse struct {
	world       *world.World
	location    engine.Location
	nippedPiece *engine.Dot // last nipped piece
	stop        chan struct{}
}

// Corpse are created when a snake dies
func NewCorpse(world *world.World, location engine.Location) (*Corpse, error) {
	// TODO: Check location.

	// TODO: Pass stopper.

	corpse := &Corpse{}

	if location, _ := world.CreateObjectAvailableDots(corpse, location); len(location) > 0 {
		corpse.world = world
		corpse.location = location
		corpse.stop = make(chan struct{}, 0)
		go corpse.run()
		return corpse, nil
	}

	return nil, errors.New("")
}

func (c *Corpse) NutritionalValue(dot *engine.Dot) uint16 {
	if c.location.Contains(dot) {
		newDots := c.location.Delete(dot)

		if len(c.location) > 0 {
			c.world.UpdateObjectAvailableDots(c, c.location, newDots)
			c.location = newDots
			c.nippedPiece = dot
		} else {
			c.world.DeleteObject(c, c.location)
			close(c.stop)
		}

		return 2
	}

	return 0
}

func (c *Corpse) run() {
	var timer = time.NewTimer(corpseMaxExperience)
	defer timer.Stop()
	select {
	case <-timer.C:
		c.world.DeleteObject(c, c.location)
		close(c.stop)
	case <-c.stop:
	}
}
<commit_msg>Implement Stringer interface in corpse.Corpse<commit_after>package corpse

import (
	"errors"
	"fmt"
	"time"

	"github.com/ivan1993spb/snake-server/engine"
	"github.com/ivan1993spb/snake-server/world"
)

// Time for which corpse will be lie on playground
const corpseMaxExperience = time.Second * 15

// Snakes can eat corpses
type Corpse struct {
	world       *world.World
	location    engine.Location
	nippedPiece *engine.Dot // last nipped piece
	stop        chan struct{}
}

// Corpse are created when a snake dies
func NewCorpse(world *world.World, location engine.Location) (*Corpse, error) {
	// TODO: Check location.

	// TODO: Pass stopper.

	corpse := &Corpse{}

	if location, _ := world.CreateObjectAvailableDots(corpse, location); len(location) > 0 {
		corpse.world = world
		corpse.location = location
		corpse.stop = make(chan struct{}, 0)
		go corpse.run()
		return corpse, nil
	}

	return nil, errors.New("")
}

func (c *Corpse) String() string {
	return fmt.Sprintln("corpse", c.location)
}

func (c *Corpse) NutritionalValue(dot *engine.Dot) uint16 {
	if c.location.Contains(dot) {
		newDots := c.location.Delete(dot)

		if len(c.location) > 0 {
			c.world.UpdateObjectAvailableDots(c, c.location, newDots)
			c.location = newDots
			c.nippedPiece = dot
		} else {
			c.world.DeleteObject(c, c.location)
			close(c.stop)
		}

		return 2
	}

	return 0
}

func (c *Corpse) run() {
	var timer = time.NewTimer(corpseMaxExperience)
	defer timer.Stop()
	select {
	case <-timer.C:
		c.world.DeleteObject(c, c.location)
		close(c.stop)
	case <-c.stop:
	}
}
<|endoftext|>


<commit_before>package csimanager

import (
	"context"
	"sync"
	"time"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nomad/client/dynamicplugins"
	"github.com/hashicorp/nomad/plugins/csi"
)

const managerFingerprintInterval = 30 * time.Second

// instanceManager is used to manage the fingerprinting and supervision of a
// single CSI Plugin.
type instanceManager struct {
	info   *dynamicplugins.PluginInfo
	logger hclog.Logger

	updater UpdateNodeCSIInfoFunc

	shutdownCtx         context.Context
	shutdownCtxCancelFn context.CancelFunc
	shutdownCh          chan struct{}

	// mountPoint is the root of the mount dir where plugin specific data may be
	// stored and where mount points will be created
	mountPoint string

	fp *pluginFingerprinter

	volumeManager        *volumeManager
	volumeManagerMu      sync.RWMutex
	volumeManagerSetupCh chan struct{}
	volumeManagerSetup   bool

	client csi.CSIPlugin
}

func newInstanceManager(logger hclog.Logger, updater UpdateNodeCSIInfoFunc, p *dynamicplugins.PluginInfo) *instanceManager {
	ctx, cancelFn := context.WithCancel(context.Background())
	logger = logger.Named(p.Name)
	return &instanceManager{
		logger:  logger,
		info:    p,
		updater: updater,

		fp: &pluginFingerprinter{
			logger:                logger.Named("fingerprinter"),
			info:                  p,
			fingerprintNode:       p.Type == dynamicplugins.PluginTypeCSINode,
			fingerprintController: p.Type == dynamicplugins.PluginTypeCSIController,
		},

		mountPoint: p.Options["MountPoint"],

		volumeManagerSetupCh: make(chan struct{}),

		shutdownCtx:         ctx,
		shutdownCtxCancelFn: cancelFn,
		shutdownCh:          make(chan struct{}),
	}
}

func (i *instanceManager) run() {
	c, err := csi.NewClient(i.info.ConnectionInfo.SocketPath, i.logger)
	if err != nil {
		i.logger.Error("failed to setup instance manager client", "error", err)
		close(i.shutdownCh)
		return
	}
	i.client = c

	go i.runLoop()
}

// VolumeMounter returns the volume manager that is configured for the given plugin
// instance. If called before the volume manager has been setup, it will block until
// the volume manager is ready or the context is closed.
func (i *instanceManager) VolumeMounter(ctx context.Context) (VolumeMounter, error) {
	var vm VolumeMounter
	i.volumeManagerMu.RLock()
	if i.volumeManagerSetup {
		vm = i.volumeManager
	}
	i.volumeManagerMu.RUnlock()

	if vm != nil {
		return vm, nil
	}

	select {
	case <-i.volumeManagerSetupCh:
		i.volumeManagerMu.RLock()
		vm = i.volumeManager
		i.volumeManagerMu.RUnlock()
		return vm, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (i *instanceManager) requestCtxWithTimeout(timeout time.Duration) (context.Context, context.CancelFunc) {
	return context.WithTimeout(i.shutdownCtx, timeout)
}

func (i *instanceManager) runLoop() {
	timer := time.NewTimer(0)
	for {
		select {
		case <-i.shutdownCtx.Done():
			if i.client != nil {
				i.client.Close()
				i.client = nil
			}
			close(i.shutdownCh)
			return
		case <-timer.C:
			ctx, cancelFn := i.requestCtxWithTimeout(managerFingerprintInterval)

			info := i.fp.fingerprint(ctx)
			cancelFn()
			i.updater(i.info.Name, info)

			// TODO: refactor this lock into a faster, goroutine-local check
			i.volumeManagerMu.RLock()
			// When we've had a successful fingerprint, and the volume manager is not yet setup,
			// and one is required (we're running a node plugin), then set one up now.
			if i.fp.hadFirstSuccessfulFingerprint && !i.volumeManagerSetup && i.fp.fingerprintNode {
				i.volumeManagerMu.RUnlock()
				i.volumeManagerMu.Lock()
				i.volumeManager = newVolumeManager(i.logger, i.client, i.mountPoint, info.NodeInfo.RequiresNodeStageVolume)
				i.volumeManagerSetup = true
				close(i.volumeManagerSetupCh)
				i.volumeManagerMu.Unlock()
			} else {
				i.volumeManagerMu.RUnlock()
			}

			timer.Reset(managerFingerprintInterval)
		}
	}
}

func (i *instanceManager) shutdown() {
	i.shutdownCtxCancelFn()
	<-i.shutdownCh
}
<commit_msg>csimanager: Instantiate fingerprint manager's csiclient<commit_after>package csimanager

import (
	"context"
	"sync"
	"time"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nomad/client/dynamicplugins"
	"github.com/hashicorp/nomad/plugins/csi"
)

const managerFingerprintInterval = 30 * time.Second

// instanceManager is used to manage the fingerprinting and supervision of a
// single CSI Plugin.
type instanceManager struct {
	info   *dynamicplugins.PluginInfo
	logger hclog.Logger

	updater UpdateNodeCSIInfoFunc

	shutdownCtx         context.Context
	shutdownCtxCancelFn context.CancelFunc
	shutdownCh          chan struct{}

	// mountPoint is the root of the mount dir where plugin specific data may be
	// stored and where mount points will be created
	mountPoint string

	fp *pluginFingerprinter

	volumeManager        *volumeManager
	volumeManagerMu      sync.RWMutex
	volumeManagerSetupCh chan struct{}
	volumeManagerSetup   bool

	client csi.CSIPlugin
}

func newInstanceManager(logger hclog.Logger, updater UpdateNodeCSIInfoFunc, p *dynamicplugins.PluginInfo) *instanceManager {
	ctx, cancelFn := context.WithCancel(context.Background())
	logger = logger.Named(p.Name)
	return &instanceManager{
		logger:  logger,
		info:    p,
		updater: updater,

		fp: &pluginFingerprinter{
			logger:                logger.Named("fingerprinter"),
			info:                  p,
			fingerprintNode:       p.Type == dynamicplugins.PluginTypeCSINode,
			fingerprintController: p.Type == dynamicplugins.PluginTypeCSIController,
		},

		mountPoint: p.Options["MountPoint"],

		volumeManagerSetupCh: make(chan struct{}),

		shutdownCtx:         ctx,
		shutdownCtxCancelFn: cancelFn,
		shutdownCh:          make(chan struct{}),
	}
}

func (i *instanceManager) run() {
	c, err := csi.NewClient(i.info.ConnectionInfo.SocketPath, i.logger)
	if err != nil {
		i.logger.Error("failed to setup instance manager client", "error", err)
		close(i.shutdownCh)
		return
	}
	i.client = c
	i.fp.client = c

	go i.runLoop()
}

// VolumeMounter returns the volume manager that is configured for the given plugin
// instance. If called before the volume manager has been setup, it will block until
// the volume manager is ready or the context is closed.
func (i *instanceManager) VolumeMounter(ctx context.Context) (VolumeMounter, error) {
	var vm VolumeMounter
	i.volumeManagerMu.RLock()
	if i.volumeManagerSetup {
		vm = i.volumeManager
	}
	i.volumeManagerMu.RUnlock()

	if vm != nil {
		return vm, nil
	}

	select {
	case <-i.volumeManagerSetupCh:
		i.volumeManagerMu.RLock()
		vm = i.volumeManager
		i.volumeManagerMu.RUnlock()
		return vm, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (i *instanceManager) requestCtxWithTimeout(timeout time.Duration) (context.Context, context.CancelFunc) {
	return context.WithTimeout(i.shutdownCtx, timeout)
}

func (i *instanceManager) runLoop() {
	timer := time.NewTimer(0)
	for {
		select {
		case <-i.shutdownCtx.Done():
			if i.client != nil {
				i.client.Close()
				i.client = nil
			}
			close(i.shutdownCh)
			return
		case <-timer.C:
			ctx, cancelFn := i.requestCtxWithTimeout(managerFingerprintInterval)

			info := i.fp.fingerprint(ctx)
			cancelFn()
			i.updater(i.info.Name, info)

			// TODO: refactor this lock into a faster, goroutine-local check
			i.volumeManagerMu.RLock()
			// When we've had a successful fingerprint, and the volume manager is not yet setup,
			// and one is required (we're running a node plugin), then set one up now.
			if i.fp.hadFirstSuccessfulFingerprint && !i.volumeManagerSetup && i.fp.fingerprintNode {
				i.volumeManagerMu.RUnlock()
				i.volumeManagerMu.Lock()
				i.volumeManager = newVolumeManager(i.logger, i.client, i.mountPoint, info.NodeInfo.RequiresNodeStageVolume)
				i.volumeManagerSetup = true
				close(i.volumeManagerSetupCh)
				i.volumeManagerMu.Unlock()
			} else {
				i.volumeManagerMu.RUnlock()
			}

			timer.Reset(managerFingerprintInterval)
		}
	}
}

func (i *instanceManager) shutdown() {
	i.shutdownCtxCancelFn()
	<-i.shutdownCh
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/rschoen/fireworks-server/lib"
	"net/http"
)

func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path[1:5] != "api/" {
		http.FileServer(http.Dir(lib.ClientDirectory)).ServeHTTP(w, r)
		return
	}

	m, ok := lib.DecodeMove(r.PostFormValue("data"))
	if !ok {
		fmt.Printf("Malformed message, Discarding.")
		fmt.Fprintf(w, "Error: malformed JSON message.")
		return
	}
	var game *lib.Game
	for _, ongoingGame := range s.games {
		if ongoingGame.GameID == m.Game {
			game = ongoingGame
		}
	}

	if r.URL.Path[5:] == "join" {
		// create game if it doesn't exist
		if game == nil {
			game = new(lib.Game)
			game.GameID = m.Game
			game.Initialize()
			s.games = append(s.games, game)
		} else {
			fmt.Println(game)
		}

		player := game.GetPlayerByID(m.Player)
		// add player if it doesn't exist
		if player == nil {
			game.AddPlayer(m.Player)
		} else {
			fmt.Println(player)
		}

		fmt.Fprintf(w, lib.EncodeGame(game.CreateState(m.Player)))
		return
	}

	player := game.GetPlayerByID(m.Player)

	if game == nil {
		fmt.Printf("Attempting to make a move on a nonexistent game.")
		fmt.Fprintf(w, "Error: Attempted to make a move on nonexistent game.")
		return
	}

	if player == nil {
		fmt.Printf("Attempting to make a move with nonexistent player.")
		fmt.Fprintf(w, "Error: Attempting to make a move with nonexistent player.")
		return
	}

	if r.URL.Path[5:] == "start" {
		if game.Started {
			fmt.Printf("Attempting to start already started game.")
			fmt.Fprintf(w, "Error: Attempting to start already started game.")
			return
		}
		game.Start()
		fmt.Fprintf(w, lib.EncodeGame(game.CreateState(m.Player)))
		return
	}

	if r.URL.Path[5:] == "move" {
		game.ProcessMove(m)
		fmt.Printf("Global game state: %#v\n\n", *game)
		fmt.Fprintf(w, lib.EncodeGame(game.CreateState(m.Player)))
		return
	}

	fmt.Fprintf(w, "done")
}

type Server struct {
	games []*lib.Game
}

func main() {
	// check to make sure no other server is running

	// initialize server
	s := Server{}
	s.games = make([]*lib.Game, 0, lib.MaxConcurrentGames)

	// listen for connections
	http.HandleFunc("/", s.handler)
	http.ListenAndServe(":"+lib.Port, nil)

}
<commit_msg>Fix slice out-of-bounds on interpreting URL<commit_after>package main

import (
	"fmt"
	"github.com/rschoen/fireworks-server/lib"
	"net/http"
)

func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
	if len(r.URL.Path) < 5 || r.URL.Path[1:5] != "api/" {
		http.FileServer(http.Dir(lib.ClientDirectory)).ServeHTTP(w, r)
		return
	}

	m, ok := lib.DecodeMove(r.PostFormValue("data"))
	if !ok {
		fmt.Printf("Malformed message, Discarding.")
		fmt.Fprintf(w, "Error: malformed JSON message.")
		return
	}
	var game *lib.Game
	for _, ongoingGame := range s.games {
		if ongoingGame.GameID == m.Game {
			game = ongoingGame
		}
	}

	if r.URL.Path[5:] == "join" {
		// create game if it doesn't exist
		if game == nil {
			game = new(lib.Game)
			game.GameID = m.Game
			game.Initialize()
			s.games = append(s.games, game)
		} else {
			fmt.Println(game)
		}

		player := game.GetPlayerByID(m.Player)
		// add player if it doesn't exist
		if player == nil {
			game.AddPlayer(m.Player)
		} else {
			fmt.Println(player)
		}

		fmt.Fprintf(w, lib.EncodeGame(game.CreateState(m.Player)))
		return
	}

	player := game.GetPlayerByID(m.Player)

	if game == nil {
		fmt.Printf("Attempting to make a move on a nonexistent game.")
		fmt.Fprintf(w, "Error: Attempted to make a move on nonexistent game.",)
		return
	}

	if player == nil {
		fmt.Printf("Attempting to make a move with nonexistent player.")
		fmt.Fprintf(w, "Error: Attempting to make a move with nonexistent player.",)
		return
	}

	if r.URL.Path[5:] == "start" {
		if game.Started {
			fmt.Printf("Attempting to start already started game.")
			fmt.Fprintf(w, "Error: Attempting to start already started game.",)
			return
		}
		game.Start()
		fmt.Fprintf(w, lib.EncodeGame(game.CreateState(m.Player)))
		return
	}

	if r.URL.Path[5:] == "move" {
		game.ProcessMove(m)
		fmt.Printf("Global game state: %#v\n\n", *game)
		fmt.Fprintf(w, lib.EncodeGame(game.CreateState(m.Player)))
		return
	}

    fmt.Fprintf(w, "done")
}

type Server struct {
	games   []*lib.Game
}

func main() {
	// check to make sure no other server is running

	// initialize server
	s := Server{}
	s.games = make([]*lib.Game, 0, lib.MaxConcurrentGames)

	// listen for connections
	http.HandleFunc("/", s.handler)
    http.ListenAndServe(":"+lib.Port, nil)

}

<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"fmt"

	"github.com/coreos/fleet/job"
)

var (
	flagLines  int
	flagFollow bool
	flagSudo   bool
	cmdJournal = &Command{
		Name:    "journal",
		Summary: "Print the journal of a unit in the cluster to stdout",
		Usage:   "[--lines=N] [-f|--follow] <unit>",
		Run:     runJournal,
		Description: `Outputs the journal of a unit by connecting to the machine that the unit occupies.

Read the last 10 lines:
	fleetctl journal foo.service

Read the last 100 lines:
	fleetctl journal --lines 100 foo.service

This command does not work with global units.`,
	}
)

func init() {
	cmdJournal.Flags.IntVar(&flagLines, "lines", 10, "Number of recent log lines to return")
	cmdJournal.Flags.BoolVar(&flagFollow, "follow", false, "Continuously print new entries as they are appended to the journal.")
	cmdJournal.Flags.BoolVar(&flagFollow, "f", false, "Shorthand for --follow")
	cmdJournal.Flags.BoolVar(&flagSudo, "sudo", false, "Execute journal command with sudo")
	cmdJournal.Flags.BoolVar(&flagSudo, "s", false, "Shorthand for --sudo")
}

func runJournal(args []string) (exit int) {
	if len(args) != 1 {
		stderr("One unit file must be provided.")
		return 1
	}
	name := unitNameMangle(args[0])

	u, err := cAPI.Unit(name)
	if err != nil {
		stderr("Error retrieving unit %s: %v", name, err)
		return 1
	} else if u == nil {
		stderr("Unit %s does not exist.", name)
		return 1
	} else if suToGlobal(*u) {
		stderr("Unable to retrieve journal of global unit %s.", name)
		return 1
	} else if job.JobState(u.CurrentState) == job.JobStateInactive {
		stderr("Unit %s does not appear to be running.", name)
		return 1
	}

	command := fmt.Sprintf("journalctl --unit %s --no-pager -n %d", name, flagLines)

	if flagSudo {
		command = "sudo " + command
	}

	if flagFollow {
		command += " -f"
	}

	return runCommand(command, u.MachineID)
}
<commit_msg>fleetctl: Remove shortened sudo option<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"fmt"

	"github.com/coreos/fleet/job"
)

var (
	flagLines  int
	flagFollow bool
	flagSudo   bool
	cmdJournal = &Command{
		Name:    "journal",
		Summary: "Print the journal of a unit in the cluster to stdout",
		Usage:   "[--lines=N] [-f|--follow] <unit>",
		Run:     runJournal,
		Description: `Outputs the journal of a unit by connecting to the machine that the unit occupies.

Read the last 10 lines:
	fleetctl journal foo.service

Read the last 100 lines:
	fleetctl journal --lines 100 foo.service

This command does not work with global units.`,
	}
)

func init() {
	cmdJournal.Flags.IntVar(&flagLines, "lines", 10, "Number of recent log lines to return")
	cmdJournal.Flags.BoolVar(&flagFollow, "follow", false, "Continuously print new entries as they are appended to the journal.")
	cmdJournal.Flags.BoolVar(&flagFollow, "f", false, "Shorthand for --follow")
	cmdJournal.Flags.BoolVar(&flagSudo, "sudo", false, "Execute journal command with sudo")
}

func runJournal(args []string) (exit int) {
	if len(args) != 1 {
		stderr("One unit file must be provided.")
		return 1
	}
	name := unitNameMangle(args[0])

	u, err := cAPI.Unit(name)
	if err != nil {
		stderr("Error retrieving unit %s: %v", name, err)
		return 1
	} else if u == nil {
		stderr("Unit %s does not exist.", name)
		return 1
	} else if suToGlobal(*u) {
		stderr("Unable to retrieve journal of global unit %s.", name)
		return 1
	} else if job.JobState(u.CurrentState) == job.JobStateInactive {
		stderr("Unit %s does not appear to be running.", name)
		return 1
	}

	command := fmt.Sprintf("journalctl --unit %s --no-pager -n %d", name, flagLines)

	if flagSudo {
		command = "sudo " + command
	}

	if flagFollow {
		command += " -f"
	}

	return runCommand(command, u.MachineID)
}
<|endoftext|>


<commit_before>// Copyright 2016-2021 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package libsacloud

// Version バージョン
const Version = "2.10.0"
<commit_msg>Bump to v2.11.0<commit_after>// Copyright 2016-2021 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package libsacloud

// Version バージョン
const Version = "2.11.0"
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-couchstore"
)

type dbOperation uint8

const (
	db_store_item = dbOperation(iota)
	db_delete_item
	db_compact
)

type dbqitem struct {
	dbname string
	k      string
	data   []byte
	op     dbOperation
}

type dbWriter struct {
	ch   chan dbqitem
	quit chan bool
	db   *couchstore.Couchstore
}

var dbLock = sync.Mutex{}
var dbConns = map[string]*dbWriter{}

func dbPath(n string) string {
	return filepath.Join(*dbRoot, n) + ".couch"
}

func dbBase(n string) string {
	left := 0
	right := len(n)
	if strings.HasPrefix(n, *dbRoot) {
		left = len(*dbRoot)
		if n[left] == '/' {
			left++
		}
	}
	if strings.HasSuffix(n, ".couch") {
		right = len(n) - 6
	}
	return n[left:right]
}

func dbopen(name string) (*couchstore.Couchstore, error) {
	db, err := couchstore.Open(dbPath(name), false)
	return db, err
}

func dbcreate(path string) error {
	db, err := couchstore.Open(path, true)
	if err != nil {
		return err
	}
	db.Close()
	return nil
}

func dbdelete(dbname string) error {
	dbLock.Lock()
	defer dbLock.Unlock()

	writer := dbConns[dbname]
	if writer != nil {
		writer.quit <- true
	}
	delete(dbConns, dbname)

	return os.Remove(dbPath(dbname))
}

func dblist(root string) []string {
	rv := []string{}
	filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
		if !info.IsDir() && strings.HasSuffix(p, ".couch") {
			rv = append(rv, dbBase(p))
		}
		return err
	})
	return rv
}

func dbWriteLoop(dq *dbWriter) {
	defer dq.db.Close()

	queued := 0
	bulk := dq.db.Bulk()
	defer bulk.Close()
	defer bulk.Commit()

	t := time.NewTimer(*flushTime)

	for {
		select {
		case <-dq.quit:
			return
		case qi := <-dq.ch:
			switch qi.op {
			case db_store_item:
				bulk.Set(couchstore.NewDocInfo(qi.k,
					couchstore.DocIsCompressed),
					couchstore.NewDocument(qi.k,
						string(qi.data)))
			case db_delete_item:
				bulk.Delete(couchstore.NewDocInfo(qi.k, 0))
			default:
				log.Panicf("Unhandled case: %v", qi.op)
			}
			queued++
			if queued >= *maxOpQueue {
				start := time.Now()
				bulk.Commit()
				log.Printf("Flush of %d items took %v",
					queued, time.Since(start))
				queued = 0
				t.Stop()
				t = time.NewTimer(*flushTime)
			}
		case <-t.C:
			if queued > 0 {
				start := time.Now()
				bulk.Commit()
				log.Printf("Flush of %d items from timer took %v",
					queued, time.Since(start))
				queued = 0
			}
			t = time.NewTimer(*flushTime)
		}
	}
}

func dbWriteFun(dbname string) (*dbWriter, error) {
	db, err := dbopen(dbname)
	if err != nil {
		return nil, err
	}

	writer := &dbWriter{
		make(chan dbqitem),
		make(chan bool),
		db,
	}

	go dbWriteLoop(writer)

	return writer, nil
}

func getOrCreateDB(dbname string) (*dbWriter, error) {
	dbLock.Lock()
	defer dbLock.Unlock()

	writer := dbConns[dbname]
	var err error
	if writer == nil {
		writer, err = dbWriteFun(dbname)
		if err != nil {
			return nil, err
		}
		dbConns[dbname] = writer
	}
	return writer, nil
}

func dbstore(dbname string, k string, body []byte) error {
	writer, err := getOrCreateDB(dbname)
	if err != nil {
		return err
	}

	writer.ch <- dbqitem{dbname, k, body, db_store_item}

	return nil
}

func parseKey(s string) int64 {
	t, err := time.Parse(time.RFC3339Nano, s)
	if err != nil {
		return -1
	}
	return t.UnixNano()
}
<commit_msg>Allow a bit of a message backlog on the op buffering.<commit_after>package main

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-couchstore"
)

type dbOperation uint8

const (
	db_store_item = dbOperation(iota)
	db_delete_item
	db_compact
)

type dbqitem struct {
	dbname string
	k      string
	data   []byte
	op     dbOperation
}

type dbWriter struct {
	ch   chan dbqitem
	quit chan bool
	db   *couchstore.Couchstore
}

var dbLock = sync.Mutex{}
var dbConns = map[string]*dbWriter{}

func dbPath(n string) string {
	return filepath.Join(*dbRoot, n) + ".couch"
}

func dbBase(n string) string {
	left := 0
	right := len(n)
	if strings.HasPrefix(n, *dbRoot) {
		left = len(*dbRoot)
		if n[left] == '/' {
			left++
		}
	}
	if strings.HasSuffix(n, ".couch") {
		right = len(n) - 6
	}
	return n[left:right]
}

func dbopen(name string) (*couchstore.Couchstore, error) {
	db, err := couchstore.Open(dbPath(name), false)
	return db, err
}

func dbcreate(path string) error {
	db, err := couchstore.Open(path, true)
	if err != nil {
		return err
	}
	db.Close()
	return nil
}

func dbdelete(dbname string) error {
	dbLock.Lock()
	defer dbLock.Unlock()

	writer := dbConns[dbname]
	if writer != nil {
		writer.quit <- true
	}
	delete(dbConns, dbname)

	return os.Remove(dbPath(dbname))
}

func dblist(root string) []string {
	rv := []string{}
	filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
		if !info.IsDir() && strings.HasSuffix(p, ".couch") {
			rv = append(rv, dbBase(p))
		}
		return err
	})
	return rv
}

func dbWriteLoop(dq *dbWriter) {
	defer dq.db.Close()

	queued := 0
	bulk := dq.db.Bulk()
	defer bulk.Close()
	defer bulk.Commit()

	t := time.NewTimer(*flushTime)

	for {
		select {
		case <-dq.quit:
			return
		case qi := <-dq.ch:
			switch qi.op {
			case db_store_item:
				bulk.Set(couchstore.NewDocInfo(qi.k,
					couchstore.DocIsCompressed),
					couchstore.NewDocument(qi.k,
						string(qi.data)))
			case db_delete_item:
				bulk.Delete(couchstore.NewDocInfo(qi.k, 0))
			default:
				log.Panicf("Unhandled case: %v", qi.op)
			}
			queued++
			if queued >= *maxOpQueue {
				start := time.Now()
				bulk.Commit()
				log.Printf("Flush of %d items took %v",
					queued, time.Since(start))
				queued = 0
				t.Stop()
				t = time.NewTimer(*flushTime)
			}
		case <-t.C:
			if queued > 0 {
				start := time.Now()
				bulk.Commit()
				log.Printf("Flush of %d items from timer took %v",
					queued, time.Since(start))
				queued = 0
			}
			t = time.NewTimer(*flushTime)
		}
	}
}

func dbWriteFun(dbname string) (*dbWriter, error) {
	db, err := dbopen(dbname)
	if err != nil {
		return nil, err
	}

	writer := &dbWriter{
		make(chan dbqitem, *maxOpQueue),
		make(chan bool),
		db,
	}

	go dbWriteLoop(writer)

	return writer, nil
}

func getOrCreateDB(dbname string) (*dbWriter, error) {
	dbLock.Lock()
	defer dbLock.Unlock()

	writer := dbConns[dbname]
	var err error
	if writer == nil {
		writer, err = dbWriteFun(dbname)
		if err != nil {
			return nil, err
		}
		dbConns[dbname] = writer
	}
	return writer, nil
}

func dbstore(dbname string, k string, body []byte) error {
	writer, err := getOrCreateDB(dbname)
	if err != nil {
		return err
	}

	writer.ch <- dbqitem{dbname, k, body, db_store_item}

	return nil
}

func parseKey(s string) int64 {
	t, err := time.Parse(time.RFC3339Nano, s)
	if err != nil {
		return -1
	}
	return t.UnixNano()
}
<|endoftext|>


<commit_before>// Package blaze integrates the mgojq package to handle asynchronous jobs.
package blaze

import (
	"time"

	"github.com/256dpi/fire"
	"github.com/256dpi/fire/coal"
	"github.com/256dpi/mgojq"
	"gopkg.in/mgo.v2/bson"
)

// Job is the coal model for the mgojq.Job type.
type Job struct {
	coal.Base `json:"-" bson:",inline" coal:"jobs"`
	Name      string    `json:"name"`
	Params    bson.M    `json:"params"`
	Status    string    `json:"status"`
	Created   time.Time `json:"created"`
	Delayed   time.Time `json:"delayed"`
	Started   time.Time `json:"started"`
	Ended     time.Time `json:"ended"`
	Attempts  int       `json:"attempts"`
	Result    bson.M    `json:"result"`
	Error     string    `json:"error"`
	Reason    string    `json:"reason"`
}

// JobController will return a basic controller that provides access to the jobs.
// At least one authorizer should be provided that restricts access to administrators.
func JobController(store *coal.Store, authorizers ...fire.Callback) *fire.Controller {
	return &fire.Controller{
		Model:       &Job{},
		Sorters:     []string{"name", "status", "created", "started", "ended", "attempts"},
		Filters:     []string{"name", "status"},
		Store:       store,
		Authorizers: authorizers,
	}
}

// C will return the correct mgojq.Collection.
func C(store *coal.SubStore) *mgojq.Collection {
	return mgojq.Wrap(store.C(&Job{}))
}
<commit_msg>added omitempty tags<commit_after>// Package blaze integrates the mgojq package to handle asynchronous jobs.
package blaze

import (
	"time"

	"github.com/256dpi/fire"
	"github.com/256dpi/fire/coal"
	"github.com/256dpi/mgojq"
	"gopkg.in/mgo.v2/bson"
)

// Job is the coal model for the mgojq.Job type.
type Job struct {
	coal.Base `json:"-" bson:",inline" coal:"jobs"`
	Name      string    `json:"name"`
	Params    bson.M    `json:"params"`
	Status    string    `json:"status"`
	Created   time.Time `json:"created"`
	Delayed   time.Time `json:"delayed" bson:",omitempty"`
	Started   time.Time `json:"started" bson:",omitempty"`
	Ended     time.Time `json:"ended" bson:",omitempty"`
	Attempts  int       `json:"attempts"`
	Result    bson.M    `json:"result" bson:",omitempty"`
	Error     string    `json:"error" bson:",omitempty"`
	Reason    string    `json:"reason" bson:",omitempty"`
}

// JobController will return a basic controller that provides access to the jobs.
// At least one authorizer should be provided that restricts access to administrators.
func JobController(store *coal.Store, authorizers ...fire.Callback) *fire.Controller {
	return &fire.Controller{
		Model:       &Job{},
		Sorters:     []string{"name", "status", "created", "started", "ended", "attempts"},
		Filters:     []string{"name", "status"},
		Store:       store,
		Authorizers: authorizers,
	}
}

// C will return the correct mgojq.Collection.
func C(store *coal.SubStore) *mgojq.Collection {
	return mgojq.Wrap(store.C(&Job{}))
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"strconv"
	"sync"
	"time"
)

const (
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer = 16384
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs = 100
	// Max delay before forcing a flush to Elasticearch
	BulkDelaySeconds = 5
	// maximum wait shutdown seconds
	MAX_SHUTDOWN_SECS = 5
)

type ErrorBuffer struct {
	Err error
	Buf *bytes.Buffer
}

// A bulk indexer creates goroutines, and channels for connecting and sending data
// to elasticsearch in bulk, using buffers.
type BulkIndexer struct {
	conn *Conn

	// We are creating a variable defining the func responsible for sending
	// to allow a mock sendor for test purposes
	Sender func(*bytes.Buffer) error

	// The refresh parameter can be set to true in order to refresh the
	// relevant primary and replica shards immediately after the bulk
	// operation has occurred
	Refresh bool

	// If we encounter an error in sending, we are going to retry for this long
	// before returning an error
	// if 0 it will not retry
	RetryForSeconds int

	// channel for getting errors
	ErrorChannel chan *ErrorBuffer

	// channel for sending to background indexer
	bulkChannel chan []byte

	// numErrors is a running total of errors seen
	numErrors uint64

	// shutdown channel
	shutdownChan chan chan struct{}
	// channel to shutdown timer
	timerDoneChan chan struct{}

	// Channel to send a complete byte.Buffer to the http sendor
	sendBuf chan *bytes.Buffer
	// byte buffer for docs that have been converted to bytes, but not yet sent
	buf *bytes.Buffer
	// Buffer for Max number of time before forcing flush
	BufferDelayMax time.Duration
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer int // 1048576
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs int // 100

	// Number of documents we have send through so far on this session
	docCt int
	// Max number of http conns in flight at one time
	maxConns int
	// If we are indexing enough docs per bufferdelaymax, we won't need to do time
	// based eviction, else we do.
	needsTimeBasedFlush bool
	// Lock for document writes/operations
	mu sync.Mutex
	// Wait Group for the http sends
	sendWg *sync.WaitGroup
}

func (b *BulkIndexer) NumErrors() uint64 {
	return b.numErrors
}

func (c *Conn) NewBulkIndexer(maxConns int) *BulkIndexer {
	b := BulkIndexer{conn: c, sendBuf: make(chan *bytes.Buffer, maxConns)}
	b.needsTimeBasedFlush = true
	b.buf = new(bytes.Buffer)
	b.maxConns = maxConns
	b.BulkMaxBuffer = BulkMaxBuffer
	b.BulkMaxDocs = BulkMaxDocs
	b.BufferDelayMax = time.Duration(BulkDelaySeconds) * time.Second
	b.bulkChannel = make(chan []byte, 100)
	b.sendWg = new(sync.WaitGroup)
	b.timerDoneChan = make(chan struct{})
	return &b
}

// A bulk indexer with more control over error handling
//    @maxConns is the max number of in flight http requests
//    @retrySeconds is # of seconds to wait before retrying falied requests
//
//   done := make(chan bool)
//   BulkIndexerGlobalRun(100, done)
func (c *Conn) NewBulkIndexerErrors(maxConns, retrySeconds int) *BulkIndexer {
	b := c.NewBulkIndexer(maxConns)
	b.RetryForSeconds = retrySeconds
	b.ErrorChannel = make(chan *ErrorBuffer, 20)
	return b
}

// Starts this bulk Indexer running, this Run opens a go routine so is
// Non blocking
func (b *BulkIndexer) Start() {
	b.shutdownChan = make(chan chan struct{})

	go func() {
		// XXX(j): Refactor this stuff to use an interface.
		if b.Sender == nil {
			b.Sender = b.Send
		}
		// Backwards compatibility
		b.startHttpSender()
		b.startDocChannel()
		b.startTimer()
		ch := <-b.shutdownChan
		time.Sleep(2 * time.Millisecond)
		b.Flush()
		b.shutdown()
		ch <- struct{}{}
	}()
}

// Stop stops the bulk indexer, blocking the caller until it is complete.
func (b *BulkIndexer) Stop() {
	ch := make(chan struct{})
	b.shutdownChan <- ch
	select {
	case <-ch:
		// done
	case <-time.After(time.Second * time.Duration(MAX_SHUTDOWN_SECS)):
		// timeout!
	}
}

// Make a channel that will close when the given WaitGroup is done.
func wgChan(wg *sync.WaitGroup) <-chan interface{} {
	ch := make(chan interface{})
	go func() {
		wg.Wait()
		close(ch)
	}()
	return ch
}

func (b *BulkIndexer) PendingDocuments() int {
	return b.docCt
}

// Flush all current documents to ElasticSearch
func (b *BulkIndexer) Flush() {
	b.mu.Lock()
	if b.docCt > 0 {
		b.send(b.buf)
	}
	b.mu.Unlock()
}

func (b *BulkIndexer) startHttpSender() {

	// this sends http requests to elasticsearch it uses maxConns to open up that
	// many goroutines, each of which will synchronously call ElasticSearch
	// in theory, the whole set will cause a backup all the way to IndexBulk if
	// we have consumed all maxConns
	for i := 0; i < b.maxConns; i++ {
		go func() {
			for buf := range b.sendBuf {
				b.sendWg.Add(1)
				// Copy for the potential re-send.
				bufCopy := bytes.NewBuffer(buf.Bytes())
				err := b.Sender(buf)

				// Perhaps a b.FailureStrategy(err)  ??  with different types of strategies
				//  1.  Retry, then panic
				//  2.  Retry then return error and let runner decide
				//  3.  Retry, then log to disk?   retry later?
				if err != nil {
					if b.RetryForSeconds > 0 {
						time.Sleep(time.Second * time.Duration(b.RetryForSeconds))
						err = b.Sender(bufCopy)
						if err == nil {
							// Successfully re-sent with no error
							b.sendWg.Done()
							continue
						}
					}
					if b.ErrorChannel != nil {
						b.ErrorChannel <- &ErrorBuffer{err, buf}
					}
				}
				b.sendWg.Done()
			}
		}()
	}
}

// start a timer for checking back and forcing flush ever BulkDelaySeconds seconds
// even if we haven't hit max messages/size
func (b *BulkIndexer) startTimer() {
	ticker := time.NewTicker(b.BufferDelayMax)
	go func() {
		for {
			select {
			case <-ticker.C:
				b.mu.Lock()
				// don't send unless last sendor was the time,
				// otherwise an indication of other thresholds being hit
				// where time isn't needed
				if b.buf.Len() > 0 && b.needsTimeBasedFlush {
					b.needsTimeBasedFlush = true
					b.send(b.buf)
				} else if b.buf.Len() > 0 {
					b.needsTimeBasedFlush = true
				}
				b.mu.Unlock()
			case <-b.timerDoneChan:
				// shutdown this go routine
				return
			}

		}
	}()
}

func (b *BulkIndexer) startDocChannel() {
	// This goroutine accepts incoming byte arrays from the IndexBulk function and
	// writes to buffer
	go func() {
		for docBytes := range b.bulkChannel {
			b.mu.Lock()
			b.docCt += 1
			b.buf.Write(docBytes)
			if b.buf.Len() >= b.BulkMaxBuffer || b.docCt >= b.BulkMaxDocs {
				b.needsTimeBasedFlush = false
				//log.Printf("Send due to size:  docs=%d  bufsize=%d", b.docCt, b.buf.Len())
				b.send(b.buf)
			}
			b.mu.Unlock()
		}
	}()
}

func (b *BulkIndexer) send(buf *bytes.Buffer) {
	//b2 := *b.buf
	b.sendBuf <- buf
	b.buf = new(bytes.Buffer)
	//	b.buf.Reset()
	b.docCt = 0
}

func (b *BulkIndexer) shutdown() {
	// This must be called after Flush()
	close(b.timerDoneChan)
	close(b.sendBuf)
	close(b.bulkChannel)
	<-wgChan(b.sendWg)
}

// The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
// it operates by buffering requests, and ocassionally flushing to elasticsearch
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Delete(index, _type, id string) {
	queryLine := fmt.Sprintf("{\"delete\":{\"_index\":%q,\"_type\":%q,\"_id\":%q}}\n", index, _type, id)
	b.bulkChannel <- []byte(queryLine)
	return
}

func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, parent, ttl string, date *time.Time, script string) error {

	var data map[string]interface{} = make(map[string]interface{})
	data["script"] = script
	return b.Update(index, _type, id, parent, ttl, date, data)
}

func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool) error {

	var data map[string]interface{} = make(map[string]interface{})

	data["doc"] = partialDoc
	if upsert {
		data["doc_as_upsert"] = true
	}
	return b.Update(index, _type, id, parent, ttl, date, data)
}

// This does the actual send of a buffer, which has already been formatted
// into bytes of ES formatted bulk data
func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
	type responseStruct struct {
		Took   int64                    `json:"took"`
		Errors bool                     `json:"errors"`
		Items  []map[string]interface{} `json:"items"`
	}

	response := responseStruct{}

	body, err := b.conn.DoCommand("POST", fmt.Sprintf("/_bulk?refresh=%t", b.Refresh), nil, buf)

	if err != nil {
		b.numErrors += 1
		return err
	}
	// check for response errors, bulk insert will give 200 OK but then include errors in response
	jsonErr := json.Unmarshal(body, &response)
	if jsonErr == nil {
		if response.Errors {
			b.numErrors += uint64(len(response.Items))
			return fmt.Errorf("Bulk Insertion Error. Failed item count [%d]", len(response.Items))
		}
	}
	return nil
}

// Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) ([]byte, error) {
	// only index and update are currently supported
	if op != "index" && op != "update" {
		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
	}

	// First line
	buf := bytes.Buffer{}
	buf.WriteString(fmt.Sprintf(`{"%s":{"_index":"`, op))
	buf.WriteString(index)
	buf.WriteString(`","_type":"`)
	buf.WriteString(_type)
	buf.WriteString(`"`)
	if len(id) > 0 {
		buf.WriteString(`,"_id":"`)
		buf.WriteString(id)
		buf.WriteString(`"`)
	}

	if len(parent) > 0 {
		buf.WriteString(`,"_parent":"`)
		buf.WriteString(parent)
		buf.WriteString(`"`)
	}

	if op == "update" {
		buf.WriteString(`,"_retry_on_conflict":3`)
	}

	if len(ttl) > 0 {
		buf.WriteString(`,"ttl":"`)
		buf.WriteString(ttl)
		buf.WriteString(`"`)
	}
	if date != nil {
		buf.WriteString(`,"_timestamp":"`)
		buf.WriteString(strconv.FormatInt(date.UnixNano()/1e6, 10))
		buf.WriteString(`"`)
	}

	buf.WriteString(`}}`)
	buf.WriteRune('\n')
	//buf.WriteByte('\n')
	switch v := data.(type) {
	case *bytes.Buffer:
		io.Copy(&buf, v)
	case []byte:
		buf.Write(v)
	case string:
		buf.WriteString(v)
	default:
		body, jsonErr := json.Marshal(data)
		if jsonErr != nil {
			return nil, jsonErr
		}
		buf.Write(body)
	}
	buf.WriteRune('\n')
	return buf.Bytes(), nil
}
<commit_msg>address race condition waiting for goroutines to complete.<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"strconv"
	"sync"
	"time"
)

const (
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer = 16384
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs = 100
	// Max delay before forcing a flush to Elasticearch
	BulkDelaySeconds = 5
	// maximum wait shutdown seconds
	MAX_SHUTDOWN_SECS = 5
)

type ErrorBuffer struct {
	Err error
	Buf *bytes.Buffer
}

// A bulk indexer creates goroutines, and channels for connecting and sending data
// to elasticsearch in bulk, using buffers.
type BulkIndexer struct {
	conn *Conn

	// We are creating a variable defining the func responsible for sending
	// to allow a mock sendor for test purposes
	Sender func(*bytes.Buffer) error

	// The refresh parameter can be set to true in order to refresh the
	// relevant primary and replica shards immediately after the bulk
	// operation has occurred
	Refresh bool

	// If we encounter an error in sending, we are going to retry for this long
	// before returning an error
	// if 0 it will not retry
	RetryForSeconds int

	// channel for getting errors
	ErrorChannel chan *ErrorBuffer

	// channel for sending to background indexer
	bulkChannel chan []byte

	// numErrors is a running total of errors seen
	numErrors uint64

	// shutdown channel
	shutdownChan chan chan struct{}
	// channel to shutdown timer
	timerDoneChan chan struct{}

	// Channel to send a complete byte.Buffer to the http sendor
	sendBuf chan *bytes.Buffer
	// byte buffer for docs that have been converted to bytes, but not yet sent
	buf *bytes.Buffer
	// Buffer for Max number of time before forcing flush
	BufferDelayMax time.Duration
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer int // 1048576
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs int // 100

	// Number of documents we have send through so far on this session
	docCt int
	// Max number of http conns in flight at one time
	maxConns int
	// If we are indexing enough docs per bufferdelaymax, we won't need to do time
	// based eviction, else we do.
	needsTimeBasedFlush bool
	// Lock for document writes/operations
	mu sync.Mutex
	// Wait Group for the http sends
	sendWg *sync.WaitGroup
}

func (b *BulkIndexer) NumErrors() uint64 {
	return b.numErrors
}

func (c *Conn) NewBulkIndexer(maxConns int) *BulkIndexer {
	b := BulkIndexer{conn: c, sendBuf: make(chan *bytes.Buffer, maxConns)}
	b.needsTimeBasedFlush = true
	b.buf = new(bytes.Buffer)
	b.maxConns = maxConns
	b.BulkMaxBuffer = BulkMaxBuffer
	b.BulkMaxDocs = BulkMaxDocs
	b.BufferDelayMax = time.Duration(BulkDelaySeconds) * time.Second
	b.bulkChannel = make(chan []byte, 100)
	b.sendWg = new(sync.WaitGroup)
	b.timerDoneChan = make(chan struct{})
	return &b
}

// A bulk indexer with more control over error handling
//    @maxConns is the max number of in flight http requests
//    @retrySeconds is # of seconds to wait before retrying falied requests
//
//   done := make(chan bool)
//   BulkIndexerGlobalRun(100, done)
func (c *Conn) NewBulkIndexerErrors(maxConns, retrySeconds int) *BulkIndexer {
	b := c.NewBulkIndexer(maxConns)
	b.RetryForSeconds = retrySeconds
	b.ErrorChannel = make(chan *ErrorBuffer, 20)
	return b
}

// Starts this bulk Indexer running, this Run opens a go routine so is
// Non blocking
func (b *BulkIndexer) Start() {
	b.shutdownChan = make(chan chan struct{})

	go func() {
		// XXX(j): Refactor this stuff to use an interface.
		if b.Sender == nil {
			b.Sender = b.Send
		}
		// Backwards compatibility
		b.startHttpSender()
		b.startDocChannel()
		b.startTimer()
		ch := <-b.shutdownChan
		time.Sleep(2 * time.Millisecond)
		b.Flush()
		b.shutdown()
		ch <- struct{}{}
	}()
}

// Stop stops the bulk indexer, blocking the caller until it is complete.
func (b *BulkIndexer) Stop() {
	ch := make(chan struct{})
	b.shutdownChan <- ch
	select {
	case <-ch:
		// done
	case <-time.After(time.Second * time.Duration(MAX_SHUTDOWN_SECS)):
		// timeout!
	}
}

func (b *BulkIndexer) PendingDocuments() int {
	return b.docCt
}

// Flush all current documents to ElasticSearch
func (b *BulkIndexer) Flush() {
	b.mu.Lock()
	if b.docCt > 0 {
		b.send(b.buf)
	}
	b.mu.Unlock()
}

func (b *BulkIndexer) startHttpSender() {

	// this sends http requests to elasticsearch it uses maxConns to open up that
	// many goroutines, each of which will synchronously call ElasticSearch
	// in theory, the whole set will cause a backup all the way to IndexBulk if
	// we have consumed all maxConns
	for i := 0; i < b.maxConns; i++ {
		go func() {
			for buf := range b.sendBuf {
				b.sendWg.Add(1)
				// Copy for the potential re-send.
				bufCopy := bytes.NewBuffer(buf.Bytes())
				err := b.Sender(buf)

				// Perhaps a b.FailureStrategy(err)  ??  with different types of strategies
				//  1.  Retry, then panic
				//  2.  Retry then return error and let runner decide
				//  3.  Retry, then log to disk?   retry later?
				if err != nil {
					if b.RetryForSeconds > 0 {
						time.Sleep(time.Second * time.Duration(b.RetryForSeconds))
						err = b.Sender(bufCopy)
						if err == nil {
							// Successfully re-sent with no error
							b.sendWg.Done()
							continue
						}
					}
					if b.ErrorChannel != nil {
						b.ErrorChannel <- &ErrorBuffer{err, buf}
					}
				}
				b.sendWg.Done()
			}
		}()
	}
}

// start a timer for checking back and forcing flush ever BulkDelaySeconds seconds
// even if we haven't hit max messages/size
func (b *BulkIndexer) startTimer() {
	ticker := time.NewTicker(b.BufferDelayMax)
	go func() {
		for {
			select {
			case <-ticker.C:
				b.mu.Lock()
				// don't send unless last sendor was the time,
				// otherwise an indication of other thresholds being hit
				// where time isn't needed
				if b.buf.Len() > 0 && b.needsTimeBasedFlush {
					b.needsTimeBasedFlush = true
					b.send(b.buf)
				} else if b.buf.Len() > 0 {
					b.needsTimeBasedFlush = true
				}
				b.mu.Unlock()
			case <-b.timerDoneChan:
				// shutdown this go routine
				return
			}

		}
	}()
}

func (b *BulkIndexer) startDocChannel() {
	// This goroutine accepts incoming byte arrays from the IndexBulk function and
	// writes to buffer
	go func() {
		for docBytes := range b.bulkChannel {
			b.mu.Lock()
			b.docCt += 1
			b.buf.Write(docBytes)
			if b.buf.Len() >= b.BulkMaxBuffer || b.docCt >= b.BulkMaxDocs {
				b.needsTimeBasedFlush = false
				//log.Printf("Send due to size:  docs=%d  bufsize=%d", b.docCt, b.buf.Len())
				b.send(b.buf)
			}
			b.mu.Unlock()
		}
	}()
}

func (b *BulkIndexer) send(buf *bytes.Buffer) {
	//b2 := *b.buf
	b.sendBuf <- buf
	b.buf = new(bytes.Buffer)
	//	b.buf.Reset()
	b.docCt = 0
}

func (b *BulkIndexer) shutdown() {
	// This must be called after Flush()
	close(b.timerDoneChan)
	close(b.sendBuf)
	close(b.bulkChannel)
	b.sendWg.Wait()
}

// The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
// it operates by buffering requests, and ocassionally flushing to elasticsearch
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Delete(index, _type, id string) {
	queryLine := fmt.Sprintf("{\"delete\":{\"_index\":%q,\"_type\":%q,\"_id\":%q}}\n", index, _type, id)
	b.bulkChannel <- []byte(queryLine)
	return
}

func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, parent, ttl string, date *time.Time, script string) error {

	var data map[string]interface{} = make(map[string]interface{})
	data["script"] = script
	return b.Update(index, _type, id, parent, ttl, date, data)
}

func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool) error {

	var data map[string]interface{} = make(map[string]interface{})

	data["doc"] = partialDoc
	if upsert {
		data["doc_as_upsert"] = true
	}
	return b.Update(index, _type, id, parent, ttl, date, data)
}

// This does the actual send of a buffer, which has already been formatted
// into bytes of ES formatted bulk data
func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
	type responseStruct struct {
		Took   int64                    `json:"took"`
		Errors bool                     `json:"errors"`
		Items  []map[string]interface{} `json:"items"`
	}

	response := responseStruct{}

	body, err := b.conn.DoCommand("POST", fmt.Sprintf("/_bulk?refresh=%t", b.Refresh), nil, buf)

	if err != nil {
		b.numErrors += 1
		return err
	}
	// check for response errors, bulk insert will give 200 OK but then include errors in response
	jsonErr := json.Unmarshal(body, &response)
	if jsonErr == nil {
		if response.Errors {
			b.numErrors += uint64(len(response.Items))
			return fmt.Errorf("Bulk Insertion Error. Failed item count [%d]", len(response.Items))
		}
	}
	return nil
}

// Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) ([]byte, error) {
	// only index and update are currently supported
	if op != "index" && op != "update" {
		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
	}

	// First line
	buf := bytes.Buffer{}
	buf.WriteString(fmt.Sprintf(`{"%s":{"_index":"`, op))
	buf.WriteString(index)
	buf.WriteString(`","_type":"`)
	buf.WriteString(_type)
	buf.WriteString(`"`)
	if len(id) > 0 {
		buf.WriteString(`,"_id":"`)
		buf.WriteString(id)
		buf.WriteString(`"`)
	}

	if len(parent) > 0 {
		buf.WriteString(`,"_parent":"`)
		buf.WriteString(parent)
		buf.WriteString(`"`)
	}

	if op == "update" {
		buf.WriteString(`,"_retry_on_conflict":3`)
	}

	if len(ttl) > 0 {
		buf.WriteString(`,"ttl":"`)
		buf.WriteString(ttl)
		buf.WriteString(`"`)
	}
	if date != nil {
		buf.WriteString(`,"_timestamp":"`)
		buf.WriteString(strconv.FormatInt(date.UnixNano()/1e6, 10))
		buf.WriteString(`"`)
	}

	buf.WriteString(`}}`)
	buf.WriteRune('\n')
	//buf.WriteByte('\n')
	switch v := data.(type) {
	case *bytes.Buffer:
		io.Copy(&buf, v)
	case []byte:
		buf.Write(v)
	case string:
		buf.WriteString(v)
	default:
		body, jsonErr := json.Marshal(data)
		if jsonErr != nil {
			return nil, jsonErr
		}
		buf.Write(body)
	}
	buf.WriteRune('\n')
	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/pfs"

	"github.com/julienschmidt/httprouter"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

// HTTPPort specifies the port the server will listen on
const HTTPPort = 652
const apiVersion = "v1"

type flushWriter struct {
	f http.Flusher
	w io.Writer
}

func (fw *flushWriter) Write(p []byte) (n int, err error) {
	n, err = fw.w.Write(p)
	if fw.f != nil {
		fw.f.Flush()
	}
	return
}

// HTTPServer serves GetFile requests over HTTP
// e.g. http://localhost:30652/v1/pfs/repos/foo/commits/b7a1923be56744f6a3f1525ec222dc3b/files/ttt.log
type HTTPServer struct {
	driver *driver
	*httprouter.Router
	loginPath string
}

func newHTTPServer(address string, etcdAddresses []string, etcdPrefix string, cacheSize int64) (*HTTPServer, error) {
	d, err := newDriver(address, etcdAddresses, etcdPrefix, cacheSize)
	if err != nil {
		return nil, err
	}
	router := httprouter.New()
	s := &HTTPServer{
		d,
		router,
		fmt.Sprintf("/%v/auth/login", apiVersion),
	}

	router.GET(fmt.Sprintf("/%v/pfs/repos/:repoName/commits/:commitID/files/*filePath", apiVersion), s.getFileHandler)
	router.POST(s.loginPath, s.authLoginHandler)
	router.POST(fmt.Sprintf("/%v/auth/logout", apiVersion), s.authLogoutHandler)
	// Debug method (to check login cookies):
	router.GET(s.loginPath, s.loginForm)
	router.NotFound = &notFoundRouter{}
	return s, nil
}

func (s *HTTPServer) getFileHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	pfsFile := &pfs.File{
		Commit: &pfs.Commit{
			ID: ps.ByName("commitID"),
			Repo: &pfs.Repo{
				Name: ps.ByName("repoName"),
			},
		},
		Path: ps.ByName("filePath"),
	}
	filePaths := strings.Split(ps.ByName("filePath"), "/")
	fileName := filePaths[len(filePaths)-1]
	ctx := context.Background()
	for _, cookie := range r.Cookies() {
		if cookie.Name == auth.ContextTokenKey {
			ctx = metadata.NewIncomingContext(
				ctx,
				metadata.Pairs(auth.ContextTokenKey, cookie.Value),
			)
		}
	}
	// Since we can't seek, open a separate reader to sniff mimetype
	mimeReader, err := s.driver.getFile(ctx, pfsFile, 0, 0)
	if err != nil {
		panic(err)
	}
	buffer := make([]byte, 512)
	_, err = mimeReader.Read(buffer)
	if err != nil && err != io.EOF {
		panic(err)
	}
	contentType := http.DetectContentType(buffer)

	file, err := s.driver.getFile(ctx, pfsFile, 0, 0)
	if err != nil {
		panic(err)
	}
	w.Header().Add("Content-Type", contentType)
	downloadValues := r.URL.Query()["download"]
	if len(downloadValues) == 1 && downloadValues[0] == "true" {
		w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=\"%v\"", fileName))
	}
	fw := flushWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}
	io.Copy(&fw, file)
}

type loginRequestPayload struct {
	Token string
}

func (s *HTTPServer) authLoginHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	token := r.FormValue("Token")
	if token == "" {
		// Return 500
		http.Error(w, "empty token provided", http.StatusInternalServerError)
		return
	}
	w.Header().Add("Set-Cookie", fmt.Sprintf("%v=%v;path=/", auth.ContextTokenKey,
		token))
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.WriteHeader(http.StatusOK)
}

func (s *HTTPServer) authLogoutHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	w.Header().Add("Set-Cookie", fmt.Sprintf("%v=;path=/", auth.ContextTokenKey))
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.WriteHeader(http.StatusOK)
}

type notFoundRouter struct {
}

func (s notFoundRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(http.StatusNotFound)
	b := strings.NewReader("route not found")
	fw := flushWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}
	io.Copy(&fw, b)
}
func (s *HTTPServer) loginForm(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	content := fmt.Sprintf(`
<!DOCTYPE html>
<html>
<body>
<form action="%v" method="post">
<input name="Token">
<input type="submit">
</form>
</body>
</html>
	`, s.loginPath)
	b := strings.NewReader(content)
	fw := flushWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}
	io.Copy(&fw, b)
}
<commit_msg>We dont need buffered io for the other routes<commit_after>package server

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/pfs"

	"github.com/julienschmidt/httprouter"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

// HTTPPort specifies the port the server will listen on
const HTTPPort = 652
const apiVersion = "v1"

type flushWriter struct {
	f http.Flusher
	w io.Writer
}

func (fw *flushWriter) Write(p []byte) (n int, err error) {
	n, err = fw.w.Write(p)
	if fw.f != nil {
		fw.f.Flush()
	}
	return
}

// HTTPServer serves GetFile requests over HTTP
// e.g. http://localhost:30652/v1/pfs/repos/foo/commits/b7a1923be56744f6a3f1525ec222dc3b/files/ttt.log
type HTTPServer struct {
	driver *driver
	*httprouter.Router
	loginPath string
}

func newHTTPServer(address string, etcdAddresses []string, etcdPrefix string, cacheSize int64) (*HTTPServer, error) {
	d, err := newDriver(address, etcdAddresses, etcdPrefix, cacheSize)
	if err != nil {
		return nil, err
	}
	router := httprouter.New()
	s := &HTTPServer{
		d,
		router,
		fmt.Sprintf("/%v/auth/login", apiVersion),
	}

	router.GET(fmt.Sprintf("/%v/pfs/repos/:repoName/commits/:commitID/files/*filePath", apiVersion), s.getFileHandler)
	router.POST(s.loginPath, s.authLoginHandler)
	router.POST(fmt.Sprintf("/%v/auth/logout", apiVersion), s.authLogoutHandler)
	// Debug method (to check login cookies):
	router.GET(s.loginPath, s.loginForm)
	router.NotFound = &notFoundRouter{}
	return s, nil
}

func (s *HTTPServer) getFileHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	pfsFile := &pfs.File{
		Commit: &pfs.Commit{
			ID: ps.ByName("commitID"),
			Repo: &pfs.Repo{
				Name: ps.ByName("repoName"),
			},
		},
		Path: ps.ByName("filePath"),
	}
	filePaths := strings.Split(ps.ByName("filePath"), "/")
	fileName := filePaths[len(filePaths)-1]
	ctx := context.Background()
	for _, cookie := range r.Cookies() {
		if cookie.Name == auth.ContextTokenKey {
			ctx = metadata.NewIncomingContext(
				ctx,
				metadata.Pairs(auth.ContextTokenKey, cookie.Value),
			)
		}
	}
	// Since we can't seek, open a separate reader to sniff mimetype
	mimeReader, err := s.driver.getFile(ctx, pfsFile, 0, 0)
	if err != nil {
		panic(err)
	}
	buffer := make([]byte, 512)
	_, err = mimeReader.Read(buffer)
	if err != nil && err != io.EOF {
		panic(err)
	}
	contentType := http.DetectContentType(buffer)

	file, err := s.driver.getFile(ctx, pfsFile, 0, 0)
	if err != nil {
		panic(err)
	}
	w.Header().Add("Content-Type", contentType)
	downloadValues := r.URL.Query()["download"]
	if len(downloadValues) == 1 && downloadValues[0] == "true" {
		w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=\"%v\"", fileName))
	}
	fw := flushWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}
	io.Copy(&fw, file)
}

type loginRequestPayload struct {
	Token string
}

func (s *HTTPServer) authLoginHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	token := r.FormValue("Token")
	if token == "" {
		// Return 500
		http.Error(w, "empty token provided", http.StatusInternalServerError)
		return
	}
	w.Header().Add("Set-Cookie", fmt.Sprintf("%v=%v;path=/", auth.ContextTokenKey,
		token))
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.WriteHeader(http.StatusOK)
}

func (s *HTTPServer) authLogoutHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	w.Header().Add("Set-Cookie", fmt.Sprintf("%v=;path=/", auth.ContextTokenKey))
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.WriteHeader(http.StatusOK)
}

type notFoundRouter struct {
}

func (s notFoundRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(http.StatusNotFound)
	io.Copy(&w, strings.NewReader("route not found"))
}
func (s *HTTPServer) loginForm(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	content := fmt.Sprintf(`
<!DOCTYPE html>
<html>
<body>
<form action="%v" method="post">
<input name="Token">
<input type="submit">
</form>
</body>
</html>
	`, s.loginPath)
	io.Copy(&w, strings.NewReader(content))
}
<|endoftext|>


<commit_before>package bot

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"

	discord "github.com/bwmarrin/discordgo"
	log "github.com/sirupsen/logrus"

	"github.com/GinjaNinja32/DisGoIRC/format"
)

const maxTries = 5

func retryErrors(desc string, f func() error) {
	attempt := 1
	for {
		err := f()
		if err == nil {
			return
		}
		if attempt >= maxTries {
			log.Fatalf("Failed to %s [final attempt]: %s", desc, err)
		}
		log.Errorf("Failed to %s [attempt %d/%d]: %s", desc, attempt, maxTries, err)
		time.Sleep(time.Second)
		attempt++
	}
}

// DiscordConfig represents the required config to connect to Discord
type DiscordConfig struct {
	Token        string `json:"token"`
	UseNicknames bool   `json:"use_nicknames"`
}

var (
	dBotID      string
	dSession    *discord.Session
	dGuilds     = map[string]string{}
	dGuildChans = map[string]map[string]string{}

	dMsgQueue = make(chan func())
)

func dInit() {
	retryErrors("initialise Discord session", func() (err error) {
		dSession, err = discord.New(fmt.Sprintf("Bot %s", conf.Discord.Token))
		return
	})

	retryErrors("get own Discord user", func() (err error) {
		u, err := dSession.User("@me")
		if err == nil {
			dBotID = u.ID
		}
		return
	})

	var guilds []*discord.UserGuild
	retryErrors("get guilds", func() (err error) {
		guilds, err = dSession.UserGuilds(99, "", "")
		return
	})

	for _, g := range guilds {
		var chans []*discord.Channel
		retryErrors(fmt.Sprintf("get channels for %s", g.Name), func() (err error) {
			chans, err = dSession.GuildChannels(g.ID)
			return
		})

		dGuilds[g.Name] = g.ID
		dGuildChans[g.Name] = map[string]string{}
		for _, c := range chans {
			if c.Type == discord.ChannelTypeGuildText {
				dGuildChans[g.Name][c.Name] = c.ID
			}
		}
	}

	dSession.AddHandler(dMessageCreate)

	retryErrors("connect to Discord", dSession.Open)

	go func() {
		for f := range dMsgQueue {
			f()
		}
	}()

	log.Infof("Connected to Discord")
}

func dMessageCreate(s *discord.Session, m *discord.MessageCreate) {
	if m.Author.ID == dBotID {
		return
	}

	c, err := s.Channel(m.ChannelID)
	if err != nil {
		log.Errorf("Failed to get channel for incoming message with CID %s: %s", m.ChannelID, err)
		return
	}

	guildID := c.GuildID

	g, err := s.Guild(guildID)
	if err != nil {
		log.Errorf("Failed to get guild with ID %s: %s", guildID, err)
		return
	}

	channel := fmt.Sprintf("%s#%s", g.Name, c.Name)
	authorName := getDisplayNameForUser(m.Author, g.Members)

	if m.Content != "" {
		message := convertMentionsForIRC(g, m)

		dispatchMessageToIRC(authorName, channel, message)
	}
	for _, a := range m.Attachments {
		incomingDiscord(authorName, channel, a.ProxyURL)
	}
}
func convertMentionsForIRC(g *discord.Guild, m *discord.MessageCreate) string {
	message := m.Content

	// Channels
	for _, c := range g.Channels {
		if c.Type != discord.ChannelTypeGuildText {
			continue
		}
		find := fmt.Sprintf("<#%s>", c.ID)
		replace := fmt.Sprintf("#%s", c.Name)
		message = strings.Replace(message, find, replace, -1)
	}

	// Users
	for _, u := range g.Members {
		display := getDisplayNameForMember(u)
		if display == "" {
			log.Errorf("%s/%q/%q had an invalid display name", u.User.ID, u.User.Username, u.Nick)
			continue
		}
		find := fmt.Sprintf("<@%s>", u.User.ID)
		find2 := fmt.Sprintf("<@!%s>", u.User.ID)
		replace := fmt.Sprintf("@%s", display)
		message = strings.Replace(message, find, replace, -1)
		message = strings.Replace(message, find2, replace, -1)
	}

	// Roles
	for _, r := range g.Roles {
		find := fmt.Sprintf("<@&%s>", r.ID)
		replace := fmt.Sprintf("@%s", r.Name)
		message = strings.Replace(message, find, replace, -1)
	}

	// Emojis
	for _, e := range g.Emojis {
		find := fmt.Sprintf("<:%s:%s>", e.Name, e.ID)
		replace := fmt.Sprintf(":%s:", e.Name)
		message = strings.Replace(message, find, replace, -1)
	}

	return message
}

func dispatchMessageToIRC(authorName, channel, message string) {
	// Multiline
	lines := strings.Split(message, "\n")
	lines, forceClip := clipLinesForIRC(lines)
	if len(lines) > 3 || forceClip {
		url := uploadToPtpb(message)

		n := 2
		if len(lines) < 2 {
			n = len(lines)
		}

		for _, line := range lines[:n] {
			incomingDiscord(authorName, channel, line)
		}
		incomingDiscord("[SYSTEM]", channel, fmt.Sprintf("full message from %s: %s", iAddAntiPing(authorName), url))
	} else {
		for _, line := range lines {
			incomingDiscord(authorName, channel, line)
		}
	}
}

func clipLinesForIRC(s []string) ([]string, bool) {
	ret := []string{}
	anyLineForceClip := false

	for _, line := range s {
		if len(line) < 300 {
			ret = append(ret, line)
		} else {
			words := strings.Split(line, " ")
			for len(words) != 0 {
				l := words[0]
				words = words[1:]
				for len(words) != 0 && len(l)+len(words[0]) < 300 {
					l = l + " " + words[0]
					words = words[1:]
				}

				anyLineForceClip = anyLineForceClip || len(l) > 300
				ret = append(ret, l)
			}
		}
	}

	return ret, anyLineForceClip
}

func uploadToPtpb(s string) string {
	resp, err := http.PostForm("https://ptpb.pw/",
		url.Values{"c": {s}, "p": {"1"}})

	if err != nil {
		log.Errorf("Failed to upload to PTPB: %s", err)
		return "Failed to upload to PTPB"
	}
	defer func() {
		resp.Body.Close() // nolint: gosec, errcheck
	}()
	if resp.StatusCode == http.StatusOK {
		return resp.Header.Get("Location")
	}

	log.Errorf("Failed to upload to PTPB: HTTP %d", resp.StatusCode)
	ret, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("Failed to read body: %s", err)
	} else {
		log.Errorf(string(ret))
	}
	return fmt.Sprintf("Failed to upload to PTPB: HTTP %d", resp.StatusCode)

}

var discordEscaper = strings.NewReplacer(
	"\\", "\\\\",
	"*", "\\*",
	"_", "\\_",
)

// StringReplace represents a single string replacement
type StringReplace struct {
	Find    string
	Replace string
}

// StringReplaceGroup represents a group of string replacements to be performed longest-match-first
type StringReplaceGroup []StringReplace

// Add adds a find/replace pair to this group
func (s *StringReplaceGroup) Add(find, replace string) {
	*s = append(*s, StringReplace{find, replace})
}

// Replace performs the replacement represented by this group on the string `str`, returning the result.
// Replacements are performed in length order, longest first.
func (s *StringReplaceGroup) Replace(str string) string {
	sort.Sort(s)
	for _, r := range *s {
		str = strings.Replace(str, r.Find, r.Replace, -1)
	}
	return str
}

func (s StringReplaceGroup) Len() int      { return len(s) }
func (s StringReplaceGroup) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s StringReplaceGroup) Less(i, j int) bool {
	si := s[i].Find
	sj := s[j].Find

	if len(si) != len(sj) {
		return len(si) > len(sj)
	}

	return si < sj
}

func dOutgoing(nick, channel string, messageParsed format.FormattedString) {
	chanParts := strings.Split(channel, "#")
	guildID := dGuilds[chanParts[0]]
	chanID := dGuildChans[chanParts[0]][chanParts[1]]

	g, err := dSession.Guild(guildID)
	if err != nil {
		log.Errorf("Failed to get guild with ID %s: %s", guildID, err)
		return
	}

	message := messageParsed.RenderDiscord()

	// Channels
	for _, c := range g.Channels {
		if c.Type != discord.ChannelTypeGuildText {
			continue
		}
		find := discordEscaper.Replace(fmt.Sprintf("#%s", c.Name))
		replace := fmt.Sprintf("<#\xff%s>", c.ID) // \xff to avoid replacing #channel with <#numbers> then #numbers matching another channel
		message = strings.Replace(message, find, replace, -1)
	}

	// Users
	var sr StringReplaceGroup
	for _, u := range g.Members {
		display := getDisplayNameForMember(u)
		if display == "" {
			log.Errorf("%s/%q/%q had an invalid display name", u.User.ID, u.User.Username, u.Nick)
			continue
		}
		find := discordEscaper.Replace(fmt.Sprintf("@%s", display))
		replace := fmt.Sprintf("<@\xff%s>", u.User.ID) // \xff to avoid replacing @user with <@numbers> then @numbers matching another user
		sr.Add(find, replace)
	}
	message = sr.Replace(message)

	// Roles
	for _, r := range g.Roles {
		find := discordEscaper.Replace(fmt.Sprintf("@%s", r.Name))
		replace := fmt.Sprintf("<@&%s>", r.ID)
		message = strings.Replace(message, find, replace, -1)
	}

	message = strings.Replace(message, "\xff", "", -1) // remove the \xff we added, we don't need it any more

	// Emojis
	for _, e := range g.Emojis {
		find := fmt.Sprintf(":%s:", e.Name)
		replace := fmt.Sprintf("<:%s:%s>", e.Name, e.ID)
		message = strings.Replace(message, find, replace, -1)
	}

	dMsgQueue <- func() {
		_, err := dSession.ChannelMessageSend(chanID, fmt.Sprintf("**<%s>** %s", nick, message))
		if err != nil {
			log.Errorf("Failed to send message to %s: <%s> %s", chanID, nick, message)
		}
	}
}

func getDisplayNameForMember(member *discord.Member) string {
	if conf.Discord.UseNicknames && member.Nick != "" {
		return member.Nick
	}

	return member.User.Username
}

func getDisplayNameForUser(user *discord.User, members []*discord.Member) string {
	if conf.Discord.UseNicknames {
		for _, m := range members {
			if m.User.ID == user.ID {
				return getDisplayNameForMember(m)
			}
		}
	}

	return user.Username
}
<commit_msg>Enforce word-boundary at the end of ping replacements<commit_after>package bot

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"

	discord "github.com/bwmarrin/discordgo"
	log "github.com/sirupsen/logrus"

	"github.com/GinjaNinja32/DisGoIRC/format"
)

const maxTries = 5

func retryErrors(desc string, f func() error) {
	attempt := 1
	for {
		err := f()
		if err == nil {
			return
		}
		if attempt >= maxTries {
			log.Fatalf("Failed to %s [final attempt]: %s", desc, err)
		}
		log.Errorf("Failed to %s [attempt %d/%d]: %s", desc, attempt, maxTries, err)
		time.Sleep(time.Second)
		attempt++
	}
}

// DiscordConfig represents the required config to connect to Discord
type DiscordConfig struct {
	Token        string `json:"token"`
	UseNicknames bool   `json:"use_nicknames"`
}

var (
	dBotID      string
	dSession    *discord.Session
	dGuilds     = map[string]string{}
	dGuildChans = map[string]map[string]string{}

	dMsgQueue = make(chan func())
)

func dInit() {
	retryErrors("initialise Discord session", func() (err error) {
		dSession, err = discord.New(fmt.Sprintf("Bot %s", conf.Discord.Token))
		return
	})

	retryErrors("get own Discord user", func() (err error) {
		u, err := dSession.User("@me")
		if err == nil {
			dBotID = u.ID
		}
		return
	})

	var guilds []*discord.UserGuild
	retryErrors("get guilds", func() (err error) {
		guilds, err = dSession.UserGuilds(99, "", "")
		return
	})

	for _, g := range guilds {
		var chans []*discord.Channel
		retryErrors(fmt.Sprintf("get channels for %s", g.Name), func() (err error) {
			chans, err = dSession.GuildChannels(g.ID)
			return
		})

		dGuilds[g.Name] = g.ID
		dGuildChans[g.Name] = map[string]string{}
		for _, c := range chans {
			if c.Type == discord.ChannelTypeGuildText {
				dGuildChans[g.Name][c.Name] = c.ID
			}
		}
	}

	dSession.AddHandler(dMessageCreate)

	retryErrors("connect to Discord", dSession.Open)

	go func() {
		for f := range dMsgQueue {
			f()
		}
	}()

	log.Infof("Connected to Discord")
}

func dMessageCreate(s *discord.Session, m *discord.MessageCreate) {
	if m.Author.ID == dBotID {
		return
	}

	c, err := s.Channel(m.ChannelID)
	if err != nil {
		log.Errorf("Failed to get channel for incoming message with CID %s: %s", m.ChannelID, err)
		return
	}

	guildID := c.GuildID

	g, err := s.Guild(guildID)
	if err != nil {
		log.Errorf("Failed to get guild with ID %s: %s", guildID, err)
		return
	}

	channel := fmt.Sprintf("%s#%s", g.Name, c.Name)
	authorName := getDisplayNameForUser(m.Author, g.Members)

	if m.Content != "" {
		message := convertMentionsForIRC(g, m)

		dispatchMessageToIRC(authorName, channel, message)
	}
	for _, a := range m.Attachments {
		incomingDiscord(authorName, channel, a.ProxyURL)
	}
}
func convertMentionsForIRC(g *discord.Guild, m *discord.MessageCreate) string {
	message := m.Content

	// Channels
	for _, c := range g.Channels {
		if c.Type != discord.ChannelTypeGuildText {
			continue
		}
		find := fmt.Sprintf("<#%s>", c.ID)
		replace := fmt.Sprintf("#%s", c.Name)
		message = strings.Replace(message, find, replace, -1)
	}

	// Users
	for _, u := range g.Members {
		display := getDisplayNameForMember(u)
		if display == "" {
			log.Errorf("%s/%q/%q had an invalid display name", u.User.ID, u.User.Username, u.Nick)
			continue
		}
		find := fmt.Sprintf("<@%s>", u.User.ID)
		find2 := fmt.Sprintf("<@!%s>", u.User.ID)
		replace := fmt.Sprintf("@%s", display)
		message = strings.Replace(message, find, replace, -1)
		message = strings.Replace(message, find2, replace, -1)
	}

	// Roles
	for _, r := range g.Roles {
		find := fmt.Sprintf("<@&%s>", r.ID)
		replace := fmt.Sprintf("@%s", r.Name)
		message = strings.Replace(message, find, replace, -1)
	}

	// Emojis
	for _, e := range g.Emojis {
		find := fmt.Sprintf("<:%s:%s>", e.Name, e.ID)
		replace := fmt.Sprintf(":%s:", e.Name)
		message = strings.Replace(message, find, replace, -1)
	}

	return message
}

func dispatchMessageToIRC(authorName, channel, message string) {
	// Multiline
	lines := strings.Split(message, "\n")
	lines, forceClip := clipLinesForIRC(lines)
	if len(lines) > 3 || forceClip {
		url := uploadToPtpb(message)

		n := 2
		if len(lines) < 2 {
			n = len(lines)
		}

		for _, line := range lines[:n] {
			incomingDiscord(authorName, channel, line)
		}
		incomingDiscord("[SYSTEM]", channel, fmt.Sprintf("full message from %s: %s", iAddAntiPing(authorName), url))
	} else {
		for _, line := range lines {
			incomingDiscord(authorName, channel, line)
		}
	}
}

func clipLinesForIRC(s []string) ([]string, bool) {
	ret := []string{}
	anyLineForceClip := false

	for _, line := range s {
		if len(line) < 300 {
			ret = append(ret, line)
		} else {
			words := strings.Split(line, " ")
			for len(words) != 0 {
				l := words[0]
				words = words[1:]
				for len(words) != 0 && len(l)+len(words[0]) < 300 {
					l = l + " " + words[0]
					words = words[1:]
				}

				anyLineForceClip = anyLineForceClip || len(l) > 300
				ret = append(ret, l)
			}
		}
	}

	return ret, anyLineForceClip
}

func uploadToPtpb(s string) string {
	resp, err := http.PostForm("https://ptpb.pw/",
		url.Values{"c": {s}, "p": {"1"}})

	if err != nil {
		log.Errorf("Failed to upload to PTPB: %s", err)
		return "Failed to upload to PTPB"
	}
	defer func() {
		resp.Body.Close() // nolint: gosec, errcheck
	}()
	if resp.StatusCode == http.StatusOK {
		return resp.Header.Get("Location")
	}

	log.Errorf("Failed to upload to PTPB: HTTP %d", resp.StatusCode)
	ret, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("Failed to read body: %s", err)
	} else {
		log.Errorf(string(ret))
	}
	return fmt.Sprintf("Failed to upload to PTPB: HTTP %d", resp.StatusCode)

}

var discordEscaper = strings.NewReplacer(
	"\\", "\\\\",
	"*", "\\*",
	"_", "\\_",
)

// StringReplace represents a single string replacement
type StringReplace struct {
	Find    string
	Replace string
}

// StringReplaceGroup represents a group of string replacements to be performed longest-match-first
type StringReplaceGroup []StringReplace

// Add adds a find/replace pair to this group
func (s *StringReplaceGroup) Add(find, replace string) {
	*s = append(*s, StringReplace{find, replace})
}

// Replace performs the replacement represented by this group on the string `str`, returning the result.
// Replacements are performed in length order, longest first.
func (s *StringReplaceGroup) Replace(str string) string {
	sort.Sort(s)
	for _, r := range *s {
		str = regexp.MustCompile(regexp.QuoteMeta(r.Find)+`\b`).ReplaceAllString(str, r.Replace)
	}
	return str
}

func (s StringReplaceGroup) Len() int      { return len(s) }
func (s StringReplaceGroup) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s StringReplaceGroup) Less(i, j int) bool {
	si := s[i].Find
	sj := s[j].Find

	if len(si) != len(sj) {
		return len(si) > len(sj)
	}

	return si < sj
}

func dOutgoing(nick, channel string, messageParsed format.FormattedString) {
	chanParts := strings.Split(channel, "#")
	guildID := dGuilds[chanParts[0]]
	chanID := dGuildChans[chanParts[0]][chanParts[1]]

	g, err := dSession.Guild(guildID)
	if err != nil {
		log.Errorf("Failed to get guild with ID %s: %s", guildID, err)
		return
	}

	message := messageParsed.RenderDiscord()

	// Channels
	for _, c := range g.Channels {
		if c.Type != discord.ChannelTypeGuildText {
			continue
		}
		find := discordEscaper.Replace(fmt.Sprintf("#%s", c.Name))
		replace := fmt.Sprintf("<#\xff%s>", c.ID) // \xff to avoid replacing #channel with <#numbers> then #numbers matching another channel
		message = strings.Replace(message, find, replace, -1)
	}

	// Users
	var sr StringReplaceGroup
	for _, u := range g.Members {
		display := getDisplayNameForMember(u)
		if display == "" {
			log.Errorf("%s/%q/%q had an invalid display name", u.User.ID, u.User.Username, u.Nick)
			continue
		}
		find := discordEscaper.Replace(fmt.Sprintf("@%s", display))
		replace := fmt.Sprintf("<@\xff%s>", u.User.ID) // \xff to avoid replacing @user with <@numbers> then @numbers matching another user
		sr.Add(find, replace)
	}
	message = sr.Replace(message)

	// Roles
	for _, r := range g.Roles {
		find := discordEscaper.Replace(fmt.Sprintf("@%s", r.Name))
		replace := fmt.Sprintf("<@&%s>", r.ID)
		message = strings.Replace(message, find, replace, -1)
	}

	message = strings.Replace(message, "\xff", "", -1) // remove the \xff we added, we don't need it any more

	// Emojis
	for _, e := range g.Emojis {
		find := fmt.Sprintf(":%s:", e.Name)
		replace := fmt.Sprintf("<:%s:%s>", e.Name, e.ID)
		message = strings.Replace(message, find, replace, -1)
	}

	dMsgQueue <- func() {
		_, err := dSession.ChannelMessageSend(chanID, fmt.Sprintf("**<%s>** %s", nick, message))
		if err != nil {
			log.Errorf("Failed to send message to %s: <%s> %s", chanID, nick, message)
		}
	}
}

func getDisplayNameForMember(member *discord.Member) string {
	if conf.Discord.UseNicknames && member.Nick != "" {
		return member.Nick
	}

	return member.User.Username
}

func getDisplayNameForUser(user *discord.User, members []*discord.Member) string {
	if conf.Discord.UseNicknames {
		for _, m := range members {
			if m.User.ID == user.ID {
				return getDisplayNameForMember(m)
			}
		}
	}

	return user.Username
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
)

func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, cb CopyCallback) error {
	mediafile, err := LocalMediaPath(ptr.Oid)
	if err != nil {
		return err
	}

	stat, statErr := os.Stat(mediafile)
	if statErr == nil && stat != nil {
		fileSize := stat.Size()
		if fileSize == 0 || fileSize != ptr.Size {
			tracerx.Printf("Removing %s, size %d is invalid", mediafile, fileSize)
			os.RemoveAll(mediafile)
			stat = nil
		}
	}

	var wErr *WrappedError
	if statErr != nil || stat == nil {
		wErr = downloadFile(writer, ptr, workingfile, mediafile, cb)
	} else {
		wErr = readLocalFile(writer, ptr, mediafile, cb)
	}

	if wErr != nil {
		return &SmudgeError{ptr.Oid, mediafile, wErr}
	}

	return nil
}

func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb CopyCallback) *WrappedError {
	fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size))
	reader, size, wErr := Download(filepath.Base(mediafile))
	if reader != nil {
		defer reader.Close()
	}

	if wErr != nil {
		wErr.Errorf("Error downloading %s.", mediafile)
		return wErr
	}

	if ptr.Size == 0 {
		ptr.Size = size
	}

	mediaFile, err := contentaddressable.NewFile(mediafile)
	if err != nil {
		return Errorf(err, "Error opening media file buffer.")
	}

	_, err = CopyWithCallback(mediaFile, reader, ptr.Size, cb)
	if err == nil {
		err = mediaFile.Accept()
	}
	mediaFile.Close()

	if err != nil {
		return Errorf(err, "Error buffering media file.")
	}

	return readLocalFile(writer, ptr, mediafile, nil)
}

func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, cb CopyCallback) *WrappedError {
	reader, err := os.Open(mediafile)
	if err != nil {
		return Errorf(err, "Error opening media file.")
	}
	defer reader.Close()

	if ptr.Size == 0 {
		if stat, _ := os.Stat(mediafile); stat != nil {
			ptr.Size = stat.Size()
		}
	}

	_, err = CopyWithCallback(writer, reader, ptr.Size, cb)
	return Errorf(err, "Error reading from media file.")
}

type SmudgeError struct {
	Oid      string
	Filename string
	*WrappedError
}
<commit_msg>Fixed contentaddressable import<commit_after>package lfs

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
	contentaddressable "github.com/technoweenie/go-contentaddressable"
)

func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, cb CopyCallback) error {
	mediafile, err := LocalMediaPath(ptr.Oid)
	if err != nil {
		return err
	}

	stat, statErr := os.Stat(mediafile)
	if statErr == nil && stat != nil {
		fileSize := stat.Size()
		if fileSize == 0 || fileSize != ptr.Size {
			tracerx.Printf("Removing %s, size %d is invalid", mediafile, fileSize)
			os.RemoveAll(mediafile)
			stat = nil
		}
	}

	var wErr *WrappedError
	if statErr != nil || stat == nil {
		wErr = downloadFile(writer, ptr, workingfile, mediafile, cb)
	} else {
		wErr = readLocalFile(writer, ptr, mediafile, cb)
	}

	if wErr != nil {
		return &SmudgeError{ptr.Oid, mediafile, wErr}
	}

	return nil
}

func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb CopyCallback) *WrappedError {
	fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size))
	reader, size, wErr := Download(filepath.Base(mediafile))
	if reader != nil {
		defer reader.Close()
	}

	if wErr != nil {
		wErr.Errorf("Error downloading %s.", mediafile)
		return wErr
	}

	if ptr.Size == 0 {
		ptr.Size = size
	}

	mediaFile, err := contentaddressable.NewFile(mediafile)
	if err != nil {
		return Errorf(err, "Error opening media file buffer.")
	}

	_, err = CopyWithCallback(mediaFile, reader, ptr.Size, cb)
	if err == nil {
		err = mediaFile.Accept()
	}
	mediaFile.Close()

	if err != nil {
		return Errorf(err, "Error buffering media file.")
	}

	return readLocalFile(writer, ptr, mediafile, nil)
}

func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, cb CopyCallback) *WrappedError {
	reader, err := os.Open(mediafile)
	if err != nil {
		return Errorf(err, "Error opening media file.")
	}
	defer reader.Close()

	if ptr.Size == 0 {
		if stat, _ := os.Stat(mediafile); stat != nil {
			ptr.Size = stat.Size()
		}
	}

	_, err = CopyWithCallback(writer, reader, ptr.Size, cb)
	return Errorf(err, "Error reading from media file.")
}

type SmudgeError struct {
	Oid      string
	Filename string
	*WrappedError
}
<|endoftext|>