<commit_before>package overseer

import (
	"bytes"
	"crypto/rand"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"sync"
	"syscall"
	"time"
)

var tmpBinPath = filepath.Join(os.TempDir(), "overseer-"+token())

//a overseer master process
type master struct {
	*Config
	slaveID             int
	slaveCmd            *exec.Cmd
	slaveExtraFiles     []*os.File
	binPath, tmpBinPath string
	binPerms            os.FileMode
	binHash             []byte
	restartMux          sync.Mutex
	restarting          bool
	restartedAt         time.Time
	restarted           chan bool
	awaitingUSR1        bool
	descriptorsReleased chan bool
	signalledAt         time.Time
	printCheckUpdate    bool
}

func (mp *master) run() error {
	mp.debugf("run")
	if err := mp.checkBinary(); err != nil {
		return err
	}
	if mp.Config.Fetcher != nil {
		if err := mp.Config.Fetcher.Init(); err != nil {
			mp.warnf("fetcher init failed (%s). fetcher disabled.", err)
			mp.Config.Fetcher = nil
		}
	}
	mp.setupSignalling()
	if err := mp.retreiveFileDescriptors(); err != nil {
		return err
	}
	if mp.Config.Fetcher != nil {
		mp.printCheckUpdate = true
		mp.fetch()
		go mp.fetchLoop()
	}
	return mp.forkLoop()
}

func (mp *master) checkBinary() error {
	//get path to binary and confirm its writable
	binPath, err := os.Executable()
	if err != nil {
		return fmt.Errorf("failed to find binary path (%s)", err)
	}
	mp.binPath = binPath
	if info, err := os.Stat(binPath); err != nil {
		return fmt.Errorf("failed to stat binary (%s)", err)
	} else if info.Size() == 0 {
		return fmt.Errorf("binary file is empty")
	} else {
		//copy permissions
		mp.binPerms = info.Mode()
	}
	f, err := os.Open(binPath)
	if err != nil {
		return fmt.Errorf("cannot read binary (%s)", err)
	}
	//initial hash of file
	hash := sha1.New()
	io.Copy(hash, f)
	mp.binHash = hash.Sum(nil)
	f.Close()
	//test bin<->tmpbin moves
	if mp.Config.Fetcher != nil {
		if err := move(tmpBinPath, mp.binPath); err != nil {
			return fmt.Errorf("cannot move binary (%s)", err)
		}
		if err := move(mp.binPath, tmpBinPath); err != nil {
			return fmt.Errorf("cannot move binary back (%s)", err)
		}
	}
	return nil
}

func (mp *master) setupSignalling() {
	//updater-forker comms
	mp.restarted = make(chan bool)
	mp.descriptorsReleased = make(chan bool)
	//read all master process signals
	signals := make(chan os.Signal)
	signal.Notify(signals)
	go func() {
		for s := range signals {
			mp.handleSignal(s)
		}
	}()
}

func (mp *master) handleSignal(s os.Signal) {
	if s == mp.RestartSignal {
		//user initiated manual restart
		go mp.triggerRestart()
	} else if s.String() == "child exited" {
		// will occur on every restart, ignore it
	} else
	//**during a restart** a SIGUSR1 signals
	//to the master process that, the file
	//descriptors have been released
	if mp.awaitingUSR1 && s == SIGUSR1 {
		mp.debugf("signaled, sockets ready")
		mp.awaitingUSR1 = false
		mp.descriptorsReleased <- true
	} else
	//while the slave process is running, proxy
	//all signals through
	if mp.slaveCmd != nil && mp.slaveCmd.Process != nil {
		mp.debugf("proxy signal (%s)", s)
		mp.sendSignal(s)
	} else
	//otherwise if not running, kill on CTRL+c
	if s == os.Interrupt {
		mp.debugf("interupt with no slave")
		os.Exit(1)
	} else {
		mp.debugf("signal discarded (%s), no slave process", s)
	}
}

func (mp *master) sendSignal(s os.Signal) {
	if mp.slaveCmd != nil && mp.slaveCmd.Process != nil {
		if err := mp.slaveCmd.Process.Signal(s); err != nil {
			mp.debugf("signal failed (%s), assuming slave process died unexpectedly", err)
			os.Exit(1)
		}
	}
}

func (mp *master) retreiveFileDescriptors() error {
	mp.slaveExtraFiles = make([]*os.File, len(mp.Config.Addresses))
	for i, addr := range mp.Config.Addresses {
		a, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return fmt.Errorf("Invalid address %s (%s)", addr, err)
		}
		l, err := net.ListenTCP("tcp", a)
		if err != nil {
			return err
		}
		f, err := l.File()
		if err != nil {
			return fmt.Errorf("Failed to retreive fd for: %s (%s)", addr, err)
		}
		if err := l.Close(); err != nil {
			return fmt.Errorf("Failed to close listener for: %s (%s)", addr, err)
		}
		mp.slaveExtraFiles[i] = f
	}
	return nil
}

//fetchLoop is run in a goroutine
func (mp *master) fetchLoop() {
	min := mp.Config.MinFetchInterval
	time.Sleep(min)
	for {
		t0 := time.Now()
		mp.fetch()
		//duration fetch of fetch
		diff := time.Now().Sub(t0)
		if diff < min {
			delay := min - diff
			//ensures at least MinFetchInterval delay.
			//should be throttled by the fetcher!
			time.Sleep(delay)
		}
	}
}

func (mp *master) fetch() {
	if mp.restarting {
		return //skip if restarting
	}
	if mp.printCheckUpdate {
		mp.debugf("checking for updates...")
	}
	reader, err := mp.Fetcher.Fetch()
	if err != nil {
		mp.debugf("failed to get latest version: %s", err)
		return
	}
	if reader == nil {
		if mp.printCheckUpdate {
			mp.debugf("no updates")
		}
		mp.printCheckUpdate = false
		return //fetcher has explicitly said there are no updates
	}
	mp.printCheckUpdate = true
	mp.debugf("streaming update...")
	//optional closer
	if closer, ok := reader.(io.Closer); ok {
		defer closer.Close()
	}
	tmpBin, err := os.OpenFile(tmpBinPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		mp.warnf("failed to open temp binary: %s", err)
		return
	}
	defer func() {
		tmpBin.Close()
		os.Remove(tmpBinPath)
	}()
	//tee off to sha1
	hash := sha1.New()
	reader = io.TeeReader(reader, hash)
	//write to a temp file
	_, err = io.Copy(tmpBin, reader)
	if err != nil {
		mp.warnf("failed to write temp binary: %s", err)
		return
	}
	//compare hash
	newHash := hash.Sum(nil)
	if bytes.Equal(mp.binHash, newHash) {
		mp.debugf("hash match - skip")
		return
	}
	//copy permissions
	if err := chmod(tmpBin, mp.binPerms); err != nil {
		mp.warnf("failed to make temp binary executable: %s", err)
		return
	}
	if err := chown(tmpBin, uid, gid); err != nil {
		mp.warnf("failed to change owner of binary: %s", err)
		return
	}
	if _, err := tmpBin.Stat(); err != nil {
		mp.warnf("failed to stat temp binary: %s", err)
		return
	}
	tmpBin.Close()
	if _, err := os.Stat(tmpBinPath); err != nil {
		mp.warnf("failed to stat temp binary by path: %s", err)
		return
	}
	if mp.Config.PreUpgrade != nil {
		if err := mp.Config.PreUpgrade(tmpBinPath); err != nil {
			mp.warnf("user cancelled upgrade: %s", err)
			return
		}
	}
	//overseer sanity check, dont replace our good binary with a non-executable file
	tokenIn := token()
	cmd := exec.Command(tmpBinPath)
	cmd.Env = append(os.Environ(), []string{envBinCheck + "=" + tokenIn}...)
	cmd.Args = os.Args
	returned := false
	go func() {
		time.Sleep(5 * time.Second)
		if !returned {
			mp.warnf("sanity check against fetched executable timed-out, check overseer is running")
			if cmd.Process != nil {
				cmd.Process.Kill()
			}
		}
	}()
	tokenOut, err := cmd.CombinedOutput()
	returned = true
	if err != nil {
		mp.warnf("failed to run temp binary: %s (%s) output \"%s\"", err, tmpBinPath, tokenOut)
		return
	}
	if tokenIn != string(tokenOut) {
		mp.warnf("sanity check failed")
		return
	}
	//overwrite!
	if err := overwrite(mp.binPath, tmpBinPath); err != nil {
		mp.warnf("failed to overwrite binary: %s", err)
		return
	}
	mp.debugf("upgraded binary (%x -> %x)", mp.binHash[:12], newHash[:12])
	mp.binHash = newHash
	//binary successfully replaced
	if !mp.Config.NoRestartAfterFetch {
		mp.triggerRestart()
	}
	//and keep fetching...
	return
}

func (mp *master) triggerRestart() {
	if mp.restarting {
		mp.debugf("already graceful restarting")
		return //skip
	} else if mp.slaveCmd == nil || mp.restarting {
		mp.debugf("no slave process")
		return //skip
	}
	mp.debugf("graceful restart triggered")
	mp.restarting = true
	mp.awaitingUSR1 = true
	mp.signalledAt = time.Now()
	mp.sendSignal(mp.Config.RestartSignal) //ask nicely to terminate
	select {
	case <-mp.restarted:
		//success
		mp.debugf("restart success")
	case <-time.After(mp.TerminateTimeout):
		//times up mr. process, we did ask nicely!
		mp.debugf("graceful timeout, forcing exit")
		mp.sendSignal(os.Kill)
	}
}

//not a real fork
func (mp *master) forkLoop() error {
	//loop, restart command
	for {
		if err := mp.fork(); err != nil {
			return err
		}
	}
}

func (mp *master) fork() error {
	mp.debugf("starting %s", mp.binPath)
	cmd := exec.Command(mp.binPath)
	//mark this new process as the "active" slave process.
	//this process is assumed to be holding the socket files.
	mp.slaveCmd = cmd
	mp.slaveID++
	//provide the slave process with some state
	e := os.Environ()
	e = append(e, envBinID+"="+hex.EncodeToString(mp.binHash))
	e = append(e, envBinPath+"="+mp.binPath)
	e = append(e, envSlaveID+"="+strconv.Itoa(mp.slaveID))
	e = append(e, envIsSlave+"=1")
	e = append(e, envNumFDs+"="+strconv.Itoa(len(mp.slaveExtraFiles)))
	cmd.Env = e
	//inherit master args/stdfiles
	cmd.Args = os.Args
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	//include socket files
	cmd.ExtraFiles = mp.slaveExtraFiles
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Failed to start slave process: %s", err)
	}
	//was scheduled to restart, notify success
	if mp.restarting {
		mp.restartedAt = time.Now()
		mp.restarting = false
		mp.restarted <- true
	}
	//convert wait into channel
	cmdwait := make(chan error)
	go func() {
		cmdwait <- cmd.Wait()
	}()
	//wait....
	select {
	case err := <-cmdwait:
		//program exited before releasing descriptors
		//proxy exit code out to master
		code := 0
		if err != nil {
			code = 1
			if exiterr, ok := err.(*exec.ExitError); ok {
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					code = status.ExitStatus()
				}
			}
		}
		mp.debugf("prog exited with %d", code)
		//if a restarts are disabled or if it was an
		//unexpected crash, proxy this exit straight
		//through to the main process
		if mp.NoRestart || !mp.restarting {
			os.Exit(code)
		}
	case <-mp.descriptorsReleased:
		//if descriptors are released, the program
		//has yielded control of its sockets and
		//a parallel instance of the program can be
		//started safely. it should serve state.Listeners
		//to ensure downtime is kept at <1sec. The previous
		//cmd.Wait() will still be consumed though the
		//result will be discarded.
	}
	return nil
}

func (mp *master) debugf(f string, args ...interface{}) {
	if mp.Config.Debug {
		log.Printf("[overseer master] "+f, args...)
	}
}

func (mp *master) warnf(f string, args ...interface{}) {
	if mp.Config.Debug || !mp.Config.NoWarn {
		log.Printf("[overseer master] "+f, args...)
	}
}

func token() string {
	buff := make([]byte, 8)
	rand.Read(buff)
	return hex.EncodeToString(buff)
}
<commit_msg>fix windows update<commit_after>package overseer

import (
	"bytes"
	"crypto/rand"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"sync"
	"syscall"
	"time"
)

var tmpBinPath = filepath.Join(os.TempDir(), "overseer-"+token()+extension())

//a overseer master process
type master struct {
	*Config
	slaveID             int
	slaveCmd            *exec.Cmd
	slaveExtraFiles     []*os.File
	binPath, tmpBinPath string
	binPerms            os.FileMode
	binHash             []byte
	restartMux          sync.Mutex
	restarting          bool
	restartedAt         time.Time
	restarted           chan bool
	awaitingUSR1        bool
	descriptorsReleased chan bool
	signalledAt         time.Time
	printCheckUpdate    bool
}

func (mp *master) run() error {
	mp.debugf("run")
	if err := mp.checkBinary(); err != nil {
		return err
	}
	if mp.Config.Fetcher != nil {
		if err := mp.Config.Fetcher.Init(); err != nil {
			mp.warnf("fetcher init failed (%s). fetcher disabled.", err)
			mp.Config.Fetcher = nil
		}
	}
	mp.setupSignalling()
	if err := mp.retreiveFileDescriptors(); err != nil {
		return err
	}
	if mp.Config.Fetcher != nil {
		mp.printCheckUpdate = true
		mp.fetch()
		go mp.fetchLoop()
	}
	return mp.forkLoop()
}

func (mp *master) checkBinary() error {
	//get path to binary and confirm its writable
	binPath, err := os.Executable()
	if err != nil {
		return fmt.Errorf("failed to find binary path (%s)", err)
	}
	mp.binPath = binPath
	if info, err := os.Stat(binPath); err != nil {
		return fmt.Errorf("failed to stat binary (%s)", err)
	} else if info.Size() == 0 {
		return fmt.Errorf("binary file is empty")
	} else {
		//copy permissions
		mp.binPerms = info.Mode()
	}
	f, err := os.Open(binPath)
	if err != nil {
		return fmt.Errorf("cannot read binary (%s)", err)
	}
	//initial hash of file
	hash := sha1.New()
	io.Copy(hash, f)
	mp.binHash = hash.Sum(nil)
	f.Close()
	//test bin<->tmpbin moves
	if mp.Config.Fetcher != nil {
		if err := move(tmpBinPath, mp.binPath); err != nil {
			return fmt.Errorf("cannot move binary (%s)", err)
		}
		if err := move(mp.binPath, tmpBinPath); err != nil {
			return fmt.Errorf("cannot move binary back (%s)", err)
		}
	}
	return nil
}

func (mp *master) setupSignalling() {
	//updater-forker comms
	mp.restarted = make(chan bool)
	mp.descriptorsReleased = make(chan bool)
	//read all master process signals
	signals := make(chan os.Signal)
	signal.Notify(signals)
	go func() {
		for s := range signals {
			mp.handleSignal(s)
		}
	}()
}

func (mp *master) handleSignal(s os.Signal) {
	if s == mp.RestartSignal {
		//user initiated manual restart
		go mp.triggerRestart()
	} else if s.String() == "child exited" {
		// will occur on every restart, ignore it
	} else
	//**during a restart** a SIGUSR1 signals
	//to the master process that, the file
	//descriptors have been released
	if mp.awaitingUSR1 && s == SIGUSR1 {
		mp.debugf("signaled, sockets ready")
		mp.awaitingUSR1 = false
		mp.descriptorsReleased <- true
	} else
	//while the slave process is running, proxy
	//all signals through
	if mp.slaveCmd != nil && mp.slaveCmd.Process != nil {
		mp.debugf("proxy signal (%s)", s)
		mp.sendSignal(s)
	} else
	//otherwise if not running, kill on CTRL+c
	if s == os.Interrupt {
		mp.debugf("interupt with no slave")
		os.Exit(1)
	} else {
		mp.debugf("signal discarded (%s), no slave process", s)
	}
}

func (mp *master) sendSignal(s os.Signal) {
	if mp.slaveCmd != nil && mp.slaveCmd.Process != nil {
		if err := mp.slaveCmd.Process.Signal(s); err != nil {
			mp.debugf("signal failed (%s), assuming slave process died unexpectedly", err)
			os.Exit(1)
		}
	}
}

func (mp *master) retreiveFileDescriptors() error {
	mp.slaveExtraFiles = make([]*os.File, len(mp.Config.Addresses))
	for i, addr := range mp.Config.Addresses {
		a, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return fmt.Errorf("Invalid address %s (%s)", addr, err)
		}
		l, err := net.ListenTCP("tcp", a)
		if err != nil {
			return err
		}
		f, err := l.File()
		if err != nil {
			return fmt.Errorf("Failed to retreive fd for: %s (%s)", addr, err)
		}
		if err := l.Close(); err != nil {
			return fmt.Errorf("Failed to close listener for: %s (%s)", addr, err)
		}
		mp.slaveExtraFiles[i] = f
	}
	return nil
}

//fetchLoop is run in a goroutine
func (mp *master) fetchLoop() {
	min := mp.Config.MinFetchInterval
	time.Sleep(min)
	for {
		t0 := time.Now()
		mp.fetch()
		//duration fetch of fetch
		diff := time.Now().Sub(t0)
		if diff < min {
			delay := min - diff
			//ensures at least MinFetchInterval delay.
			//should be throttled by the fetcher!
			time.Sleep(delay)
		}
	}
}

func (mp *master) fetch() {
	if mp.restarting {
		return //skip if restarting
	}
	if mp.printCheckUpdate {
		mp.debugf("checking for updates...")
	}
	reader, err := mp.Fetcher.Fetch()
	if err != nil {
		mp.debugf("failed to get latest version: %s", err)
		return
	}
	if reader == nil {
		if mp.printCheckUpdate {
			mp.debugf("no updates")
		}
		mp.printCheckUpdate = false
		return //fetcher has explicitly said there are no updates
	}
	mp.printCheckUpdate = true
	mp.debugf("streaming update...")
	//optional closer
	if closer, ok := reader.(io.Closer); ok {
		defer closer.Close()
	}
	tmpBin, err := os.OpenFile(tmpBinPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		mp.warnf("failed to open temp binary: %s", err)
		return
	}
	defer func() {
		tmpBin.Close()
		os.Remove(tmpBinPath)
	}()
	//tee off to sha1
	hash := sha1.New()
	reader = io.TeeReader(reader, hash)
	//write to a temp file
	_, err = io.Copy(tmpBin, reader)
	if err != nil {
		mp.warnf("failed to write temp binary: %s", err)
		return
	}
	//compare hash
	newHash := hash.Sum(nil)
	if bytes.Equal(mp.binHash, newHash) {
		mp.debugf("hash match - skip")
		return
	}
	//copy permissions
	if err := chmod(tmpBin, mp.binPerms); err != nil {
		mp.warnf("failed to make temp binary executable: %s", err)
		return
	}
	if err := chown(tmpBin, uid, gid); err != nil {
		mp.warnf("failed to change owner of binary: %s", err)
		return
	}
	if _, err := tmpBin.Stat(); err != nil {
		mp.warnf("failed to stat temp binary: %s", err)
		return
	}
	tmpBin.Close()
	if _, err := os.Stat(tmpBinPath); err != nil {
		mp.warnf("failed to stat temp binary by path: %s", err)
		return
	}
	if mp.Config.PreUpgrade != nil {
		if err := mp.Config.PreUpgrade(tmpBinPath); err != nil {
			mp.warnf("user cancelled upgrade: %s", err)
			return
		}
	}
	//overseer sanity check, dont replace our good binary with a non-executable file
	tokenIn := token()
	cmd := exec.Command(tmpBinPath)
	cmd.Env = append(os.Environ(), []string{envBinCheck + "=" + tokenIn}...)
	cmd.Args = os.Args
	returned := false
	go func() {
		time.Sleep(5 * time.Second)
		if !returned {
			mp.warnf("sanity check against fetched executable timed-out, check overseer is running")
			if cmd.Process != nil {
				cmd.Process.Kill()
			}
		}
	}()
	tokenOut, err := cmd.CombinedOutput()
	returned = true
	if err != nil {
		mp.warnf("failed to run temp binary: %s (%s) output \"%s\"", err, tmpBinPath, tokenOut)
		return
	}
	if tokenIn != string(tokenOut) {
		mp.warnf("sanity check failed")
		return
	}
	//overwrite!
	if err := overwrite(mp.binPath, tmpBinPath); err != nil {
		mp.warnf("failed to overwrite binary: %s", err)
		return
	}
	mp.debugf("upgraded binary (%x -> %x)", mp.binHash[:12], newHash[:12])
	mp.binHash = newHash
	//binary successfully replaced
	if !mp.Config.NoRestartAfterFetch {
		mp.triggerRestart()
	}
	//and keep fetching...
	return
}

func (mp *master) triggerRestart() {
	if mp.restarting {
		mp.debugf("already graceful restarting")
		return //skip
	} else if mp.slaveCmd == nil || mp.restarting {
		mp.debugf("no slave process")
		return //skip
	}
	mp.debugf("graceful restart triggered")
	mp.restarting = true
	mp.awaitingUSR1 = true
	mp.signalledAt = time.Now()
	mp.sendSignal(mp.Config.RestartSignal) //ask nicely to terminate
	select {
	case <-mp.restarted:
		//success
		mp.debugf("restart success")
	case <-time.After(mp.TerminateTimeout):
		//times up mr. process, we did ask nicely!
		mp.debugf("graceful timeout, forcing exit")
		mp.sendSignal(os.Kill)
	}
}

//not a real fork
func (mp *master) forkLoop() error {
	//loop, restart command
	for {
		if err := mp.fork(); err != nil {
			return err
		}
	}
}

func (mp *master) fork() error {
	mp.debugf("starting %s", mp.binPath)
	cmd := exec.Command(mp.binPath)
	//mark this new process as the "active" slave process.
	//this process is assumed to be holding the socket files.
	mp.slaveCmd = cmd
	mp.slaveID++
	//provide the slave process with some state
	e := os.Environ()
	e = append(e, envBinID+"="+hex.EncodeToString(mp.binHash))
	e = append(e, envBinPath+"="+mp.binPath)
	e = append(e, envSlaveID+"="+strconv.Itoa(mp.slaveID))
	e = append(e, envIsSlave+"=1")
	e = append(e, envNumFDs+"="+strconv.Itoa(len(mp.slaveExtraFiles)))
	cmd.Env = e
	//inherit master args/stdfiles
	cmd.Args = os.Args
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	//include socket files
	cmd.ExtraFiles = mp.slaveExtraFiles
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Failed to start slave process: %s", err)
	}
	//was scheduled to restart, notify success
	if mp.restarting {
		mp.restartedAt = time.Now()
		mp.restarting = false
		mp.restarted <- true
	}
	//convert wait into channel
	cmdwait := make(chan error)
	go func() {
		cmdwait <- cmd.Wait()
	}()
	//wait....
	select {
	case err := <-cmdwait:
		//program exited before releasing descriptors
		//proxy exit code out to master
		code := 0
		if err != nil {
			code = 1
			if exiterr, ok := err.(*exec.ExitError); ok {
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					code = status.ExitStatus()
				}
			}
		}
		mp.debugf("prog exited with %d", code)
		//if a restarts are disabled or if it was an
		//unexpected crash, proxy this exit straight
		//through to the main process
		if mp.NoRestart || !mp.restarting {
			os.Exit(code)
		}
	case <-mp.descriptorsReleased:
		//if descriptors are released, the program
		//has yielded control of its sockets and
		//a parallel instance of the program can be
		//started safely. it should serve state.Listeners
		//to ensure downtime is kept at <1sec. The previous
		//cmd.Wait() will still be consumed though the
		//result will be discarded.
	}
	return nil
}

func (mp *master) debugf(f string, args ...interface{}) {
	if mp.Config.Debug {
		log.Printf("[overseer master] "+f, args...)
	}
}

func (mp *master) warnf(f string, args ...interface{}) {
	if mp.Config.Debug || !mp.Config.NoWarn {
		log.Printf("[overseer master] "+f, args...)
	}
}

func token() string {
	buff := make([]byte, 8)
	rand.Read(buff)
	return hex.EncodeToString(buff)
}

// On Windows, include the .exe extension, noop otherwise.
func extension() string {
	if runtime.GOOS == "windows" {
		return ".exe"
	}

	return ""
}
<|endoftext|>


<commit_before>// Copyright 2017 Diego Bernardes. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package subscription

import (
	"bytes"
	"context"
	"encoding/json"
	"net/http"

	"github.com/pkg/errors"

	"github.com/diegobernardes/flare"
	"github.com/diegobernardes/flare/infra/task"
)

// Trigger is used to process the signals on documents change.
type Trigger struct {
	document   flare.DocumentRepositorier
	repository flare.SubscriptionRepositorier
	httpClient *http.Client
	pusher     task.Pusher
}

func (t *Trigger) marshal(documentId string, action string) ([]byte, error) {
	content, err := json.Marshal(map[string]string{
		"documentId": documentId,
		"action":     action,
	})
	if err != nil {
		return nil, errors.Wrap(err, "error during message marshal")
	}
	return content, nil
}

func (t *Trigger) unmarshal(rawContent []byte) (map[string]string, error) {
	content := make(map[string]string)
	if err := json.Unmarshal(rawContent, &content); err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal message")
	}
	return content, nil
}

// Update the document change signal.
func (t *Trigger) Update(ctx context.Context, document *flare.Document) error {
	content, err := t.marshal(document.Id, flare.SubscriptionTriggerUpdate)
	if err != nil {
		return errors.Wrap(err, "error during trigger")
	}

	if err = t.pusher.Push(ctx, content); err != nil {
		return errors.Wrap(err, "error during message delivery")
	}
	return nil
}

// Delete the document change signal.
func (t *Trigger) Delete(ctx context.Context, document *flare.Document) error {
	content, err := t.marshal(document.Id, flare.SubscriptionTriggerDelete)
	if err != nil {
		return errors.Wrap(err, "error during trigger")
	}

	if err = t.pusher.Push(ctx, content); err != nil {
		return errors.Wrap(err, "error during message delivery")
	}
	return nil
}

// Process is used to consume the tasks.
func (t *Trigger) Process(ctx context.Context, rawContent []byte) error {
	content, err := t.unmarshal(rawContent)
	if err != nil {
		return errors.Wrap(err, "could not unmarshal the message")
	}

	documentId, ok := content["documentId"]
	if !ok {
		return errors.New("missing documentId")
	}

	action, ok := content["action"]
	if !ok {
		return errors.New("missing action")
	}

	document, err := t.document.FindOne(ctx, documentId)
	if err != nil {
		return errors.Wrap(err, "error during document find")
	}

	if err = t.repository.Trigger(ctx, action, document, t.exec(document)); err != nil {
		return errors.Wrap(err, "error during message process")
	}
	return nil
}

func (t *Trigger) exec(
	document *flare.Document,
) func(context.Context, flare.Subscription, string) error {
	return func(ctx context.Context, sub flare.Subscription, kind string) error {
		content, err := t.buildContent(document, sub, kind)
		if err != nil {
			return errors.Wrap(err, "error during content build")
		}

		buf := bytes.NewBuffer(content)
		req, err := http.NewRequest(sub.Endpoint.Method, sub.Endpoint.URL.String(), buf)
		if err != nil {
			return errors.Wrap(err, "error during http request create")
		}
		req = req.WithContext(ctx)

		for key, values := range sub.Endpoint.Headers {
			for _, value := range values {
				req.Header.Add(key, value)
			}
		}

		resp, err := t.httpClient.Do(req)
		if err != nil {
			return errors.Wrap(err, "error during http request")
		}

		for _, status := range sub.Delivery.Success {
			if status == resp.StatusCode {
				return nil
			}
		}

		for _, status := range sub.Delivery.Discard {
			if status == resp.StatusCode {
				return nil
			}
		}

		return errors.Errorf(
			"success and discard status don't match with the response value '%d'", resp.StatusCode,
		)
	}
}

func (t *Trigger) buildContent(
	document *flare.Document, sub flare.Subscription, kind string,
) ([]byte, error) {
	rawContent := map[string]interface{}{
		"id":               document.Id,
		"changeFieldValue": document.ChangeFieldValue,
		"updatedAt":        document.UpdatedAt.String(),
		"action":           kind,
	}
	if sub.Data != nil {
		replacer, err := sub.Resource.WildcardReplace(document.Id)
		if err != nil {
			return nil, errors.Wrap(err, "failed to extract the wildcards from document id")
		}

		for key, rawValue := range sub.Data {
			if value, ok := rawValue.(string); ok {
				sub.Data[key] = replacer(value)
			}
		}

		rawContent["data"] = sub.Data
	}

	content, err := json.Marshal(rawContent)
	if err != nil {
		return nil, errors.Wrap(err, "error during response generate")
	}
	return content, nil
}

// Init initialize the Trigger.
func (t *Trigger) Init(options ...func(*Trigger)) error {
	for _, option := range options {
		option(t)
	}

	if t.document == nil {
		return errors.New("document repository not found")
	}

	if t.pusher == nil {
		return errors.New("pusher not found")
	}

	if t.repository == nil {
		return errors.New("repository not found")
	}

	if t.httpClient == nil {
		return errors.New("httpClient not found")
	}

	return nil
}

// TriggerRepository set the repository on Trigger.
func TriggerRepository(repository flare.SubscriptionRepositorier) func(*Trigger) {
	return func(t *Trigger) {
		t.repository = repository
	}
}

// TriggerHTTPClient set the httpClient on Trigger.
func TriggerHTTPClient(httpClient *http.Client) func(*Trigger) {
	return func(t *Trigger) {
		t.httpClient = httpClient
	}
}

// TriggerPusher set the pusher that gonna receive the trigger notifications.
func TriggerPusher(pusher task.Pusher) func(*Trigger) {
	return func(t *Trigger) {
		t.pusher = pusher
	}
}

// TriggerDocumentRepository set the document repository.
func TriggerDocumentRepository(repo flare.DocumentRepositorier) func(*Trigger) {
	return func(t *Trigger) {
		t.document = repo
	}
}
<commit_msg>subscription: trigger send the document with it<commit_after>// Copyright 2017 Diego Bernardes. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package subscription

import (
	"bytes"
	"context"
	"encoding/json"
	"net/http"

	"github.com/pkg/errors"

	"github.com/diegobernardes/flare"
	"github.com/diegobernardes/flare/infra/task"
)

// Trigger is used to process the signals on documents change.
type Trigger struct {
	document   flare.DocumentRepositorier
	repository flare.SubscriptionRepositorier
	httpClient *http.Client
	pusher     task.Pusher
}

func (t *Trigger) marshal(document *flare.Document, action string) ([]byte, error) {
	type raw struct {
		Document flare.Document
		Action   string
	}

	content, err := json.Marshal(raw{
		Document: *document,
		Action:   action,
	})
	if err != nil {
		return nil, errors.Wrap(err, "error during message marshal")
	}
	return content, nil
}

func (t *Trigger) unmarshal(rawContent []byte) (map[string]interface{}, error) {
	type raw struct {
		Document flare.Document
		Action   string
	}

	var r raw
	if err := json.Unmarshal(rawContent, &r); err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal message")
	}

	return map[string]interface{}{
		"document": r.Document,
		"action":   r.Action,
	}, nil
}

// Update the document change signal.
func (t *Trigger) Update(ctx context.Context, document *flare.Document) error {
	content, err := t.marshal(document, flare.SubscriptionTriggerUpdate)
	if err != nil {
		return errors.Wrap(err, "error during trigger")
	}

	if err = t.pusher.Push(ctx, content); err != nil {
		return errors.Wrap(err, "error during message delivery")
	}
	return nil
}

// Delete the document change signal.
func (t *Trigger) Delete(ctx context.Context, document *flare.Document) error {
	content, err := t.marshal(document, flare.SubscriptionTriggerDelete)
	if err != nil {
		return errors.Wrap(err, "error during trigger")
	}

	if err = t.pusher.Push(ctx, content); err != nil {
		return errors.Wrap(err, "error during message delivery")
	}
	return nil
}

// Process is used to consume the tasks.
func (t *Trigger) Process(ctx context.Context, rawContent []byte) error {
	content, err := t.unmarshal(rawContent)
	if err != nil {
		return errors.Wrap(err, "could not unmarshal the message")
	}

	document, ok := content["document"].(flare.Document)
	if !ok {
		return errors.New("missing document")
	}

	action, ok := content["action"].(string)
	if !ok {
		return errors.New("missing action")
	}

	if err = t.repository.Trigger(ctx, action, &document, t.exec(&document)); err != nil {
		return errors.Wrap(err, "error during message process")
	}
	return nil
}

func (t *Trigger) exec(
	document *flare.Document,
) func(context.Context, flare.Subscription, string) error {
	return func(ctx context.Context, sub flare.Subscription, kind string) error {
		content, err := t.buildContent(document, sub, kind)
		if err != nil {
			return errors.Wrap(err, "error during content build")
		}

		buf := bytes.NewBuffer(content)
		req, err := http.NewRequest(sub.Endpoint.Method, sub.Endpoint.URL.String(), buf)
		if err != nil {
			return errors.Wrap(err, "error during http request create")
		}
		req = req.WithContext(ctx)

		for key, values := range sub.Endpoint.Headers {
			for _, value := range values {
				req.Header.Add(key, value)
			}
		}

		resp, err := t.httpClient.Do(req)
		if err != nil {
			return errors.Wrap(err, "error during http request")
		}

		for _, status := range sub.Delivery.Success {
			if status == resp.StatusCode {
				return nil
			}
		}

		for _, status := range sub.Delivery.Discard {
			if status == resp.StatusCode {
				return nil
			}
		}

		return errors.Errorf(
			"success and discard status don't match with the response value '%d'", resp.StatusCode,
		)
	}
}

func (t *Trigger) buildContent(
	document *flare.Document, sub flare.Subscription, kind string,
) ([]byte, error) {
	rawContent := map[string]interface{}{
		"id":               document.Id,
		"changeFieldValue": document.ChangeFieldValue,
		"updatedAt":        document.UpdatedAt.String(),
		"action":           kind,
	}
	if sub.Data != nil {
		replacer, err := sub.Resource.WildcardReplace(document.Id)
		if err != nil {
			return nil, errors.Wrap(err, "failed to extract the wildcards from document id")
		}

		for key, rawValue := range sub.Data {
			if value, ok := rawValue.(string); ok {
				sub.Data[key] = replacer(value)
			}
		}

		rawContent["data"] = sub.Data
	}

	content, err := json.Marshal(rawContent)
	if err != nil {
		return nil, errors.Wrap(err, "error during response generate")
	}
	return content, nil
}

// Init initialize the Trigger.
func (t *Trigger) Init(options ...func(*Trigger)) error {
	for _, option := range options {
		option(t)
	}

	if t.document == nil {
		return errors.New("document repository not found")
	}

	if t.pusher == nil {
		return errors.New("pusher not found")
	}

	if t.repository == nil {
		return errors.New("repository not found")
	}

	if t.httpClient == nil {
		return errors.New("httpClient not found")
	}

	return nil
}

// TriggerRepository set the repository on Trigger.
func TriggerRepository(repository flare.SubscriptionRepositorier) func(*Trigger) {
	return func(t *Trigger) {
		t.repository = repository
	}
}

// TriggerHTTPClient set the httpClient on Trigger.
func TriggerHTTPClient(httpClient *http.Client) func(*Trigger) {
	return func(t *Trigger) {
		t.httpClient = httpClient
	}
}

// TriggerPusher set the pusher that gonna receive the trigger notifications.
func TriggerPusher(pusher task.Pusher) func(*Trigger) {
	return func(t *Trigger) {
		t.pusher = pusher
	}
}

// TriggerDocumentRepository set the document repository.
func TriggerDocumentRepository(repo flare.DocumentRepositorier) func(*Trigger) {
	return func(t *Trigger) {
		t.document = repo
	}
}
<|endoftext|>


<commit_before>package system

import (
	"testing"
	"time"

	"github.com/rqlite/rqlite/cluster"
	"github.com/rqlite/rqlite/command"
	"github.com/rqlite/rqlite/tcp"
)

const shortWait = 5 * time.Second

// Test_StoreClientSideBySide operates on the same store directly, and via
// RPC, and ensures results are the same for basically the same operation.
func Test_StoreClientSideBySide(t *testing.T) {
	node := mustNewLeaderNode()
	defer node.Deprovision()
	leaderAddr, err := node.Store.LeaderAddr()
	if err != nil {
		t.Fatalf("failed to get leader Raft address: %s", err.Error())
	}

	client := cluster.NewClient(mustNewDialer(cluster.MuxClusterHeader, false, false), 30*time.Second)

	res, err := node.Store.Execute(executeRequestFromString("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"))
	if err != nil {
		t.Fatalf("failed to execute on local: %s", err.Error())
	}
	if exp, got := "[{}]", asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	res, err = client.Execute(executeRequestFromString("CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"), leaderAddr, "", "", shortWait)
	if err != nil {
		t.Fatalf("failed to execute via remote: %s", err.Error())
	}
	if exp, got := "[{}]", asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	res, err = node.Store.Execute(executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`))
	if err != nil {
		t.Fatalf("failed to execute on local: %s", err.Error())
	}
	if exp, got := `[{"last_insert_id":1,"rows_affected":1}]`, asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	res, err = client.Execute(executeRequestFromString(`INSERT INTO bar(name) VALUES("fiona")`), leaderAddr, "", "", shortWait)
	if err != nil {
		t.Fatalf("failed to execute via remote: %s", err.Error())
	}
	if exp, got := `[{"last_insert_id":1,"rows_affected":1}]`, asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	rows, err := node.Store.Query(queryRequestFromString(`SELECT * FROM foo`))
	if err != nil {
		t.Fatalf("failed to query on local: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	rows, err = node.Store.Query(queryRequestFromString(`SELECT * FROM bar`))
	if err != nil {
		t.Fatalf("failed to query on local: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	rows, err = client.Query(queryRequestFromString(`SELECT * FROM foo`), leaderAddr, "", "", shortWait)
	if err != nil {
		t.Fatalf("failed to query via remote: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	rows, err = client.Query(queryRequestFromString(`SELECT * FROM bar`), leaderAddr, "", "", shortWait)
	if err != nil {
		t.Fatalf("failed to query via remote: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	rows, err = node.Store.Query(queryRequestFromString(`SELECT * FROM qux`))
	if err != nil {
		t.Fatalf("failed to query on local: %s", err.Error())
	}
	if exp, got := `[{"error":"no such table: qux"}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	rows, err = client.Query(queryRequestFromString(`SELECT * FROM qux`), leaderAddr, "", "", shortWait)
	if err != nil {
		t.Fatalf("failed to query via remote: %s", err.Error())
	}
	if exp, got := `[{"error":"no such table: qux"}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
}

// Test_MultiNodeCluster tests formation of a 3-node cluster and query
// against all nodes to test requests are forwarded to leader transparently.
func Test_MultiNodeClusterRequestForwardOK(t *testing.T) {
	node1 := mustNewLeaderNode()
	defer node1.Deprovision()

	node2 := mustNewNode(false)
	defer node2.Deprovision()
	if err := node2.Join(node1); err != nil {
		t.Fatalf("node failed to join leader: %s", err.Error())
	}
	_, err := node2.WaitForLeader()
	if err != nil {
		t.Fatalf("failed waiting for leader: %s", err.Error())
	}

	// Get the new leader, in case it changed.
	c := Cluster{node1, node2}
	leader, err := c.Leader()
	if err != nil {
		t.Fatalf("failed to find cluster leader: %s", err.Error())
	}

	node3 := mustNewNode(false)
	defer node3.Deprovision()
	if err := node3.Join(leader); err != nil {
		t.Fatalf("node failed to join leader: %s", err.Error())
	}
	_, err = node3.WaitForLeader()
	if err != nil {
		t.Fatalf("failed waiting for leader: %s", err.Error())
	}

	// Get the new leader, in case it changed.
	c = Cluster{node1, node2, node3}
	leader, err = c.Leader()
	if err != nil {
		t.Fatalf("failed to find cluster leader: %s", err.Error())
	}

	followers, err := c.Followers()
	if err != nil {
		t.Fatalf("failed to get followers: %s", err.Error())
	}
	if len(followers) != 2 {
		t.Fatalf("got incorrect number of followers: %d", len(followers))
	}

	res, err := followers[0].Execute(`CREATE TABLE foo (id integer not null primary key, name text)`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	res, err = followers[1].Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{"last_insert_id":1,"rows_affected":1}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	res, err = leader.Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{"last_insert_id":2,"rows_affected":1}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	rows, err := followers[0].Query(`SELECT COUNT(*) FROM foo`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[2]]}]}`, rows; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}
}

// Test_MultiNodeClusterQueuedRequestForwardOK tests that queued writes are forwarded
// correctly.
func Test_MultiNodeClusterQueuedRequestForwardOK(t *testing.T) {
	node1 := mustNewLeaderNode()
	defer node1.Deprovision()

	node2 := mustNewNode(false)
	defer node2.Deprovision()
	if err := node2.Join(node1); err != nil {
		t.Fatalf("node failed to join leader: %s", err.Error())
	}
	_, err := node2.WaitForLeader()
	if err != nil {
		t.Fatalf("failed waiting for leader: %s", err.Error())
	}

	// Get the new leader, in case it changed.
	c := Cluster{node1, node2}
	leader, err := c.Leader()
	if err != nil {
		t.Fatalf("failed to find cluster leader: %s", err.Error())
	}

	// Create table and confirm its existence.
	res, err := leader.Execute(`CREATE TABLE foo (id integer not null primary key, name text)`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}
	rows, err := leader.Query(`SELECT COUNT(*) FROM foo`)
	if err != nil {
		t.Fatalf("failed to query for count: %s", err.Error())
	}
	if exp, got := `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[0]]}]}`, rows; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	// Write a request to a follower's queue, checking it's eventually sent to the leader.
	followers, err := c.Followers()
	if err != nil {
		t.Fatalf("failed to get followers: %s", err.Error())
	}
	if len(followers) != 1 {
		t.Fatalf("got incorrect number of followers: %d", len(followers))
	}
	res, err = followers[0].ExecuteQueued(`INSERT INTO foo(name) VALUES("fiona")`, false)
	if err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}

	ticker := time.NewTicker(10 * time.Millisecond)
	timer := time.NewTimer(5 * time.Second)
	for {
		select {
		case <-ticker.C:
			r, err := leader.Query(`SELECT COUNT(*) FROM foo`)
			if err != nil {
				t.Fatalf("failed to query for count: %s", err.Error())
			}
			if r == `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[1]]}]}` {
				return
			}
		case <-timer.C:
			t.Fatalf("timed out waiting for queued writes")
		}
	}
}

func executeRequestFromString(s string) *command.ExecuteRequest {
	return executeRequestFromStrings([]string{s})
}

// queryRequestFromStrings converts a slice of strings into a command.ExecuteRequest
func executeRequestFromStrings(s []string) *command.ExecuteRequest {
	stmts := make([]*command.Statement, len(s))
	for i := range s {
		stmts[i] = &command.Statement{
			Sql: s[i],
		}
	}
	return &command.ExecuteRequest{
		Request: &command.Request{
			Statements:  stmts,
			Transaction: false,
		},
		Timings: false,
	}
}

func queryRequestFromString(s string) *command.QueryRequest {
	return queryRequestFromStrings([]string{s})
}

// queryRequestFromStrings converts a slice of strings into a command.QueryRequest
func queryRequestFromStrings(s []string) *command.QueryRequest {
	stmts := make([]*command.Statement, len(s))
	for i := range s {
		stmts[i] = &command.Statement{
			Sql: s[i],
		}
	}
	return &command.QueryRequest{
		Request: &command.Request{
			Statements:  stmts,
			Transaction: false,
		},
		Timings: false,
	}
}

func mustNewDialer(header byte, remoteEncrypted, skipVerify bool) *tcp.Dialer {
	return tcp.NewDialer(header, remoteEncrypted, skipVerify)
}
<commit_msg>More use of cluster.Credentials<commit_after>package system

import (
	"testing"
	"time"

	"github.com/rqlite/rqlite/cluster"
	"github.com/rqlite/rqlite/command"
	"github.com/rqlite/rqlite/tcp"
)

const shortWait = 5 * time.Second

// Test_StoreClientSideBySide operates on the same store directly, and via
// RPC, and ensures results are the same for basically the same operation.
func Test_StoreClientSideBySide(t *testing.T) {
	node := mustNewLeaderNode()
	defer node.Deprovision()
	leaderAddr, err := node.Store.LeaderAddr()
	if err != nil {
		t.Fatalf("failed to get leader Raft address: %s", err.Error())
	}

	client := cluster.NewClient(mustNewDialer(cluster.MuxClusterHeader, false, false), 30*time.Second)

	res, err := node.Store.Execute(executeRequestFromString("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"))
	if err != nil {
		t.Fatalf("failed to execute on local: %s", err.Error())
	}
	if exp, got := "[{}]", asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	res, err = client.Execute(executeRequestFromString("CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"), leaderAddr, nil, shortWait)
	if err != nil {
		t.Fatalf("failed to execute via remote: %s", err.Error())
	}
	if exp, got := "[{}]", asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	res, err = node.Store.Execute(executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`))
	if err != nil {
		t.Fatalf("failed to execute on local: %s", err.Error())
	}
	if exp, got := `[{"last_insert_id":1,"rows_affected":1}]`, asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	res, err = client.Execute(executeRequestFromString(`INSERT INTO bar(name) VALUES("fiona")`), leaderAddr, nil, shortWait)
	if err != nil {
		t.Fatalf("failed to execute via remote: %s", err.Error())
	}
	if exp, got := `[{"last_insert_id":1,"rows_affected":1}]`, asJSON(res); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	rows, err := node.Store.Query(queryRequestFromString(`SELECT * FROM foo`))
	if err != nil {
		t.Fatalf("failed to query on local: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	rows, err = node.Store.Query(queryRequestFromString(`SELECT * FROM bar`))
	if err != nil {
		t.Fatalf("failed to query on local: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	rows, err = client.Query(queryRequestFromString(`SELECT * FROM foo`), leaderAddr, nil, shortWait)
	if err != nil {
		t.Fatalf("failed to query via remote: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	rows, err = client.Query(queryRequestFromString(`SELECT * FROM bar`), leaderAddr, nil, shortWait)
	if err != nil {
		t.Fatalf("failed to query via remote: %s", err.Error())
	}
	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}

	rows, err = node.Store.Query(queryRequestFromString(`SELECT * FROM qux`))
	if err != nil {
		t.Fatalf("failed to query on local: %s", err.Error())
	}
	if exp, got := `[{"error":"no such table: qux"}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
	rows, err = client.Query(queryRequestFromString(`SELECT * FROM qux`), leaderAddr, nil, shortWait)
	if err != nil {
		t.Fatalf("failed to query via remote: %s", err.Error())
	}
	if exp, got := `[{"error":"no such table: qux"}]`, asJSON(rows); exp != got {
		t.Fatalf("unexpected results, expt %s, got %s", exp, got)
	}
}

// Test_MultiNodeCluster tests formation of a 3-node cluster and query
// against all nodes to test requests are forwarded to leader transparently.
func Test_MultiNodeClusterRequestForwardOK(t *testing.T) {
	node1 := mustNewLeaderNode()
	defer node1.Deprovision()

	node2 := mustNewNode(false)
	defer node2.Deprovision()
	if err := node2.Join(node1); err != nil {
		t.Fatalf("node failed to join leader: %s", err.Error())
	}
	_, err := node2.WaitForLeader()
	if err != nil {
		t.Fatalf("failed waiting for leader: %s", err.Error())
	}

	// Get the new leader, in case it changed.
	c := Cluster{node1, node2}
	leader, err := c.Leader()
	if err != nil {
		t.Fatalf("failed to find cluster leader: %s", err.Error())
	}

	node3 := mustNewNode(false)
	defer node3.Deprovision()
	if err := node3.Join(leader); err != nil {
		t.Fatalf("node failed to join leader: %s", err.Error())
	}
	_, err = node3.WaitForLeader()
	if err != nil {
		t.Fatalf("failed waiting for leader: %s", err.Error())
	}

	// Get the new leader, in case it changed.
	c = Cluster{node1, node2, node3}
	leader, err = c.Leader()
	if err != nil {
		t.Fatalf("failed to find cluster leader: %s", err.Error())
	}

	followers, err := c.Followers()
	if err != nil {
		t.Fatalf("failed to get followers: %s", err.Error())
	}
	if len(followers) != 2 {
		t.Fatalf("got incorrect number of followers: %d", len(followers))
	}

	res, err := followers[0].Execute(`CREATE TABLE foo (id integer not null primary key, name text)`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	res, err = followers[1].Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{"last_insert_id":1,"rows_affected":1}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	res, err = leader.Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{"last_insert_id":2,"rows_affected":1}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	rows, err := followers[0].Query(`SELECT COUNT(*) FROM foo`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[2]]}]}`, rows; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}
}

// Test_MultiNodeClusterQueuedRequestForwardOK tests that queued writes are forwarded
// correctly.
func Test_MultiNodeClusterQueuedRequestForwardOK(t *testing.T) {
	node1 := mustNewLeaderNode()
	defer node1.Deprovision()

	node2 := mustNewNode(false)
	defer node2.Deprovision()
	if err := node2.Join(node1); err != nil {
		t.Fatalf("node failed to join leader: %s", err.Error())
	}
	_, err := node2.WaitForLeader()
	if err != nil {
		t.Fatalf("failed waiting for leader: %s", err.Error())
	}

	// Get the new leader, in case it changed.
	c := Cluster{node1, node2}
	leader, err := c.Leader()
	if err != nil {
		t.Fatalf("failed to find cluster leader: %s", err.Error())
	}

	// Create table and confirm its existence.
	res, err := leader.Execute(`CREATE TABLE foo (id integer not null primary key, name text)`)
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if exp, got := `{"results":[{}]}`, res; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}
	rows, err := leader.Query(`SELECT COUNT(*) FROM foo`)
	if err != nil {
		t.Fatalf("failed to query for count: %s", err.Error())
	}
	if exp, got := `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[0]]}]}`, rows; exp != got {
		t.Fatalf("got incorrect response from follower exp: %s, got: %s", exp, got)
	}

	// Write a request to a follower's queue, checking it's eventually sent to the leader.
	followers, err := c.Followers()
	if err != nil {
		t.Fatalf("failed to get followers: %s", err.Error())
	}
	if len(followers) != 1 {
		t.Fatalf("got incorrect number of followers: %d", len(followers))
	}
	res, err = followers[0].ExecuteQueued(`INSERT INTO foo(name) VALUES("fiona")`, false)
	if err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}

	ticker := time.NewTicker(10 * time.Millisecond)
	timer := time.NewTimer(5 * time.Second)
	for {
		select {
		case <-ticker.C:
			r, err := leader.Query(`SELECT COUNT(*) FROM foo`)
			if err != nil {
				t.Fatalf("failed to query for count: %s", err.Error())
			}
			if r == `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[1]]}]}` {
				return
			}
		case <-timer.C:
			t.Fatalf("timed out waiting for queued writes")
		}
	}
}

func executeRequestFromString(s string) *command.ExecuteRequest {
	return executeRequestFromStrings([]string{s})
}

// queryRequestFromStrings converts a slice of strings into a command.ExecuteRequest
func executeRequestFromStrings(s []string) *command.ExecuteRequest {
	stmts := make([]*command.Statement, len(s))
	for i := range s {
		stmts[i] = &command.Statement{
			Sql: s[i],
		}
	}
	return &command.ExecuteRequest{
		Request: &command.Request{
			Statements:  stmts,
			Transaction: false,
		},
		Timings: false,
	}
}

func queryRequestFromString(s string) *command.QueryRequest {
	return queryRequestFromStrings([]string{s})
}

// queryRequestFromStrings converts a slice of strings into a command.QueryRequest
func queryRequestFromStrings(s []string) *command.QueryRequest {
	stmts := make([]*command.Statement, len(s))
	for i := range s {
		stmts[i] = &command.Statement{
			Sql: s[i],
		}
	}
	return &command.QueryRequest{
		Request: &command.Request{
			Statements:  stmts,
			Transaction: false,
		},
		Timings: false,
	}
}

func mustNewDialer(header byte, remoteEncrypted, skipVerify bool) *tcp.Dialer {
	return tcp.NewDialer(header, remoteEncrypted, skipVerify)
}
<|endoftext|>


<commit_before>package scheduling

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/skia-dev/glog"

	"golang.org/x/net/context"

	"cloud.google.com/go/pubsub"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
)

const (
	PUBSUB_TOPIC_SWARMING_TASKS       = "swarming-tasks"
	PUBSUB_FULLY_QUALIFIED_TOPIC_TMPL = "projects/%s/topics/%s"
)

var (
	PUBSUB_TOPICS                    = []string{PUBSUB_TOPIC_SWARMING_TASKS}
	PUBSUB_TOPIC_SWARMING_TASKS_FULL = fmt.Sprintf(PUBSUB_FULLY_QUALIFIED_TOPIC_TMPL, common.PROJECT_ID, PUBSUB_TOPIC_SWARMING_TASKS)
)

// InitPubSub ensures that the pub/sub topics and subscriptions needed by the
// TaskScheduler exist.
func InitPubSub() error {
	ctx := context.Background()

	// Create a client.
	client, err := pubsub.NewClient(ctx, common.PROJECT_ID)
	if err != nil {
		return err
	}

	// Create topics and subscriptions if necessary.
	for _, topicName := range PUBSUB_TOPICS {
		// Topic.
		topic := client.Topic(topicName)
		exists, err := topic.Exists(ctx)
		if err != nil {
			return err
		}
		if !exists {
			if _, err := client.CreateTopic(ctx, topicName); err != nil {
				return err
			}
		}

		// Subscription.
		subName := fmt.Sprintf("task_scheduler+%s", topicName)
		sub := client.Subscription(subName)
		exists, err = sub.Exists(ctx)
		if err != nil {
			return err
		}
		if !exists {
			c := &pubsub.PushConfig{
				Endpoint: fmt.Sprintf("https://task-scheduler.skia.org/pubsub/%s", topicName),
			}
			if _, err := client.CreateSubscription(ctx, subName, topic, 20*time.Second, c); err != nil {
				return err
			}
		}
	}
	return nil
}

// PubSubTaskMessage is a message received from Swarming via pub/sub about a
// Task.
type PubSubTaskMessage struct {
	SwarmingTaskId string `json:"task_id"`
	UserData       string `json:"userdata"`
}

// RegisterPubSubServer adds handler to r that handle pub/sub push
// notifications.
func RegisterPubSubServer(s *TaskScheduler, r *mux.Router) {
	r.HandleFunc(fmt.Sprintf("/pubsub/%s", PUBSUB_TOPIC_SWARMING_TASKS), func(w http.ResponseWriter, r *http.Request) {
		var m pubsub.Message
		if err := json.NewDecoder(r.Body).Decode(&m); err != nil {
			httputils.ReportError(w, r, err, "Failed to decode request body.")
			return
		}

		// TODO(borenet): Ensure that the auth token is correct.
		var t PubSubTaskMessage
		if err := json.Unmarshal(m.Data, &t); err != nil {
			httputils.ReportError(w, r, err, "Failed to decode PubSubTaskMessage.")
			return
		}

		glog.Infof("Got task notification from Swarming: %s", t.SwarmingTaskId)
		/*if err := s.updateTaskFromSwarming(t.SwarmingTaskId); err != nil {
			httputils.ReportError(w, r, err, "Failed to process Swarming task.")
			return
		}*/
	}).Methods(http.MethodPost)
}
<commit_msg>[task scheduler] Fix PubSub HTTP request format<commit_after>package scheduling

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/skia-dev/glog"

	"golang.org/x/net/context"

	"cloud.google.com/go/pubsub"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
)

const (
	PUBSUB_TOPIC_SWARMING_TASKS       = "swarming-tasks"
	PUBSUB_FULLY_QUALIFIED_TOPIC_TMPL = "projects/%s/topics/%s"
)

var (
	PUBSUB_TOPICS                    = []string{PUBSUB_TOPIC_SWARMING_TASKS}
	PUBSUB_TOPIC_SWARMING_TASKS_FULL = fmt.Sprintf(PUBSUB_FULLY_QUALIFIED_TOPIC_TMPL, common.PROJECT_ID, PUBSUB_TOPIC_SWARMING_TASKS)
)

// InitPubSub ensures that the pub/sub topics and subscriptions needed by the
// TaskScheduler exist.
func InitPubSub() error {
	ctx := context.Background()

	// Create a client.
	client, err := pubsub.NewClient(ctx, common.PROJECT_ID)
	if err != nil {
		return err
	}

	// Create topics and subscriptions if necessary.
	for _, topicName := range PUBSUB_TOPICS {
		// Topic.
		topic := client.Topic(topicName)
		exists, err := topic.Exists(ctx)
		if err != nil {
			return err
		}
		if !exists {
			if _, err := client.CreateTopic(ctx, topicName); err != nil {
				return err
			}
		}

		// Subscription.
		subName := fmt.Sprintf("task_scheduler+%s", topicName)
		sub := client.Subscription(subName)
		exists, err = sub.Exists(ctx)
		if err != nil {
			return err
		}
		if !exists {
			c := &pubsub.PushConfig{
				Endpoint: fmt.Sprintf("https://task-scheduler.skia.org/pubsub/%s", topicName),
			}
			if _, err := client.CreateSubscription(ctx, subName, topic, 20*time.Second, c); err != nil {
				return err
			}
		}
	}
	return nil
}

// PubSubRequest is the format of pub/sub HTTP request body.
type PubSubRequest struct {
	Message      pubsub.Message `json:"message"`
	Subscription string         `json:"subscription"`
}

// PubSubTaskMessage is a message received from Swarming via pub/sub about a
// Task.
type PubSubTaskMessage struct {
	SwarmingTaskId string `json:"task_id"`
}

// RegisterPubSubServer adds handler to r that handle pub/sub push
// notifications.
func RegisterPubSubServer(s *TaskScheduler, r *mux.Router) {
	r.HandleFunc(fmt.Sprintf("/pubsub/%s", PUBSUB_TOPIC_SWARMING_TASKS), func(w http.ResponseWriter, r *http.Request) {
		var req PubSubRequest
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
			httputils.ReportError(w, r, err, "Failed to decode request body.")
			return
		}

		// TODO(borenet): Ensure that the auth token is correct.
		var t PubSubTaskMessage
		if err := json.Unmarshal(req.Message.Data, &t); err != nil {
			httputils.ReportError(w, r, err, "Failed to decode PubSubTaskMessage.")
			return
		}

		glog.Infof("Got task notification from Swarming: %s", t.SwarmingTaskId)
		/*if err := s.updateTaskFromSwarming(t.SwarmingTaskId); err != nil {
			httputils.ReportError(w, r, err, "Failed to process Swarming task.")
			return
		}*/
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodPost)
}
<|endoftext|>


<commit_before>package analytic

import (
	"testing"

	"github.com/ready-steady/probability"
	"github.com/ready-steady/probability/uniform"
	"github.com/ready-steady/support/assert"
)

func TestCompute(t *testing.T) {
	temperature := load("002")

	cc := uint(2)
	sc := uint(len(fixtureP)) / cc

	Q := temperature.Compute(fixtureP, sc)

	assert.EqualWithin(Q, fixtureQ, 1e-12, t)
}

func BenchmarkCompute(b *testing.B) {
	temperature := load("032")

	cc := uint(32)
	sc := uint(1000)

	P := probability.Sample(uniform.New(0, 1), cc*sc)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		temperature.Compute(P, sc)
	}
}
<commit_msg>Added a benchmark for 002 in analytic<commit_after>package analytic

import (
	"testing"

	"github.com/ready-steady/probability"
	"github.com/ready-steady/probability/uniform"
	"github.com/ready-steady/support/assert"
)

func TestCompute(t *testing.T) {
	temperature := load("002")
	sc := uint(len(fixtureP)) / 2

	Q := temperature.Compute(fixtureP, sc)

	assert.EqualWithin(Q, fixtureQ, 1e-12, t)
}

func BenchmarkCompute002(b *testing.B) {
	temperature := load("002")
	sc := uint(len(fixtureP)) / 2

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		temperature.Compute(fixtureP, sc)
	}
}

func BenchmarkCompute032(b *testing.B) {
	temperature := load("032")
	cc, sc := uint(32), uint(1000)

	P := probability.Sample(uniform.New(0, 1), cc*sc)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		temperature.Compute(P, sc)
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"encoding/binary"
	"os"
	"path/filepath"
)

/*
Create storage
*/
func (m *ManagerStruct) Create(ID string) error {
	f, err := os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	m.cache.Add(ID, f)
	return nil
}

/*
SaveData ...
*/
func (m *ManagerStruct) SaveData(ID string, data []byte, offset int64) error {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	if err = binary.Write(buf, binary.BigEndian, data); err != nil {
		return err
	}
	_, err = f.WriteAt(buf.Bytes(), offset)
	return err
}

/*
DeleteData ...
*/
func (m *ManagerStruct) DeleteData(ID string) error {
	v, ok := m.cache.Peek(ID)
	if ok {
		err := v.(*os.File).Close()
		if err != nil {
			logger.Error.Println(err)
		}
	}
	path := filepath.Join(dataPath, ID)
	return os.Remove(path)
}

/*
LoadData ...
*/
func (m *ManagerStruct) LoadData(ID string, offset int64, length int64) ([]byte, error) {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return nil, err
	}
	if length == 0 {
		if info, err := f.Stat(); err == nil {
			length = info.Size()
			length -= offset
		} else {
			return nil, err
		}
	}
	data := make([]byte, length)
	_, err = f.ReadAt(data, offset)
	return data, err
}

func (m *ManagerStruct) getFileFromCache(ID string) (*os.File, error) {
	v, ok := m.cache.Get(ID)
	var f *os.File
	var err error
	if !ok {
		f, err = os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
	} else {
		f = v.(*os.File)
	}
	return f, nil
}
<commit_msg>remove redundant buffer creation in data_manager<commit_after>package storage

import (
	"os"
	"path/filepath"
)

/*
Create storage
*/
func (m *ManagerStruct) Create(ID string) error {
	f, err := os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	m.cache.Add(ID, f)
	return nil
}

/*
SaveData ...
*/
func (m *ManagerStruct) SaveData(ID string, data []byte, offset int64) error {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return err
	}
	_, err = f.WriteAt(data, offset)
	return err
}

/*
DeleteData ...
*/
func (m *ManagerStruct) DeleteData(ID string) error {
	v, ok := m.cache.Peek(ID)
	if ok {
		err := v.(*os.File).Close()
		if err != nil {
			logger.Error.Println(err)
		}
	}
	path := filepath.Join(dataPath, ID)
	return os.Remove(path)
}

/*
LoadData ...
*/
func (m *ManagerStruct) LoadData(ID string, offset int64, length int64) ([]byte, error) {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return nil, err
	}
	if length == 0 {
		if info, err := f.Stat(); err == nil {
			length = info.Size()
			length -= offset
		} else {
			return nil, err
		}
	}
	data := make([]byte, length)
	_, err = f.ReadAt(data, offset)
	return data, err
}

func (m *ManagerStruct) getFileFromCache(ID string) (*os.File, error) {
	v, ok := m.cache.Get(ID)
	var f *os.File
	var err error
	if !ok {
		f, err = os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
	} else {
		f = v.(*os.File)
	}
	return f, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"io"
	"fmt"
	"github.com/viant/toolbox"
	"path"
	"strings"
	"archive/zip"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.Reader) (io.Reader, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition:= strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result= string(result[pathRoot:])
	}
	return result
}

func copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath== toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}
		var reader io.Reader
		if object.IsContent() {
			reader, err = sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("Unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("Unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
					return err
				}
			}

			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if (destinationObject != nil && destinationObject.IsFolder()) {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName   {
						if ! strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
						}

					}
				}
			}

			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return nil
			}

		} else {

			err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("Unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL);
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("Failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<commit_msg>patched copy handler<commit_after>package storage

import (
	"io"
	"fmt"
	"github.com/viant/toolbox"
	"path"
	"strings"
	"archive/zip"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.Reader) (io.Reader, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition:= strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result= string(result[pathRoot:])
	}
	return result
}

func copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath== toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}
		var reader io.Reader
		if object.IsContent() {
			reader, err = sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("Unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("Unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
					return err
				}
			}

			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if (destinationObject != nil && destinationObject.IsFolder()) {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName   {
						if ! strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
						}

					}
				}
			}

			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return err
			}

		} else {

			err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("Unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL);
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("Failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<|endoftext|>


<commit_before>package form

import (
	"strconv"
	"strings"
	"time"
)

// Bool is a bool that implements Parser
type Bool struct {
	Data *bool
}

// Parse is an implementation of Parser
func (b Bool) Parse(d []string) error {
	switch strings.ToLower(d[0]) {
	case "on", "yes", "y":
		*b.Data = true
	case "off", "no", "n":
		*b.Data = false
	default:
		c, err := strconv.ParseBool(d[0])
		*b.Data = c
		return err
	}
	return nil
}

// Int is an int that implements Parser
type Int struct {
	Data *int
}

// Parse is an implementation of Parser
func (i Int) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 0)
	*i.Data = int(n)
	return err
}

// Int8 is an int8 that implements Parser
type Int8 struct {
	Data *int8
}

// Parse is an implementation of Parser
func (i Int8) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 8)
	*i.Data = int8(n)
	return err
}

// Int16 is an int16 that implements Parser
type Int16 struct {
	Data *int16
}

// Parse is an implementation of Parser
func (i Int16) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 16)
	*i.Data = int16(n)
	return err
}

// Int32 is an int32 that implements Parser
type Int32 struct {
	Data *int32
}

// Parse is an implementation of Parser
func (i Int32) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 32)
	*i.Data = int32(n)
	return err
}

// Int64 is an int64 that implements Parser
type Int64 struct {
	Data *int64
}

// Parse is an implementation of Parser
func (i Int64) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 64)
	*i.Data = n
	return err
}

// Uint is a uint that implements Parser
type Uint struct {
	Data *uint
}

// Parse is an implementation of Parser
func (u Uint) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 0)
	*u.Data = uint(n)
	return err
}

// Uint8 is a uint8 that implements Parser
type Uint8 struct {
	Data *uint8
}

// Parse is an implementation of Parser
func (u Uint8) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 8)
	*u.Data = uint8(n)
	return err
}

// Uint16 is a uint16 that implements Parser
type Uint16 struct {
	Data *uint16
}

// Parse is an implementation of Parser
func (u Uint16) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 16)
	*u.Data = uint16(n)
	return err
}

// Uint32 is a uint32 that implements Parser
type Uint32 struct {
	Data *uint32
}

// Parse is an implementation of Parser
func (u Uint32) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 32)
	*u.Data = uint32(n)
	return err
}

// Uint64 is a uint64 that implements Parser
type Uint64 struct {
	Data *uint64
}

// Parse is an implementation of Parser
func (u Uint64) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 64)
	*u.Data = n
	return err
}

// Float32 is a float32 that implements Parser
type Float32 struct {
	Data *float32
}

// Parse is an implementation of Parser
func (f Float32) Parse(d []string) error {
	n, err := strconv.ParseFloat(d[0], 32)
	*f.Data = float32(n)
	return err
}

// Float64 is a float64 that implements Parser
type Float64 struct {
	Data *float64
}

// Parse is an implementation of Parser
func (f Float64) Parse(d []string) error {
	n, err := strconv.ParseFloat(d[0], 64)
	*f.Data = n
	return err
}

// String is a string that implements Parser
type String struct {
	Data *string
}

// Parse is an implementation of Parser
func (s String) Parse(d []string) error {
	*s.Data = d[0]
	return nil
}

var formats = [...]string{
	time.ANSIC,
	time.Kitchen,
	time.RFC1123,
	time.RFC1123Z,
	time.RFC3339,
	time.RFC3339Nano,
	time.RFC822,
	time.RFC822Z,
	time.RFC850,
	time.RubyDate,
	time.Stamp,
	time.StampNano,
	time.StampMicro,
	time.StampMilli,
	time.UnixDate,
	"2/1/06 15:04:05",
	"2/1/2006 15:04:05",
	"02/01/06 15:04:05",
	"02/01/2006 15:04:05",
	"2/1/06 15:04",
	"2/1/2006 15:04",
	"02/01/06 15:04",
	"02/01/2006 15:04",
	"15:04:05 2/1/06",
	"15:04:05 2/1/2006",
	"15:04:05 02/01/06",
	"15:04:05 02/01/2006",
	"15:04 2/1/06",
	"15:04 2/1/2006",
	"15:04 02/01/06",
	"15:04 02/01/2006",
	"2-1-06 15:04:05",
	"2-1-2006 15:04:05",
	"02-01-06 15:04:05",
	"02-01-2006 15:04:05",
	"2-1-06 15:04",
	"2-1-2006 15:04",
	"02-01-06 15:04",
	"02-01-2006 15:04",
	"15:04:05 2-1-06",
	"15:04:05 2-1-2006",
	"15:04:05 02-01-06",
	"15:04:05 02-01-2006",
	"15:04 2-1-06",
	"15:04 2-1-2006",
	"15:04 02-01-06",
	"15:04 02-01-2006",
}

// Time is a time.Time that implements Parser
type Time struct {
	Data *time.Time
}

// Parse is an implementation of Parser
func (t Time) Parse(d []string) error {
	for _, format := range formats {
		pt, err := time.Parse(format, d[0])
		if err == nil {
			*t.Data = pt
			return nil
		}
	}
	return UnknownFormat(d[0])
}

// TimeFormat is like a Time except that it allows a specific format to be
// specified
type TimeFormat struct {
	Data   *time.Time
	Format string
}

// Parse is an implementation of Parser
func (t TimeFormat) Parse(d []string) error {
	pt, err := time.Parse(t.Format, d[0])
	if err != nil {
		*t.Data = pt
	}
	return err
}

// Errors

// UnknownFormat is an error return from Time.Parse when it cannot determine
// the time format of the given string
type UnknownFormat string

func (UnknownFormat) Error() string {
	return "unknown time format"
}
<commit_msg>Added range and regex types<commit_after>package form

import (
	"regexp"
	"strconv"
	"strings"
	"time"
)

// Bool is a bool that implements Parser
type Bool struct {
	Data *bool
}

// Parse is an implementation of Parser
func (b Bool) Parse(d []string) error {
	switch strings.ToLower(d[0]) {
	case "on", "yes", "y":
		*b.Data = true
	case "off", "no", "n":
		*b.Data = false
	default:
		c, err := strconv.ParseBool(d[0])
		*b.Data = c
		return err
	}
	return nil
}

// Int is an int that implements Parser
type Int struct {
	Data *int
}

// Parse is an implementation of Parser
func (i Int) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 0)
	*i.Data = int(n)
	return err
}

// Int8 is an int8 that implements Parser
type Int8 struct {
	Data *int8
}

// Parse is an implementation of Parser
func (i Int8) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 8)
	*i.Data = int8(n)
	return err
}

// Int16 is an int16 that implements Parser
type Int16 struct {
	Data *int16
}

// Parse is an implementation of Parser
func (i Int16) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 16)
	*i.Data = int16(n)
	return err
}

// Int32 is an int32 that implements Parser
type Int32 struct {
	Data *int32
}

// Parse is an implementation of Parser
func (i Int32) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 32)
	*i.Data = int32(n)
	return err
}

// Int64 is an int64 that implements Parser
type Int64 struct {
	Data *int64
}

// Parse is an implementation of Parser
func (i Int64) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 64)
	*i.Data = n
	return err
}

// IntBetween is an implementation of Parser that allows a signed integer
// between the Min and Max (inclusively)
type IntBetween struct {
	Parser
	Min, Max int64
}

// Parse is an implementation of Parser
func (i IntBetween) Parse(d []string) error {
	n, err := strconv.ParseInt(d[0], 10, 64)
	if err != nil {
		return err
	}
	if n >= i.Min && n <= i.Max {
		return i.Parser.Parse(d)
	}
	return OutsideBounds(d[0])
}

// Uint is a uint that implements Parser
type Uint struct {
	Data *uint
}

// Parse is an implementation of Parser
func (u Uint) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 0)
	*u.Data = uint(n)
	return err
}

// Uint8 is a uint8 that implements Parser
type Uint8 struct {
	Data *uint8
}

// Parse is an implementation of Parser
func (u Uint8) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 8)
	*u.Data = uint8(n)
	return err
}

// Uint16 is a uint16 that implements Parser
type Uint16 struct {
	Data *uint16
}

// Parse is an implementation of Parser
func (u Uint16) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 16)
	*u.Data = uint16(n)
	return err
}

// Uint32 is a uint32 that implements Parser
type Uint32 struct {
	Data *uint32
}

// Parse is an implementation of Parser
func (u Uint32) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 32)
	*u.Data = uint32(n)
	return err
}

// Uint64 is a uint64 that implements Parser
type Uint64 struct {
	Data *uint64
}

// Parse is an implementation of Parser
func (u Uint64) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 64)
	*u.Data = n
	return err
}

// UintBetween is an implementation of Parser that allows an unsigned integer
// between the Min and Max (inclusively)
type UintBetween struct {
	Parser
	Min, Max uint64
}

// Parse is an implementation of Parser
func (u UintBetween) Parse(d []string) error {
	n, err := strconv.ParseUint(d[0], 10, 64)
	if err != nil {
		return err
	}
	if n >= u.Min && n <= u.Max {
		return u.Parser.Parse(d)
	}
	return OutsideBounds(d[0])
}

// Float32 is a float32 that implements Parser
type Float32 struct {
	Data *float32
}

// Parse is an implementation of Parser
func (f Float32) Parse(d []string) error {
	n, err := strconv.ParseFloat(d[0], 32)
	*f.Data = float32(n)
	return err
}

// Float64 is a float64 that implements Parser
type Float64 struct {
	Data *float64
}

// Parse is an implementation of Parser
func (f Float64) Parse(d []string) error {
	n, err := strconv.ParseFloat(d[0], 64)
	*f.Data = n
	return err
}

// FloatBetween is an implementation of Parser that allows a float between the
// Min and Max (inclusively)
type FloatBetween struct {
	Parser
	Min, Max float64
}

// Parse is an implementation of Parser
func (f FloatBetween) Parse(d []string) error {
	n, err := strconv.ParseFloat(d[0], 64)
	if err != nil {
		return err
	}
	if n >= f.Min && n <= f.Max {
		return f.Parser.Parse(d)
	}
	return OutsideBounds(d[0])
}

// String is a string that implements Parser
type String struct {
	Data *string
}

// Parse is an implementation of Parser
func (s String) Parse(d []string) error {
	*s.Data = d[0]
	return nil
}

// RegexString is an implementation of Parser that matches a string again the
// given regular expression
type RegexString struct {
	Data  *string
	Regex *regexp.Regexp
}

// Parse is an implementation of Parser
func (r RegexString) Parse(d []string) error {
	if r.Regex.MatchString(d[0]) {
		*s.Data = d[0]
		return nil
	}
	return NoRegexMatch(d[0])
}

var formats = [...]string{
	time.ANSIC,
	time.Kitchen,
	time.RFC1123,
	time.RFC1123Z,
	time.RFC3339,
	time.RFC3339Nano,
	time.RFC822,
	time.RFC822Z,
	time.RFC850,
	time.RubyDate,
	time.Stamp,
	time.StampNano,
	time.StampMicro,
	time.StampMilli,
	time.UnixDate,
	"2/1/06 15:04:05",
	"2/1/2006 15:04:05",
	"02/01/06 15:04:05",
	"02/01/2006 15:04:05",
	"2/1/06 15:04",
	"2/1/2006 15:04",
	"02/01/06 15:04",
	"02/01/2006 15:04",
	"15:04:05 2/1/06",
	"15:04:05 2/1/2006",
	"15:04:05 02/01/06",
	"15:04:05 02/01/2006",
	"15:04 2/1/06",
	"15:04 2/1/2006",
	"15:04 02/01/06",
	"15:04 02/01/2006",
	"2-1-06 15:04:05",
	"2-1-2006 15:04:05",
	"02-01-06 15:04:05",
	"02-01-2006 15:04:05",
	"2-1-06 15:04",
	"2-1-2006 15:04",
	"02-01-06 15:04",
	"02-01-2006 15:04",
	"15:04:05 2-1-06",
	"15:04:05 2-1-2006",
	"15:04:05 02-01-06",
	"15:04:05 02-01-2006",
	"15:04 2-1-06",
	"15:04 2-1-2006",
	"15:04 02-01-06",
	"15:04 02-01-2006",
}

// Time is a time.Time that implements Parser
type Time struct {
	Data *time.Time
}

// Parse is an implementation of Parser
func (t Time) Parse(d []string) error {
	for _, format := range formats {
		pt, err := time.Parse(format, d[0])
		if err == nil {
			*t.Data = pt
			return nil
		}
	}
	return UnknownFormat(d[0])
}

// TimeFormat is like a Time except that it allows a specific format to be
// specified
type TimeFormat struct {
	Data   *time.Time
	Format string
}

// Parse is an implementation of Parser
func (t TimeFormat) Parse(d []string) error {
	pt, err := time.Parse(t.Format, d[0])
	if err != nil {
		*t.Data = pt
	}
	return err
}

// Errors

// OutsideBounds is an error returned from a Parser when the parsed value falls
// outside of the range given as an acceptable value
type OutsideBounds string

func (OutsideBounds) Error() string {
	return "value received was outside of specified bounds"
}

// NoRegexMatch is an error returned from RegexString.Parse when the given
// string does not match the given regular expression
type NoRegexMatch string

func (NoRegexMatch) Error() string {
	return "string did not match given regular expression"
}

// UnknownFormat is an error returned from Time.Parse when it cannot determine
// the time format of the given string
type UnknownFormat string

func (UnknownFormat) Error() string {
	return "unknown time format"
}
<|endoftext|>


<commit_before>//go:generate generateEmojiCodeMap -pkg discordgo
// Discordgo - Discord bindings for Go
// Available at https://github.com/bwmarrin/discordgo

// Copyright 2015-2016 Bruce Marriner <bruce@sqls.net>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains custom types, currently only a timestamp wrapper.

package discordgo

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/jonas747/gojay"
)

// Timestamp stores a timestamp, as sent by the Discord API.
type Timestamp string

// Parse parses a timestamp string into a time.Time object.
// The only time this can fail is if Discord changes their timestamp format.
func (t Timestamp) Parse() (time.Time, error) {
	return time.Parse(time.RFC3339, string(t))
}

// RESTError stores error information about a request with a bad response code.
// Message is not always present, there are cases where api calls can fail
// without returning a json message.
type RESTError struct {
	Request      *http.Request
	Response     *http.Response
	ResponseBody []byte

	Message *APIErrorMessage // Message may be nil.
}

func newRestError(req *http.Request, resp *http.Response, body []byte) *RESTError {
	restErr := &RESTError{
		Request:      req,
		Response:     resp,
		ResponseBody: body,
	}

	// Attempt to decode the error and assume no message was provided if it fails
	var msg *APIErrorMessage
	err := json.Unmarshal(body, &msg)
	if err == nil {
		restErr.Message = msg
	}

	return restErr
}

func (r RESTError) Error() string {
	return fmt.Sprintf("HTTP %s, %s", r.Response.Status, r.ResponseBody)
}

// IDSlice Is a slice of snowflake id's that properly marshals and unmarshals the way discord expects them to
// They unmarshal from string arrays and marshals back to string arrays
type IDSlice []int64

func (ids *IDSlice) UnmarshalJSON(data []byte) error {
	if len(data) < 3 {
		return nil
	}

	// Split and strip away "[" "]"
	split := strings.Split(string(data[1:len(data)-1]), ",")
	*ids = make([]int64, 0, len(split))
	for _, s := range split {
		s = strings.TrimSpace(s)
		if len(s) < 3 {
			// Empty or invalid
			continue
		}

		// Strip away quotes and parse
		parsed, err := strconv.ParseInt(s[1:len(s)-1], 10, 64)
		if err != nil {
			return err
		}

		*ids = append(*ids, parsed)
	}

	return nil
}

func (ids IDSlice) MarshalJSON() ([]byte, error) {
	// Capacity:
	// 2 brackets
	// each id is:
	//    18 characters currently, but 1 extra added for the future,
	//    1 comma
	//    2 quotes
	if len(ids) < 1 {
		return []byte("[]"), nil
	}

	outPut := make([]byte, 1, 2+(len(ids)*22))
	outPut[0] = '['

	for i, id := range ids {
		if i != 0 {
			outPut = append(outPut, '"', ',', '"')
		} else {
			outPut = append(outPut, '"')
		}
		outPut = append(outPut, []byte(strconv.FormatInt(id, 10))...)
	}

	outPut = append(outPut, '"', ']')
	return outPut, nil
}

// implement UnmarshalerJSONArray
func (ids *IDSlice) UnmarshalJSONArray(dec *gojay.Decoder) error {
	str := ""
	if err := dec.String(&str); err != nil {
		return err
	}

	parsed, err := strconv.ParseInt(str, 10, 64)
	if err != nil {
		return err
	}

	*ids = append(*ids, parsed)
	return nil
}

type EmojiName struct {
	string
}

func (emoji EmojiName) String() string {
	if codepoint, ok := emojiCodeMap[emoji.string]; ok {
		emoji.string = codepoint
	}
	// Discord does not accept the emoji qualifier character.
	// return strings.Replace(emoji.string, "\uFE0F", "", 1)
	// this no longer the case? in fact its required?
	return url.PathEscape(emoji.string)
}

// Discord is super inconsistent with with types in some places (especially presence updates,
// might aswell change them to map[string]interface{} soon because there is 0 validation)
type DiscordFloat float64

func (df *DiscordFloat) UnmarshalJSON(data []byte) error {
	var dst json.Number
	err := json.Unmarshal(data, &dst)
	if err != nil {
		return err
	}

	parsed, err := dst.Float64()
	if err != nil {
		return err
	}

	*df = DiscordFloat(parsed)
	return nil
}

type DiscordInt64 int64

func (di *DiscordInt64) UnmarshalJSON(data []byte) error {
	var dst json.Number
	err := json.Unmarshal(data, &dst)
	if err != nil {
		return err
	}

	parsed, err := dst.Int64()
	if err != nil {
		// Attempt to fallback to float, we lost some precision but eh, what can you do when discord is so freaking inconsistent
		f, err := dst.Float64()
		if err != nil {
			return err
		}
		*di = DiscordInt64(int64(f))
		return nil
	}

	*di = DiscordInt64(parsed)
	return nil
}

func DecodeSnowflake(dst *int64, dec *gojay.Decoder) error {
	var str string
	err := dec.String(&str)
	if err != nil {
		return err
	}

	parsed, err := strconv.ParseInt(str, 10, 64)
	*dst = parsed
	return err
}

type GuildEvent interface {
	GetGuildID() int64
}

type ChannelEvent interface {
	GetChannelID() int64
}
<commit_msg>lower memory usage by avoiding FixedTime form time.Parse<commit_after>//go:generate generateEmojiCodeMap -pkg discordgo
// Discordgo - Discord bindings for Go
// Available at https://github.com/bwmarrin/discordgo

// Copyright 2015-2016 Bruce Marriner <bruce@sqls.net>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains custom types, currently only a timestamp wrapper.

package discordgo

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/jonas747/gojay"
)

// Timestamp stores a timestamp, as sent by the Discord API.
type Timestamp string

// Parse parses a timestamp string into a time.Time object.
// The only time this can fail is if Discord changes their timestamp format.
func (t Timestamp) Parse() (time.Time, error) {
	tim, err := time.Parse(time.RFC3339, string(t))
	return tim.UTC(), err
}

// RESTError stores error information about a request with a bad response code.
// Message is not always present, there are cases where api calls can fail
// without returning a json message.
type RESTError struct {
	Request      *http.Request
	Response     *http.Response
	ResponseBody []byte

	Message *APIErrorMessage // Message may be nil.
}

func newRestError(req *http.Request, resp *http.Response, body []byte) *RESTError {
	restErr := &RESTError{
		Request:      req,
		Response:     resp,
		ResponseBody: body,
	}

	// Attempt to decode the error and assume no message was provided if it fails
	var msg *APIErrorMessage
	err := json.Unmarshal(body, &msg)
	if err == nil {
		restErr.Message = msg
	}

	return restErr
}

func (r RESTError) Error() string {
	return fmt.Sprintf("HTTP %s, %s", r.Response.Status, r.ResponseBody)
}

// IDSlice Is a slice of snowflake id's that properly marshals and unmarshals the way discord expects them to
// They unmarshal from string arrays and marshals back to string arrays
type IDSlice []int64

func (ids *IDSlice) UnmarshalJSON(data []byte) error {
	if len(data) < 3 {
		return nil
	}

	// Split and strip away "[" "]"
	split := strings.Split(string(data[1:len(data)-1]), ",")
	*ids = make([]int64, 0, len(split))
	for _, s := range split {
		s = strings.TrimSpace(s)
		if len(s) < 3 {
			// Empty or invalid
			continue
		}

		// Strip away quotes and parse
		parsed, err := strconv.ParseInt(s[1:len(s)-1], 10, 64)
		if err != nil {
			return err
		}

		*ids = append(*ids, parsed)
	}

	return nil
}

func (ids IDSlice) MarshalJSON() ([]byte, error) {
	// Capacity:
	// 2 brackets
	// each id is:
	//    18 characters currently, but 1 extra added for the future,
	//    1 comma
	//    2 quotes
	if len(ids) < 1 {
		return []byte("[]"), nil
	}

	outPut := make([]byte, 1, 2+(len(ids)*22))
	outPut[0] = '['

	for i, id := range ids {
		if i != 0 {
			outPut = append(outPut, '"', ',', '"')
		} else {
			outPut = append(outPut, '"')
		}
		outPut = append(outPut, []byte(strconv.FormatInt(id, 10))...)
	}

	outPut = append(outPut, '"', ']')
	return outPut, nil
}

// implement UnmarshalerJSONArray
func (ids *IDSlice) UnmarshalJSONArray(dec *gojay.Decoder) error {
	str := ""
	if err := dec.String(&str); err != nil {
		return err
	}

	parsed, err := strconv.ParseInt(str, 10, 64)
	if err != nil {
		return err
	}

	*ids = append(*ids, parsed)
	return nil
}

type EmojiName struct {
	string
}

func (emoji EmojiName) String() string {
	if codepoint, ok := emojiCodeMap[emoji.string]; ok {
		emoji.string = codepoint
	}
	// Discord does not accept the emoji qualifier character.
	// return strings.Replace(emoji.string, "\uFE0F", "", 1)
	// this no longer the case? in fact its required?
	return url.PathEscape(emoji.string)
}

// Discord is super inconsistent with with types in some places (especially presence updates,
// might aswell change them to map[string]interface{} soon because there is 0 validation)
type DiscordFloat float64

func (df *DiscordFloat) UnmarshalJSON(data []byte) error {
	var dst json.Number
	err := json.Unmarshal(data, &dst)
	if err != nil {
		return err
	}

	parsed, err := dst.Float64()
	if err != nil {
		return err
	}

	*df = DiscordFloat(parsed)
	return nil
}

type DiscordInt64 int64

func (di *DiscordInt64) UnmarshalJSON(data []byte) error {
	var dst json.Number
	err := json.Unmarshal(data, &dst)
	if err != nil {
		return err
	}

	parsed, err := dst.Int64()
	if err != nil {
		// Attempt to fallback to float, we lost some precision but eh, what can you do when discord is so freaking inconsistent
		f, err := dst.Float64()
		if err != nil {
			return err
		}
		*di = DiscordInt64(int64(f))
		return nil
	}

	*di = DiscordInt64(parsed)
	return nil
}

func DecodeSnowflake(dst *int64, dec *gojay.Decoder) error {
	var str string
	err := dec.String(&str)
	if err != nil {
		return err
	}

	parsed, err := strconv.ParseInt(str, 10, 64)
	*dst = parsed
	return err
}

type GuildEvent interface {
	GetGuildID() int64
}

type ChannelEvent interface {
	GetChannelID() int64
}
<|endoftext|>