<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor("OK", "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor("UNKNOWN", s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor("CRITICAL", s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor("CRITICAL", s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor("CRITICAL", s)
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	base := filepath.Base(command)
	monitoringEvent = base
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, base), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, base, base+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	// FIXME(nightlyone) capture at least cmd.Stderr, and optionally
	// cmd.Stdout
	cmd = exec.Command(command, flag.Args()[1:]...)

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}
}
<commit_msg>pass constants instead of strings<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor(monitorOk, "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor(monitorUnknown, s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	base := filepath.Base(command)
	monitoringEvent = base
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, base), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, base, base+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	// FIXME(nightlyone) capture at least cmd.Stderr, and optionally
	// cmd.Stdout
	cmd = exec.Command(command, flag.Args()[1:]...)

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"io"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor(monitorOk, "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor(monitorUnknown, s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	base := filepath.Base(command)
	monitoringEvent = base
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, base), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, base, base+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	cmd = exec.Command(command, flag.Args()[1:]...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(os.Stdout, stdout); err != nil {
				break
			}
		}
		wg.Done()
	}()

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}

	wg.Wait()
}
<commit_msg>refactor: derive monitoring event from command<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"io"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor(monitorOk, "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor(monitorUnknown, s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)
	monitoringEvent = filepath.Base(command)

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, monitoringEvent), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, monitoringEvent, monitoringEvent+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	cmd = exec.Command(command, flag.Args()[1:]...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(os.Stdout, stdout); err != nil {
				break
			}
		}
		wg.Done()
	}()

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright © 2017 Alexander Sosna <alexander@xxor.de>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"errors"
	"os"
	"os/exec"
	"time"

	"gogs.xxor.de/xxorde/pgGlaskugel/pkg"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	log "github.com/Sirupsen/logrus"
)

// recoverCmd represents the recover command
var recoverCmd = &cobra.Command{
	Use:   "recover WAL_FILE RECOVER_TO",
	Short: "Recovers a given WAL file",
	Long: `This command recovers a given WAL file.
	Example: archive_command = "` + myName + ` recover %f %p"

It is intended to use as an restore_command in the recovery.conf.
	Example: restore_command = '` + myName + ` recover %f %p'`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 2 {
			log.Fatal("Not enough arguments")
		}

		walName := args[0]
		walTarget := args[1]

		err := recoverWal(walTarget, walName)
		if err != nil {
			log.Fatal("recover failed ", err)
		}
		elapsed := time.Since(startTime)
		log.Info("Recovered WAL file in ", elapsed)
	},
}

func init() {
	RootCmd.AddCommand(recoverCmd)
}

// recoverWal recovers a WAL file with the configured method
func recoverWal(walTarget string, walName string) (err error) {
	archiveTo := viper.GetString("archive_to")

	switch archiveTo {
	case "file":
		return recoverFromFile(walTarget, walName)
	case "s3":
		return recoverFromS3(walTarget, walName)
	default:
		log.Fatal(archiveTo, " no valid value for archiveTo")
	}
	return errors.New("This should never be reached")
}

// recoverFromFile uses the shell command lz4 to recover WAL files
func recoverFromFile(walTarget string, walName string) (err error) {
	walSource := viper.GetString("archivedir") + "/wal/" + walName + ".zst"
	log.Debug("recoverWithZstdCommand, walTarget: ", walTarget, ", walName: ", walName, ", walSource: ", walSource)

	// Check if WAL file is already recovered
	if _, err := os.Stat(walTarget); err == nil {
		err := errors.New("WAL file is already recovered in : " + walTarget)
		return err
	}

	recoverCmd := exec.Command(cmdZstd, "-d", walSource, "-o", walTarget)
	err = recoverCmd.Run()
	return err
}

// recoverFromS3 recover from a S3 compatible object store
func recoverFromS3(walTarget string, walName string) (err error) {
	bucket := viper.GetString("s3_bucket_wal")
	walSource := walName + ".zst"

	// Initialize minio client object.
	minioClient := getS3Connection()

	// Test if bucket is there
	exists, err := minioClient.BucketExists(bucket)
	if err != nil {
		log.Fatal(err)
	}
	if !exists {
		log.Fatal("Bucket to recover from does not exists")
	}

	walObject, err := minioClient.GetObject(bucket, walSource)
	if err != nil {
		log.Fatal(err)
		return
	}
	defer walObject.Close()

	// Test if the object is accessible
	stat, err := walObject.Stat()
	if err != nil {
		log.Fatal(err)
	}
	if stat.Size <= 0 {
		log.Fatal("WAL object has size <= 0")
	}

	// command to inflate the data stream
	inflateCmd := exec.Command(cmdZstd, "-d", "-o", walTarget)

	// Watch output on stderror
	inflateStderror, err := inflateCmd.StderrPipe()
	check(err)
	go pkg.WatchOutput(inflateStderror, log.Info)

	// Assign walObject as Stdin for the inflate command
	inflateCmd.Stdin = walObject

	// Start WAL inflation
	if err := inflateCmd.Start(); err != nil {
		log.Fatal("zstd failed on startup, ", err)
	}
	log.Debug("Inflation started")

	// If there is still data in the output pipe it can be lost!
	err = inflateCmd.Wait()
	if err != nil {
		log.Fatal("inflation failed after startup, ", err)
	}
	return err
}
<commit_msg>Add debug output for recover command<commit_after>// Copyright © 2017 Alexander Sosna <alexander@xxor.de>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"errors"
	"os"
	"os/exec"
	"time"

	"gogs.xxor.de/xxorde/pgGlaskugel/pkg"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	log "github.com/Sirupsen/logrus"
)

// recoverCmd represents the recover command
var recoverCmd = &cobra.Command{
	Use:   "recover <WAL_FILE> <RECOVER_TO>",
	Short: "Recovers a given WAL file",
	Long: `This command recovers a given WAL file.
	Example: archive_command = "` + myName + ` recover %f %p"

It is intended to use as an restore_command in the recovery.conf.
	Example: restore_command = '` + myName + ` recover %f %p'`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 2 {
			log.Fatal("Not enough arguments")
		}

		walName := args[0]
		walTarget := args[1]

		err := recoverWal(walTarget, walName)
		if err != nil {
			log.Fatal("recover failed ", err)
		}
		elapsed := time.Since(startTime)
		log.Info("Recovered WAL file in ", elapsed)
	},
}

func init() {
	RootCmd.AddCommand(recoverCmd)
}

// recoverWal recovers a WAL file with the configured method
func recoverWal(walTarget string, walName string) (err error) {
	archiveTo := viper.GetString("archive_to")

	switch archiveTo {
	case "file":
		return recoverFromFile(walTarget, walName)
	case "s3":
		return recoverFromS3(walTarget, walName)
	default:
		log.Fatal(archiveTo, " no valid value for archiveTo")
	}
	return errors.New("This should never be reached")
}

// recoverFromFile uses the shell command lz4 to recover WAL files
func recoverFromFile(walTarget string, walName string) (err error) {
	log.Debug("recoverFromFile walTarget: ", walTarget, " walName: ", walName)
	walSource := viper.GetString("archivedir") + "/wal/" + walName + ".zst"
	log.Debug("recoverWithZstdCommand, walTarget: ", walTarget, ", walName: ", walName, ", walSource: ", walSource)

	// Check if WAL file is already recovered
	if _, err := os.Stat(walTarget); err == nil {
		err := errors.New("WAL file is already recovered in : " + walTarget)
		return err
	}

	recoverCmd := exec.Command(cmdZstd, "-d", walSource, "-o", walTarget)
	err = recoverCmd.Run()
	return err
}

// recoverFromS3 recover from a S3 compatible object store
func recoverFromS3(walTarget string, walName string) (err error) {
	log.Debug("recoverFromS3 walTarget: ", walTarget, " walName: ", walName)

	bucket := viper.GetString("s3_bucket_wal")
	walSource := walName + ".zst"

	// Initialize minio client object.
	minioClient := getS3Connection()

	// Test if bucket is there
	exists, err := minioClient.BucketExists(bucket)
	if err != nil {
		log.Error("Can not test for S3 bucket")
		log.Fatal(err)
	}
	if !exists {
		log.Fatal("Bucket to recover from does not exists")
	}

	walObject, err := minioClient.GetObject(bucket, walSource)
	if err != nil {
		log.Error("Can not get WAL file from S3")
		log.Fatal(err)
	}
	defer walObject.Close()

	// Test if the object is accessible
	stat, err := walObject.Stat()
	if err != nil {
		log.Error("Can not get stats for WAL file from S3, does WAL file exists?")
		log.Fatal(err)
	}
	if stat.Size <= 0 {
		log.Fatal("WAL object has size <= 0")
	}

	// command to inflate the data stream
	inflateCmd := exec.Command(cmdZstd, "-d", "-o", walTarget)

	// Watch output on stderror
	inflateStderror, err := inflateCmd.StderrPipe()
	check(err)
	go pkg.WatchOutput(inflateStderror, log.Info)

	// Assign walObject as Stdin for the inflate command
	inflateCmd.Stdin = walObject

	// Start WAL inflation
	if err := inflateCmd.Start(); err != nil {
		log.Fatal("zstd failed on startup, ", err)
	}
	log.Debug("Inflation started")

	// If there is still data in the output pipe it can be lost!
	err = inflateCmd.Wait()
	if err != nil {
		log.Fatal("inflation failed after startup, ", err)
	}
	return err
}
<|endoftext|>


<commit_before>/*
 * MinIO Client (C) 2014, 2015 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
)

// rm specific flags.
var (
	rmFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "remove recursively",
		},
		cli.BoolFlag{
			Name:  "force",
			Usage: "allow a recursive remove operation",
		},
		cli.BoolFlag{
			Name:  "dangerous",
			Usage: "allow site-wide removal of objects",
		},
		cli.BoolFlag{
			Name:  "incomplete, I",
			Usage: "remove incomplete uploads",
		},
		cli.BoolFlag{
			Name:  "fake",
			Usage: "perform a fake remove operation",
		},
		cli.BoolFlag{
			Name:  "stdin",
			Usage: "read object names from STDIN",
		},
		cli.StringFlag{
			Name:  "older-than",
			Usage: "remove objects older than L days, M hours and N minutes",
		},
		cli.StringFlag{
			Name:  "newer-than",
			Usage: "remove objects newer than L days, M hours and N minutes",
		},
	}
)

// remove a file or folder.
var rmCmd = cli.Command{
	Name:   "rm",
	Usage:  "remove objects",
	Action: mainRm,
	Before: setGlobalsFromContext,
	Flags:  append(append(rmFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET [TARGET ...]

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
   MC_ENCRYPT_KEY: list of comma delimited prefix=secret values

EXAMPLES:
   1. Remove a file.
      $ {{.HelpName}} 1999/old-backup.tgz

   2. Remove all objects recursively from bucket 'jazz-songs' matching 'louis' prefix.
      $ {{.HelpName}} --recursive s3/jazz-songs/louis/

   3. Remove all objects older than '90' days recursively from bucket 'jazz-songs' that match 'louis' prefix.
      $ {{.HelpName}} --recursive --older-than 90d s3/jazz-songs/louis/

   4. Remove all objects newer than 7 days and 10 hours recursively from bucket 'pop-songs'
      $ {{.HelpName}} --recursive --newer-than 7d10h s3/pop-songs/

   5. Remove all objects read from STDIN.
      $ {{.HelpName}} --force --stdin

   6. Remove all objects recursively from S3 host
      $ {{.HelpName}} --recursive --dangerous s3

   7. Remove all buckets and objects older than '90' days recursively from host
      $ {{.HelpName}} --recursive --dangerous --older-than 90d s3

   8. Drop all incomplete uploads on 'jazz-songs' bucket.
      $ {{.HelpName}} --incomplete --recursive s3/jazz-songs/

   9. Remove an encrypted object from Amazon S3 cloud storage.
      $ {{.HelpName}} --encrypt-key "s3/sql-backups/=32byteslongsecretkeymustbegiven1" s3/sql-backups/1999/old-backup.tgz
`,
}

// Structured message depending on the type of console.
type rmMessage struct {
	Key  string `json:"key"`
	Size int64  `json:"size"`
}

// Colorized message for console printing.
func (r rmMessage) String() string {
	return console.Colorize("Remove", fmt.Sprintf("Removing `%s`.", r.Key))
}

// JSON'ified message for scripting.
func (r rmMessage) JSON() string {
	msgBytes, e := json.MarshalIndent(r, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(msgBytes)
}

// Validate command line arguments.
func checkRmSyntax(ctx *cli.Context, encKeyDB map[string][]prefixSSEPair) {
	// Set command flags from context.
	isForce := ctx.Bool("force")
	isRecursive := ctx.Bool("recursive")
	isStdin := ctx.Bool("stdin")
	isDangerous := ctx.Bool("dangerous")
	isNamespaceRemoval := false

	for _, url := range ctx.Args() {
		// clean path for aliases like s3/.
		//Note: UNC path using / works properly in go 1.9.2 even though it breaks the UNC specification.
		url = filepath.ToSlash(filepath.Clean(url))
		// namespace removal applies only for non FS. So filter out if passed url represents a directory
		if !isAliasURLDir(url, encKeyDB) {
			_, path := url2Alias(url)
			isNamespaceRemoval = (path == "")
			break
		}
	}
	if !ctx.Args().Present() && !isStdin {
		exitCode := 1
		cli.ShowCommandHelpAndExit(ctx, "rm", exitCode)
	}

	// For all recursive operations make sure to check for 'force' flag.
	if (isRecursive || isStdin) && !isForce {
		if isNamespaceRemoval {
			fatalIf(errDummy().Trace(),
				"This operation results in site-wide removal of objects. If you are really sure, retry this command with ‘--dangerous’ and ‘--force’ flags.")
		}
		fatalIf(errDummy().Trace(),
			"Removal requires --force flag. This operation is *IRREVERSIBLE*. Please review carefully before performing this *DANGEROUS* operation.")
	}
	if (isRecursive || isStdin) && isNamespaceRemoval && !isDangerous {
		fatalIf(errDummy().Trace(),
			"This operation results in site-wide removal of objects. If you are really sure, retry this command with ‘--dangerous’ and ‘--force’ flags.")
	}
}

func removeSingle(url string, isIncomplete bool, isFake bool, olderThan, newerThan string, encKeyDB map[string][]prefixSSEPair) error {
	isRecursive := false
	contents, pErr := statURL(url, isIncomplete, isRecursive, encKeyDB)
	if pErr != nil {
		errorIf(pErr.Trace(url), "Failed to remove `"+url+"`.")
		return exitStatus(globalErrorExitStatus)
	}

	if len(contents) == 0 {
		errorIf(errDummy().Trace(url), "Failed to remove `"+url+"`. Target object is not found")
		return exitStatus(globalErrorExitStatus)
	}

	content := contents[0]

	// Skip objects older than older--than parameter if specified
	if olderThan != "" && isOlder(content.Time, olderThan) {
		return nil
	}

	// Skip objects older than older--than parameter if specified
	if newerThan != "" && isNewer(content.Time, newerThan) {
		return nil
	}

	printMsg(rmMessage{
		Key:  url,
		Size: content.Size,
	})

	if !isFake {
		targetAlias, targetURL, _ := mustExpandAlias(url)
		clnt, pErr := newClientFromAlias(targetAlias, targetURL)
		if pErr != nil {
			errorIf(pErr.Trace(url), "Invalid argument `"+url+"`.")
			return exitStatus(globalErrorExitStatus) // End of journey.
		}

		contentCh := make(chan *clientContent, 1)
		contentCh <- &clientContent{URL: *newClientURL(targetURL)}
		close(contentCh)
		isRemoveBucket := false
		errorCh := clnt.Remove(isIncomplete, isRemoveBucket, contentCh)
		for pErr := range errorCh {
			if pErr != nil {
				errorIf(pErr.Trace(url), "Failed to remove `"+url+"`.")
				switch pErr.ToGoError().(type) {
				case PathInsufficientPermission:
					// Ignore Permission error.
					continue
				}
				return exitStatus(globalErrorExitStatus)
			}
		}
	}
	return nil
}

func removeRecursive(url string, isIncomplete bool, isFake bool, olderThan, newerThan string, encKeyDB map[string][]prefixSSEPair) error {
	targetAlias, targetURL, _ := mustExpandAlias(url)
	clnt, pErr := newClientFromAlias(targetAlias, targetURL)
	if pErr != nil {
		errorIf(pErr.Trace(url), "Failed to remove `"+url+"` recursively.")
		return exitStatus(globalErrorExitStatus) // End of journey.
	}
	contentCh := make(chan *clientContent)
	isRemoveBucket := false

	errorCh := clnt.Remove(isIncomplete, isRemoveBucket, contentCh)

	isRecursive := true
	for content := range clnt.List(isRecursive, isIncomplete, DirLast) {
		if content.Err != nil {
			errorIf(content.Err.Trace(url), "Failed to remove `"+url+"` recursively.")
			switch content.Err.ToGoError().(type) {
			case PathInsufficientPermission:
				// Ignore Permission error.
				continue
			}
			close(contentCh)
			return exitStatus(globalErrorExitStatus)
		}
		urlString := content.URL.Path

		if !content.Time.IsZero() {
			// Skip objects older than --older-than parameter if specified
			if olderThan != "" && isOlder(content.Time, olderThan) {
				continue
			}

			// Skip objects newer than --newer-than parameter if specified
			if newerThan != "" && isNewer(content.Time, newerThan) {
				continue
			}
		}

		printMsg(rmMessage{
			Key:  targetAlias + urlString,
			Size: content.Size,
		})

		if !isFake {
			sent := false
			for !sent {
				select {
				case contentCh <- content:
					sent = true
				case pErr := <-errorCh:
					errorIf(pErr.Trace(urlString), "Failed to remove `"+urlString+"`.")
					switch pErr.ToGoError().(type) {
					case PathInsufficientPermission:
						// Ignore Permission error.
						continue
					}
					close(contentCh)
					return exitStatus(globalErrorExitStatus)
				}
			}
		}
	}

	close(contentCh)
	for pErr := range errorCh {
		errorIf(pErr.Trace(url), "Failed to remove `"+url+"` recursively.")
		switch pErr.ToGoError().(type) {
		case PathInsufficientPermission:
			// Ignore Permission error.
			continue
		}
		return exitStatus(globalErrorExitStatus)
	}

	return nil
}

// main for rm command.
func mainRm(ctx *cli.Context) error {
	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(ctx)
	fatalIf(err, "Unable to parse encryption keys.")

	// check 'rm' cli arguments.
	checkRmSyntax(ctx, encKeyDB)

	// rm specific flags.
	isIncomplete := ctx.Bool("incomplete")
	isRecursive := ctx.Bool("recursive")
	isFake := ctx.Bool("fake")
	isStdin := ctx.Bool("stdin")
	olderThan := ctx.String("older-than")
	newerThan := ctx.String("newer-than")

	// Set color.
	console.SetColor("Remove", color.New(color.FgGreen, color.Bold))

	var rerr error
	var e error
	// Support multiple targets.
	for _, url := range ctx.Args() {
		if isRecursive {
			e = removeRecursive(url, isIncomplete, isFake, olderThan, newerThan, encKeyDB)
		} else {
			e = removeSingle(url, isIncomplete, isFake, olderThan, newerThan, encKeyDB)
		}

		if rerr == nil {
			rerr = e
		}
	}

	if !isStdin {
		return rerr
	}

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		url := scanner.Text()
		if isRecursive {
			e = removeRecursive(url, isIncomplete, isFake, olderThan, newerThan, encKeyDB)
		} else {
			e = removeSingle(url, isIncomplete, isFake, olderThan, newerThan, encKeyDB)
		}

		if rerr == nil {
			rerr = e
		}
	}

	return rerr
}
<commit_msg>Skip error upon deletion of non existing object and force flag (#2765)<commit_after>/*
 * MinIO Client (C) 2014, 2015 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
)

// rm specific flags.
var (
	rmFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "remove recursively",
		},
		cli.BoolFlag{
			Name:  "force",
			Usage: "allow a recursive remove operation",
		},
		cli.BoolFlag{
			Name:  "dangerous",
			Usage: "allow site-wide removal of objects",
		},
		cli.BoolFlag{
			Name:  "incomplete, I",
			Usage: "remove incomplete uploads",
		},
		cli.BoolFlag{
			Name:  "fake",
			Usage: "perform a fake remove operation",
		},
		cli.BoolFlag{
			Name:  "stdin",
			Usage: "read object names from STDIN",
		},
		cli.StringFlag{
			Name:  "older-than",
			Usage: "remove objects older than L days, M hours and N minutes",
		},
		cli.StringFlag{
			Name:  "newer-than",
			Usage: "remove objects newer than L days, M hours and N minutes",
		},
	}
)

// remove a file or folder.
var rmCmd = cli.Command{
	Name:   "rm",
	Usage:  "remove objects",
	Action: mainRm,
	Before: setGlobalsFromContext,
	Flags:  append(append(rmFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET [TARGET ...]

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
   MC_ENCRYPT_KEY: list of comma delimited prefix=secret values

EXAMPLES:
   1. Remove a file.
      $ {{.HelpName}} 1999/old-backup.tgz

   2. Remove all objects recursively from bucket 'jazz-songs' matching 'louis' prefix.
      $ {{.HelpName}} --recursive s3/jazz-songs/louis/

   3. Remove all objects older than '90' days recursively from bucket 'jazz-songs' that match 'louis' prefix.
      $ {{.HelpName}} --recursive --older-than 90d s3/jazz-songs/louis/

   4. Remove all objects newer than 7 days and 10 hours recursively from bucket 'pop-songs'
      $ {{.HelpName}} --recursive --newer-than 7d10h s3/pop-songs/

   5. Remove all objects read from STDIN.
      $ {{.HelpName}} --force --stdin

   6. Remove all objects recursively from S3 host
      $ {{.HelpName}} --recursive --dangerous s3

   7. Remove all buckets and objects older than '90' days recursively from host
      $ {{.HelpName}} --recursive --dangerous --older-than 90d s3

   8. Drop all incomplete uploads on 'jazz-songs' bucket.
      $ {{.HelpName}} --incomplete --recursive s3/jazz-songs/

   9. Remove an encrypted object from Amazon S3 cloud storage.
      $ {{.HelpName}} --encrypt-key "s3/sql-backups/=32byteslongsecretkeymustbegiven1" s3/sql-backups/1999/old-backup.tgz
`,
}

// Structured message depending on the type of console.
type rmMessage struct {
	Key  string `json:"key"`
	Size int64  `json:"size"`
}

// Colorized message for console printing.
func (r rmMessage) String() string {
	return console.Colorize("Remove", fmt.Sprintf("Removing `%s`.", r.Key))
}

// JSON'ified message for scripting.
func (r rmMessage) JSON() string {
	msgBytes, e := json.MarshalIndent(r, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(msgBytes)
}

// Validate command line arguments.
func checkRmSyntax(ctx *cli.Context, encKeyDB map[string][]prefixSSEPair) {
	// Set command flags from context.
	isForce := ctx.Bool("force")
	isRecursive := ctx.Bool("recursive")
	isStdin := ctx.Bool("stdin")
	isDangerous := ctx.Bool("dangerous")
	isNamespaceRemoval := false

	for _, url := range ctx.Args() {
		// clean path for aliases like s3/.
		//Note: UNC path using / works properly in go 1.9.2 even though it breaks the UNC specification.
		url = filepath.ToSlash(filepath.Clean(url))
		// namespace removal applies only for non FS. So filter out if passed url represents a directory
		if !isAliasURLDir(url, encKeyDB) {
			_, path := url2Alias(url)
			isNamespaceRemoval = (path == "")
			break
		}
	}
	if !ctx.Args().Present() && !isStdin {
		exitCode := 1
		cli.ShowCommandHelpAndExit(ctx, "rm", exitCode)
	}

	// For all recursive operations make sure to check for 'force' flag.
	if (isRecursive || isStdin) && !isForce {
		if isNamespaceRemoval {
			fatalIf(errDummy().Trace(),
				"This operation results in site-wide removal of objects. If you are really sure, retry this command with ‘--dangerous’ and ‘--force’ flags.")
		}
		fatalIf(errDummy().Trace(),
			"Removal requires --force flag. This operation is *IRREVERSIBLE*. Please review carefully before performing this *DANGEROUS* operation.")
	}
	if (isRecursive || isStdin) && isNamespaceRemoval && !isDangerous {
		fatalIf(errDummy().Trace(),
			"This operation results in site-wide removal of objects. If you are really sure, retry this command with ‘--dangerous’ and ‘--force’ flags.")
	}
}

func removeSingle(url string, isIncomplete bool, isFake, isForce bool, olderThan, newerThan string, encKeyDB map[string][]prefixSSEPair) error {
	isRecursive := false
	contents, pErr := statURL(url, isIncomplete, isRecursive, encKeyDB)
	if pErr != nil {
		errorIf(pErr.Trace(url), "Failed to remove `"+url+"`.")
		return exitStatus(globalErrorExitStatus)
	}
	if len(contents) == 0 {
		if !isForce {
			errorIf(errDummy().Trace(url), "Failed to remove `"+url+"`. Target object is not found")
			return exitStatus(globalErrorExitStatus)
		}
		return nil
	}

	content := contents[0]

	// Skip objects older than older--than parameter if specified
	if olderThan != "" && isOlder(content.Time, olderThan) {
		return nil
	}

	// Skip objects older than older--than parameter if specified
	if newerThan != "" && isNewer(content.Time, newerThan) {
		return nil
	}

	printMsg(rmMessage{
		Key:  url,
		Size: content.Size,
	})

	if !isFake {
		targetAlias, targetURL, _ := mustExpandAlias(url)
		clnt, pErr := newClientFromAlias(targetAlias, targetURL)
		if pErr != nil {
			errorIf(pErr.Trace(url), "Invalid argument `"+url+"`.")
			return exitStatus(globalErrorExitStatus) // End of journey.
		}

		contentCh := make(chan *clientContent, 1)
		contentCh <- &clientContent{URL: *newClientURL(targetURL)}
		close(contentCh)
		isRemoveBucket := false
		errorCh := clnt.Remove(isIncomplete, isRemoveBucket, contentCh)
		for pErr := range errorCh {
			if pErr != nil {
				errorIf(pErr.Trace(url), "Failed to remove `"+url+"`.")
				switch pErr.ToGoError().(type) {
				case PathInsufficientPermission:
					// Ignore Permission error.
					continue
				}
				return exitStatus(globalErrorExitStatus)
			}
		}
	}
	return nil
}

func removeRecursive(url string, isIncomplete bool, isFake bool, olderThan, newerThan string, encKeyDB map[string][]prefixSSEPair) error {
	targetAlias, targetURL, _ := mustExpandAlias(url)
	clnt, pErr := newClientFromAlias(targetAlias, targetURL)
	if pErr != nil {
		errorIf(pErr.Trace(url), "Failed to remove `"+url+"` recursively.")
		return exitStatus(globalErrorExitStatus) // End of journey.
	}
	contentCh := make(chan *clientContent)
	isRemoveBucket := false

	errorCh := clnt.Remove(isIncomplete, isRemoveBucket, contentCh)

	isRecursive := true
	for content := range clnt.List(isRecursive, isIncomplete, DirLast) {
		if content.Err != nil {
			errorIf(content.Err.Trace(url), "Failed to remove `"+url+"` recursively.")
			switch content.Err.ToGoError().(type) {
			case PathInsufficientPermission:
				// Ignore Permission error.
				continue
			}
			close(contentCh)
			return exitStatus(globalErrorExitStatus)
		}
		urlString := content.URL.Path

		if !content.Time.IsZero() {
			// Skip objects older than --older-than parameter if specified
			if olderThan != "" && isOlder(content.Time, olderThan) {
				continue
			}

			// Skip objects newer than --newer-than parameter if specified
			if newerThan != "" && isNewer(content.Time, newerThan) {
				continue
			}
		}

		printMsg(rmMessage{
			Key:  targetAlias + urlString,
			Size: content.Size,
		})

		if !isFake {
			sent := false
			for !sent {
				select {
				case contentCh <- content:
					sent = true
				case pErr := <-errorCh:
					errorIf(pErr.Trace(urlString), "Failed to remove `"+urlString+"`.")
					switch pErr.ToGoError().(type) {
					case PathInsufficientPermission:
						// Ignore Permission error.
						continue
					}
					close(contentCh)
					return exitStatus(globalErrorExitStatus)
				}
			}
		}
	}

	close(contentCh)
	for pErr := range errorCh {
		errorIf(pErr.Trace(url), "Failed to remove `"+url+"` recursively.")
		switch pErr.ToGoError().(type) {
		case PathInsufficientPermission:
			// Ignore Permission error.
			continue
		}
		return exitStatus(globalErrorExitStatus)
	}

	return nil
}

// main for rm command.
func mainRm(ctx *cli.Context) error {
	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(ctx)
	fatalIf(err, "Unable to parse encryption keys.")

	// check 'rm' cli arguments.
	checkRmSyntax(ctx, encKeyDB)

	// rm specific flags.
	isIncomplete := ctx.Bool("incomplete")
	isRecursive := ctx.Bool("recursive")
	isFake := ctx.Bool("fake")
	isStdin := ctx.Bool("stdin")
	olderThan := ctx.String("older-than")
	newerThan := ctx.String("newer-than")
	isForce := ctx.Bool("force")

	// Set color.
	console.SetColor("Remove", color.New(color.FgGreen, color.Bold))

	var rerr error
	var e error
	// Support multiple targets.
	for _, url := range ctx.Args() {
		if isRecursive {
			e = removeRecursive(url, isIncomplete, isFake, olderThan, newerThan, encKeyDB)
		} else {
			e = removeSingle(url, isIncomplete, isFake, isForce, olderThan, newerThan, encKeyDB)
		}

		if rerr == nil {
			rerr = e
		}
	}

	if !isStdin {
		return rerr
	}

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		url := scanner.Text()
		if isRecursive {
			e = removeRecursive(url, isIncomplete, isFake, olderThan, newerThan, encKeyDB)
		} else {
			e = removeSingle(url, isIncomplete, isFake, isForce, olderThan, newerThan, encKeyDB)
		}

		if rerr == nil {
			rerr = e
		}
	}

	return rerr
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"path/filepath"

	"github.com/posener/complete"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/grafana/tanka/pkg/cmp"
	"github.com/grafana/tanka/pkg/config/v1alpha1"
	"github.com/grafana/tanka/pkg/kubernetes"
)

// Version is the current version of the tk command.
// To be overwritten at build time
var Version = "dev"

var (
	config = &v1alpha1.Config{}
	kube   *kubernetes.Kubernetes
)

// list of deprecated config keys and their alternatives
// however, they still work and are aliased internally
var deprecated = map[string]string{
	"namespace": "spec.namespace",
	"server":    "spec.apiServer",
	"team":      "metadata.labels.team",
}

func main() {
	rootCmd := &cobra.Command{
		Use:              "tk",
		Short:            "tanka <3 jsonnet",
		Version:          Version,
		TraverseChildren: true,
		// Configuration
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if len(args) == 0 {
				return
			}
			config = setupConfiguration(args[0])
			if config == nil {
				return
			}

			// Kubernetes
			kube = kubernetes.New(config.Spec)

		},
	}
	rootCmd.PersistentFlags().BoolP("verbose", "v", false, "")

	// Subcommands
	cobra.EnableCommandSorting = false

	// workflow commands
	rootCmd.AddCommand(
		applyCmd(),
		showCmd(),
		diffCmd(),
	)

	// jsonnet commands
	rootCmd.AddCommand(
		evalCmd(),
		initCmd(),
		toolCmd(),
	)

	// completion
	cmp.Handlers.Add("baseDir", complete.PredictFunc(
		func(complete.Args) []string {
			return findBaseDirs()
		},
	))

	c := complete.New("tk", cmp.Create(rootCmd))
	c.InstallName = "install-completion"
	c.UninstallName = "uninstall-completion"
	fs := &flag.FlagSet{}
	c.AddFlags(fs)
	rootCmd.Flags().AddGoFlagSet(fs)

	rootCmd.Run = func(cmd *cobra.Command, args []string) {
		if c.Complete() {
			return
		}
		_ = cmd.Help()
	}

	// Run!
	if err := rootCmd.Execute(); err != nil {
		log.Fatalln("Ouch:", rootCmd.Execute())
	}
}

func setupConfiguration(baseDir string) *v1alpha1.Config {
	viper.SetConfigName("spec")

	// if the baseDir arg is not a dir, abort
	pwd, err := filepath.Abs(baseDir)
	if err != nil {
		return nil
	}
	viper.AddConfigPath(pwd)

	// handle deprecated ksonnet spec
	for old, new := range deprecated {
		viper.RegisterAlias(new, old)
	}

	// read it
	if err := viper.ReadInConfig(); err != nil {
		// just run fine without config. Provider features won't work (apply, show, diff)
		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
			return nil
		}

		log.Fatalln(err)
	}
	checkDeprecated()

	var config v1alpha1.Config
	if err := viper.Unmarshal(&config); err != nil {
		log.Fatalln(err)
	}
	return &config
}

func checkDeprecated() {
	for old, use := range deprecated {
		if viper.IsSet(old) {
			log.Printf("Warning: `%s` is deprecated, use `%s` instead.", old, use)
		}
	}
}
<commit_msg>fix(cli): remove datetime from log (#24)<commit_after>package main

import (
	"flag"
	"log"
	"path/filepath"

	"github.com/posener/complete"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/grafana/tanka/pkg/cmp"
	"github.com/grafana/tanka/pkg/config/v1alpha1"
	"github.com/grafana/tanka/pkg/kubernetes"
)

// Version is the current version of the tk command.
// To be overwritten at build time
var Version = "dev"

var (
	config = &v1alpha1.Config{}
	kube   *kubernetes.Kubernetes
)

// list of deprecated config keys and their alternatives
// however, they still work and are aliased internally
var deprecated = map[string]string{
	"namespace": "spec.namespace",
	"server":    "spec.apiServer",
	"team":      "metadata.labels.team",
}

func main() {
	log.SetFlags(0)
	rootCmd := &cobra.Command{
		Use:              "tk",
		Short:            "tanka <3 jsonnet",
		Version:          Version,
		TraverseChildren: true,
		// Configuration
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if len(args) == 0 {
				return
			}
			config = setupConfiguration(args[0])
			if config == nil {
				return
			}

			// Kubernetes
			kube = kubernetes.New(config.Spec)

		},
	}
	rootCmd.PersistentFlags().BoolP("verbose", "v", false, "")

	// Subcommands
	cobra.EnableCommandSorting = false

	// workflow commands
	rootCmd.AddCommand(
		applyCmd(),
		showCmd(),
		diffCmd(),
	)

	// jsonnet commands
	rootCmd.AddCommand(
		evalCmd(),
		initCmd(),
		toolCmd(),
	)

	// completion
	cmp.Handlers.Add("baseDir", complete.PredictFunc(
		func(complete.Args) []string {
			return findBaseDirs()
		},
	))

	c := complete.New("tk", cmp.Create(rootCmd))
	c.InstallName = "install-completion"
	c.UninstallName = "uninstall-completion"
	fs := &flag.FlagSet{}
	c.AddFlags(fs)
	rootCmd.Flags().AddGoFlagSet(fs)

	rootCmd.Run = func(cmd *cobra.Command, args []string) {
		if c.Complete() {
			return
		}
		_ = cmd.Help()
	}

	// Run!
	if err := rootCmd.Execute(); err != nil {
		log.Fatalln("Ouch:", rootCmd.Execute())
	}
}

func setupConfiguration(baseDir string) *v1alpha1.Config {
	viper.SetConfigName("spec")

	// if the baseDir arg is not a dir, abort
	pwd, err := filepath.Abs(baseDir)
	if err != nil {
		return nil
	}
	viper.AddConfigPath(pwd)

	// handle deprecated ksonnet spec
	for old, new := range deprecated {
		viper.RegisterAlias(new, old)
	}

	// read it
	if err := viper.ReadInConfig(); err != nil {
		// just run fine without config. Provider features won't work (apply, show, diff)
		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
			return nil
		}

		log.Fatalln(err)
	}
	checkDeprecated()

	var config v1alpha1.Config
	if err := viper.Unmarshal(&config); err != nil {
		log.Fatalln(err)
	}
	return &config
}

func checkDeprecated() {
	for old, use := range deprecated {
		if viper.IsSet(old) {
			log.Printf("Warning: `%s` is deprecated, use `%s` instead.", old, use)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import "github.com/spf13/cobra"

func init() {
	RootCmd.AddCommand(versionCmd)
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Displays the version of mbt",
	Run: func(cmd *cobra.Command, args []string) {
		println("0.1.5")
	},
}
<commit_msg>Bump version<commit_after>package cmd

import "github.com/spf13/cobra"

func init() {
	RootCmd.AddCommand(versionCmd)
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Displays the version of mbt",
	Run: func(cmd *cobra.Command, args []string) {
		println("0.1.6")
	},
}
<|endoftext|>


<commit_before>package main

var Version string = "0.1.0"
<commit_msg>Bump version 0.2.0<commit_after>package main

var Version string = "0.2.0"
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"github.com/luke-chisholm6/go-cli-templates/readers"
	"github.com/luke-chisholm6/go-cli-templates/writers"
	"io"
	"reflect"
	"strings"
	"testing"
)

func TestGetTemplateContext_SliceWithInvalidStrings(t *testing.T) {
	kvSlice := []string{
		"test",
	}

	if _, err := getTemplateContext(kvSlice); err == nil {
		t.Error("Cannot split a string into a kv pair that is not in the format of k=v")
	}
}

func TestGetTemplateContext_SliceWithValidStrings(t *testing.T) {
	kvSlice := []string{
		"test=test",
		"key=value",
	}

	kvMap, err := getTemplateContext(kvSlice)
	if err != nil {
		t.Error("Cannot split a string into a kv pair that is not in the format of k=v")
	}

	kvMapComparison := map[string]string{
		"test": "test",
		"key":  "value",
	}
	if !reflect.DeepEqual(kvMap, kvMapComparison) {
		t.Errorf("Expected: %+v\nGot: %+v", kvMapComparison, kvMap)
	}

}

func TestCompileTemplate_invalid(t *testing.T) {
	templateString := "a template {{nonexistentfunction}}"
	if _, err := compileTemplate(strings.NewReader(templateString)); err == nil {
		t.Errorf("\"%v\" is an invalid template", templateString)
	}

	templateString = ""
	if _, err := compileTemplate(strings.NewReader(templateString)); err == nil {
		t.Errorf("\"%v\" is an invalid template", templateString)
	}

	alwaysErrReader := readers.NewErrorReader()
	if _, err := compileTemplate(alwaysErrReader); err == nil {
		t.Errorf("%v should always error", alwaysErrReader)
	}
}

func TestCompileTemplate_valid(t *testing.T) {
	templateString := "legit template {{.test}}"
	if _, err := compileTemplate(strings.NewReader(templateString)); err != nil {
		t.Errorf("\"%v\" is a valid template", templateString)
	}
}

func TestRender(t *testing.T) {
	templateString := "legit template {{.test}} {{.key}}"
	compiledTemplate, err := compileTemplate(strings.NewReader(templateString))
	if err != nil {
		t.Errorf("\"%v\" is a valid template", templateString)
	}

	context := map[string]string{
		"test": "test",
		"key":  "value",
	}

	writer := new(bytes.Buffer)
	err = render(compiledTemplate, context, writer)
	expected := "legit template test value"
	if err != nil {
		t.Errorf("Expected: %v, Got: %v", expected, err)
	}
	if got := writer.String(); expected != got {
		t.Errorf("Expected: %v, Got: %v", expected, got)
	}
}

func TestRun(t *testing.T) {
	input := strings.NewReader("{{.test}} {{.key}}")
	context := []string{
		"test=Hello",
		"key=world!",
	}
	writer := new(bytes.Buffer)
	run(input, context, writer)

	expected := "Hello world!"
	got := writer.String()
	if got != expected {
		t.Errorf("Expected: %v, Got: %v", expected, got)
	}
}

func TestRun_Invalid(t *testing.T) {
	templateString := "a template {{nonexistentfunction}}"
	var writer io.Writer = new(bytes.Buffer)
	context := []string{
		"test",
		"key=world!",
	}
	if err := run(strings.NewReader(templateString), context, writer); err == nil {
		t.Errorf("\"%v\" is an invalid template", templateString)
	}

	templateString = "a legit template {{.key}}"
	if err := run(strings.NewReader(templateString), context, writer); err == nil {
		t.Errorf("\"%v\" is invalid context", context)
	}

	context = []string{
		"test=success",
	}
	writer = writers.NewErrorWriter()
	if err := run(strings.NewReader(templateString), context, writer); err == nil {
		t.Errorf("\"%v\" is invalid context", context)
	}
}
<commit_msg>clean up the test suite a little bit<commit_after>package main

import (
	"bytes"
	"github.com/luke-chisholm6/go-cli-templates/readers"
	"github.com/luke-chisholm6/go-cli-templates/writers"
	"io"
	"reflect"
	"strings"
	"testing"
)

const (
	InvalidTemplateErrorString          = "\"%v\" is an invalid template"
	ValidTemplateErrorString            = "\"%v\" is a valid template"
	UnexpectedResultErrorString         = "Expected: %+v\nGot: %+v"
	InvalidTemplate_NonexistentFunction = "a template {{nonexistentfunction}}"
)

type TemplateTest struct {
	templateString string
	context        map[string]string
	contextRaw     []string
	expected       string
}

func NewLegitimateTemplateTest() *TemplateTest {
	return &TemplateTest{
		templateString: "legit template {{.test}} {{.key}}",
		context: map[string]string{
			"test": "test",
			"key":  "value",
		},
		contextRaw: []string{
			"test=test",
			"key=value",
		},
		expected: "legit template test value",
	}
}

func TestGetTemplateContext_SliceWithInvalidStrings(t *testing.T) {
	kvSlice := []string{
		"test",
	}

	if _, err := getTemplateContext(kvSlice); err == nil {
		t.Error("Cannot split a string into a kv pair that is not in the format of k=v")
	}
}

func TestGetTemplateContext_SliceWithValidStrings(t *testing.T) {
	templateTest := NewLegitimateTemplateTest()

	kvMap, err := getTemplateContext(templateTest.contextRaw)
	if err != nil {
		t.Error(err)
	}

	if !reflect.DeepEqual(kvMap, templateTest.context) {
		t.Errorf(UnexpectedResultErrorString, templateTest.context, kvMap)
	}
}

func TestCompileTemplate_invalid(t *testing.T) {
	templateString := InvalidTemplate_NonexistentFunction
	if _, err := compileTemplate(strings.NewReader(templateString)); err == nil {
		t.Errorf(InvalidTemplateErrorString, templateString)
	}

	templateString = ""
	if _, err := compileTemplate(strings.NewReader(templateString)); err == nil {
		t.Errorf(InvalidTemplateErrorString, templateString)
	}

	alwaysErrReader := readers.NewErrorReader()
	if _, err := compileTemplate(alwaysErrReader); err == nil {
		t.Errorf("%v should always error", alwaysErrReader)
	}
}

func TestCompileTemplate_valid(t *testing.T) {
	templateTest := NewLegitimateTemplateTest()
	if _, err := compileTemplate(strings.NewReader(templateTest.templateString)); err != nil {
		t.Errorf(ValidTemplateErrorString, templateTest.templateString)
	}
}

func TestRender(t *testing.T) {
	templateTest := NewLegitimateTemplateTest()
	compiledTemplate, err := compileTemplate(strings.NewReader(templateTest.templateString))
	if err != nil {
		t.Errorf(ValidTemplateErrorString, templateTest.templateString)
	}

	writer := new(bytes.Buffer)
	err = render(compiledTemplate, templateTest.context, writer)
	if err != nil {
		t.Errorf(UnexpectedResultErrorString, templateTest.expected, err)
	}
	if got := writer.String(); templateTest.expected != got {
		t.Errorf(UnexpectedResultErrorString, templateTest.expected, got)
	}
}

func TestRun(t *testing.T) {
	templateTest := NewLegitimateTemplateTest()
	input := strings.NewReader(templateTest.templateString)
	writer := new(bytes.Buffer)
	run(input, templateTest.contextRaw, writer)

	got := writer.String()
	if got != templateTest.expected {
		t.Errorf(UnexpectedResultErrorString, templateTest.expected, got)
	}
}

func TestRun_Invalid(t *testing.T) {
	var writer io.Writer = new(bytes.Buffer)
	context := []string{
		"test",
		"key=world!",
	}
	if err := run(strings.NewReader(InvalidTemplate_NonexistentFunction), context, writer); err == nil {
		t.Errorf("\"%v\" is an invalid template", InvalidTemplate_NonexistentFunction)
	}

	templateTest := NewLegitimateTemplateTest()
	if err := run(strings.NewReader(templateTest.templateString), context, writer); err == nil {
		t.Errorf("\"%v\" is invalid context", context)
	}

	writer = writers.NewErrorWriter()
	if err := run(strings.NewReader(templateTest.templateString), templateTest.contextRaw, writer); err == nil {
		t.Errorf("\"%+v\" should always error", writer)
	}
}
<|endoftext|>


<commit_before>package ttransport

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"sync"
	"testing"

	"github.com/libp2p/go-libp2p-core/mux"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"

	ma "github.com/multiformats/go-multiaddr"
)

var testData = []byte("this is some test data")

type streamAndConn struct {
	stream mux.MuxedStream
	conn   transport.CapableConn
}

func SubtestProtocols(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	rawIPAddr, _ := ma.NewMultiaddr("/ip4/1.2.3.4")
	if ta.CanDial(rawIPAddr) || tb.CanDial(rawIPAddr) {
		t.Error("nothing should be able to dial raw IP")
	}

	tprotos := make(map[int]bool)
	for _, p := range ta.Protocols() {
		tprotos[p] = true
	}

	if !ta.Proxy() {
		protos := maddr.Protocols()
		proto := protos[len(protos)-1]
		if !tprotos[proto.Code] {
			t.Errorf("transport should have reported that it supports protocol '%s' (%d)", proto.Name, proto.Code)
		}
	} else {
		found := false
		for _, proto := range maddr.Protocols() {
			if tprotos[proto.Code] {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("didn't find any matching proxy protocols in maddr: %s", maddr)
		}
	}
}

func SubtestBasic(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	list, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer list.Close()

	var (
		connA, connB transport.CapableConn
		done         = make(chan struct{})
	)
	defer func() {
		<-done
		if connA != nil {
			connA.Close()
		}
		if connB != nil {
			connB.Close()
		}
	}()

	go func() {
		defer close(done)
		var err error
		connB, err = list.Accept()
		if err != nil {
			t.Error(err)
			return
		}
		s, err := connB.AcceptStream()
		if err != nil {
			t.Error(err)
			return
		}

		buf, err := ioutil.ReadAll(s)
		if err != nil {
			t.Error(err)
			return
		}

		if !bytes.Equal(testData, buf) {
			t.Errorf("expected %s, got %s", testData, buf)
		}

		n, err := s.Write(testData)
		if err != nil {
			t.Error(err)
			return
		}
		if n != len(testData) {
			t.Error(err)
			return
		}

		err = s.Close()
		if err != nil {
			t.Error(err)
		}
	}()

	if !tb.CanDial(list.Multiaddr()) {
		t.Error("CanDial should have returned true")
	}

	connA, err = tb.Dial(ctx, list.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}

	s, err := connA.OpenStream()
	if err != nil {
		t.Fatal(err)
	}

	n, err := s.Write(testData)
	if err != nil {
		t.Fatal(err)
		return
	}

	if n != len(testData) {
		t.Fatalf("failed to write enough data (a->b)")
		return
	}
	err = s.Close()
	if err != nil {
		t.Fatal(err)
		return
	}

	buf, err := ioutil.ReadAll(s)
	if err != nil {
		t.Fatal(err)
		return
	}
	if !bytes.Equal(testData, buf) {
		t.Errorf("expected %s, got %s", testData, buf)
	}
}

func SubtestPingPong(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	streams := 100

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	list, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer list.Close()

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		c, err := list.Accept()
		if err != nil {
			t.Error(err)
			return
		}
		defer c.Close()

		var sWg sync.WaitGroup
		for i := 0; i < streams; i++ {
			s, err := c.AcceptStream()
			if err != nil {
				t.Error(err)
				return
			}

			sWg.Add(1)
			go func() {
				defer sWg.Done()

				data, err := ioutil.ReadAll(s)
				if err != nil {
					s.Reset()
					t.Error(err)
					return
				}
				if !bytes.HasPrefix(data, testData) {
					t.Errorf("expected %q to have prefix %q", string(data), string(testData))
				}

				n, err := s.Write(data)
				if err != nil {
					s.Reset()
					t.Error(err)
					return
				}

				if n != len(data) {
					s.Reset()
					t.Error(err)
					return
				}
				s.Close()
			}()
		}
		sWg.Wait()
	}()

	if !tb.CanDial(list.Multiaddr()) {
		t.Error("CanDial should have returned true")
	}

	c, err := tb.Dial(ctx, list.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	for i := 0; i < streams; i++ {
		s, err := c.OpenStream()
		if err != nil {
			t.Error(err)
			continue
		}

		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			data := []byte(fmt.Sprintf("%s - %d", testData, i))
			n, err := s.Write(data)
			if err != nil {
				s.Reset()
				t.Error(err)
				return
			}

			if n != len(data) {
				s.Reset()
				t.Error("failed to write enough data (a->b)")
				return
			}
			s.Close()

			ret, err := ioutil.ReadAll(s)
			if err != nil {
				s.Reset()
				t.Error(err)
				return
			}
			if !bytes.Equal(data, ret) {
				t.Errorf("expected %q, got %q", string(data), string(ret))
			}
		}(i)
	}
	wg.Wait()
}

func SubtestCancel(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	list, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer list.Close()

	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	c, err := tb.Dial(ctx, list.Multiaddr(), peerA)
	if err == nil {
		c.Close()
		t.Fatal("dial should have failed")
	}
}
<commit_msg>SubtestPingPong: ensure connections are closed.<commit_after>package ttransport

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"sync"
	"testing"

	"github.com/libp2p/go-libp2p-core/mux"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"

	ma "github.com/multiformats/go-multiaddr"
)

var testData = []byte("this is some test data")

type streamAndConn struct {
	stream mux.MuxedStream
	conn   transport.CapableConn
}

func SubtestProtocols(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	rawIPAddr, _ := ma.NewMultiaddr("/ip4/1.2.3.4")
	if ta.CanDial(rawIPAddr) || tb.CanDial(rawIPAddr) {
		t.Error("nothing should be able to dial raw IP")
	}

	tprotos := make(map[int]bool)
	for _, p := range ta.Protocols() {
		tprotos[p] = true
	}

	if !ta.Proxy() {
		protos := maddr.Protocols()
		proto := protos[len(protos)-1]
		if !tprotos[proto.Code] {
			t.Errorf("transport should have reported that it supports protocol '%s' (%d)", proto.Name, proto.Code)
		}
	} else {
		found := false
		for _, proto := range maddr.Protocols() {
			if tprotos[proto.Code] {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("didn't find any matching proxy protocols in maddr: %s", maddr)
		}
	}
}

func SubtestBasic(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	list, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer list.Close()

	var (
		connA, connB transport.CapableConn
		done         = make(chan struct{})
	)
	defer func() {
		<-done
		if connA != nil {
			connA.Close()
		}
		if connB != nil {
			connB.Close()
		}
	}()

	go func() {
		defer close(done)
		var err error
		connB, err = list.Accept()
		if err != nil {
			t.Error(err)
			return
		}
		s, err := connB.AcceptStream()
		if err != nil {
			t.Error(err)
			return
		}

		buf, err := ioutil.ReadAll(s)
		if err != nil {
			t.Error(err)
			return
		}

		if !bytes.Equal(testData, buf) {
			t.Errorf("expected %s, got %s", testData, buf)
		}

		n, err := s.Write(testData)
		if err != nil {
			t.Error(err)
			return
		}
		if n != len(testData) {
			t.Error(err)
			return
		}

		err = s.Close()
		if err != nil {
			t.Error(err)
		}
	}()

	if !tb.CanDial(list.Multiaddr()) {
		t.Error("CanDial should have returned true")
	}

	connA, err = tb.Dial(ctx, list.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}

	s, err := connA.OpenStream()
	if err != nil {
		t.Fatal(err)
	}

	n, err := s.Write(testData)
	if err != nil {
		t.Fatal(err)
		return
	}

	if n != len(testData) {
		t.Fatalf("failed to write enough data (a->b)")
		return
	}
	err = s.Close()
	if err != nil {
		t.Fatal(err)
		return
	}

	buf, err := ioutil.ReadAll(s)
	if err != nil {
		t.Fatal(err)
		return
	}
	if !bytes.Equal(testData, buf) {
		t.Errorf("expected %s, got %s", testData, buf)
	}
}

func SubtestPingPong(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	streams := 100

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	list, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer list.Close()

	var (
		connA, connB transport.CapableConn
	)
	defer func() {
		if connA != nil {
			connA.Close()
		}
		if connB != nil {
			connB.Close()
		}
	}()

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		var err error
		connA, err = list.Accept()
		if err != nil {
			t.Error(err)
			return
		}

		var sWg sync.WaitGroup
		for i := 0; i < streams; i++ {
			s, err := connA.AcceptStream()
			if err != nil {
				t.Error(err)
				return
			}

			sWg.Add(1)
			go func() {
				defer sWg.Done()

				data, err := ioutil.ReadAll(s)
				if err != nil {
					s.Reset()
					t.Error(err)
					return
				}
				if !bytes.HasPrefix(data, testData) {
					t.Errorf("expected %q to have prefix %q", string(data), string(testData))
				}

				n, err := s.Write(data)
				if err != nil {
					s.Reset()
					t.Error(err)
					return
				}

				if n != len(data) {
					s.Reset()
					t.Error(err)
					return
				}
				s.Close()
			}()
		}
		sWg.Wait()
	}()

	if !tb.CanDial(list.Multiaddr()) {
		t.Error("CanDial should have returned true")
	}

	connB, err = tb.Dial(ctx, list.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < streams; i++ {
		s, err := connB.OpenStream()
		if err != nil {
			t.Error(err)
			continue
		}

		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			data := []byte(fmt.Sprintf("%s - %d", testData, i))
			n, err := s.Write(data)
			if err != nil {
				s.Reset()
				t.Error(err)
				return
			}

			if n != len(data) {
				s.Reset()
				t.Error("failed to write enough data (a->b)")
				return
			}
			s.Close()

			ret, err := ioutil.ReadAll(s)
			if err != nil {
				s.Reset()
				t.Error(err)
				return
			}
			if !bytes.Equal(data, ret) {
				t.Errorf("expected %q, got %q", string(data), string(ret))
			}
		}(i)
	}
	wg.Wait()
}

func SubtestCancel(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	list, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer list.Close()

	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	c, err := tb.Dial(ctx, list.Multiaddr(), peerA)
	if err == nil {
		c.Close()
		t.Fatal("dial should have failed")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Alexander Eichhorn
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	_ "github.com/echa/go-xmp/models"
	"github.com/echa/go-xmp/models/dc"
	"github.com/echa/go-xmp/models/xmp_base"
	"github.com/echa/go-xmp/xmp"
)

var (
	debug bool
	quiet bool
	fjson bool
	fxmp  bool
	fpath bool
	forig bool
)

func init() {
	flag.BoolVar(&debug, "debug", false, "enable debugging")
	flag.BoolVar(&quiet, "quiet", false, "don't output anything")
	flag.BoolVar(&fjson, "json", false, "enable JSON output")
	flag.BoolVar(&fxmp, "xmp", false, "enable XMP output")
	flag.BoolVar(&fpath, "path", false, "enable XMP/Path output")
	flag.BoolVar(&forig, "orig", false, "enable original XMP output")
}

func fail(v interface{}) {
	fmt.Printf("Error: %s in file %s\n", v, flag.Arg(0))
	os.Exit(1)
}

func out(b []byte) {
	if quiet {
		return
	}
	fmt.Println(string(b))
}

func marshal(d *xmp.Document) []byte {
	b, err := xmp.MarshalIndent(d, "", "  ")
	if err != nil {
		fail(err)
	}
	return b
}

func unmarshal(v []byte) *xmp.Document {
	d := &xmp.Document{}
	if err := xmp.Unmarshal(v, d); err != nil {
		fail(err)
	}
	return d
}

func main() {
	flag.Parse()

	if debug {
		xmp.SetLogLevel(xmp.LogLevelDebug)
	}

	// output original when no option is selected
	if !fjson && !fxmp && !fpath && !forig && !quiet {
		forig = true
	}

	var b []byte

	if flag.NArg() > 0 {
		filename := flag.Arg(0)
		f, err := os.Open(filename)
		if err != nil {
			fail(err)
		}
		defer f.Close()

		switch filepath.Ext(filename) {
		case ".xmp":
			b, err = ioutil.ReadAll(f)
			if err != nil {
				fail(err)
			}

		default:
			bb, err := xmp.ScanPackets(f)
			if err != nil && err != io.EOF {
				fail(err)
			}
			if err == io.EOF {
				return
			}
			b = bb[0]
		}

	} else {

		// fill the document with some info
		s := xmp.NewDocument()
		s.AddModel(&xmpbase.XmpBase{
			CreatorTool: xmp.Agent,
			CreateDate:  xmp.Now(),
			ModifyDate:  xmp.Now(),
			Thumbnails: xmpbase.ThumbnailArray{
				xmpbase.Thumbnail{
					Format: "image/jpeg",
					Width:  10,
					Height: 10,
					Image:  []byte("not a real image"),
				},
			},
		})
		s.AddModel(&dc.DublinCore{
			Format:  "image/jpeg",
			Title:   xmp.NewAltString("demo"),
			Creator: xmp.NewStringList("Alexander Eichhorn"),
			Description: xmp.NewAltString(
				xmp.AltItem{Value: "Go-XMP Demo Model", Lang: "en", IsDefault: true},
				xmp.AltItem{Value: "Go-XMP Beispiel Modell", Lang: "de", IsDefault: false},
			),
		})
		b = marshal(s)
		s.Close()
	}

	if forig {
		out(b)
	}

	model := unmarshal(b)

	if fjson {
		b, err := json.MarshalIndent(model, "", "  ")
		if err != nil {
			fail(err)
		}
		out(b)
	}

	if fxmp {
		out(marshal(model))
	}

	if fpath {
		l, err := model.ListPaths()
		if err != nil {
			fail(err)
		}
		for _, v := range l {
			fmt.Printf("%s = %s\n", v.Path.String(), v.Value)
		}
	}

	model.Close()

	if debug {
		xmp.DumpStats()
	}
}
<commit_msg>add all param<commit_after>// Copyright (c) 2017 Alexander Eichhorn
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	_ "github.com/echa/go-xmp/models"
	"github.com/echa/go-xmp/models/dc"
	"github.com/echa/go-xmp/models/xmp_base"
	"github.com/echa/go-xmp/xmp"
)

var (
	debug bool
	quiet bool
	fjson bool
	fxmp  bool
	fpath bool
	forig bool
	fall  bool
)

func init() {
	flag.BoolVar(&debug, "debug", false, "enable debugging")
	flag.BoolVar(&quiet, "quiet", false, "don't output anything")
	flag.BoolVar(&fjson, "json", false, "enable JSON output")
	flag.BoolVar(&fxmp, "xmp", false, "enable XMP output")
	flag.BoolVar(&fpath, "path", false, "enable XMP/Path output")
	flag.BoolVar(&forig, "orig", false, "enable original XMP output")
	flag.BoolVar(&fall, "all", false, "ouput all embedded xmp documents")
}

func fail(v interface{}) {
	fmt.Printf("Error: %s in file %s\n", v, flag.Arg(0))
	os.Exit(1)
}

func out(b []byte) {
	if quiet {
		return
	}
	fmt.Println(string(b))
}

func marshal(d *xmp.Document) []byte {
	b, err := xmp.MarshalIndent(d, "", "  ")
	if err != nil {
		fail(err)
	}
	return b
}

func unmarshal(v []byte) *xmp.Document {
	d := &xmp.Document{}
	if err := xmp.Unmarshal(v, d); err != nil {
		fail(err)
	}
	return d
}

func main() {
	flag.Parse()

	if debug {
		xmp.SetLogLevel(xmp.LogLevelDebug)
	}

	// output original when no option is selected
	if !fjson && !fxmp && !fpath && !forig && !quiet {
		forig = true
	}

	var b []byte

	if flag.NArg() > 0 {
		filename := flag.Arg(0)
		f, err := os.Open(filename)
		if err != nil {
			fail(err)
		}
		defer f.Close()

		switch filepath.Ext(filename) {
		case ".xmp":
			b, err = ioutil.ReadAll(f)
			if err != nil {
				fail(err)
			}

		default:
			bb, err := xmp.ScanPackets(f)
			if err != nil && err != io.EOF {
				fail(err)
			}
			if err == io.EOF {
				return
			}
			if forig && fall {
				for _, b := range bb {
					out(b)
				}
				return
			}
			b = bb[0]
		}

	} else {

		// fill the document with some info
		s := xmp.NewDocument()
		s.AddModel(&xmpbase.XmpBase{
			CreatorTool: xmp.Agent,
			CreateDate:  xmp.Now(),
			ModifyDate:  xmp.Now(),
			Thumbnails: xmpbase.ThumbnailArray{
				xmpbase.Thumbnail{
					Format: "image/jpeg",
					Width:  10,
					Height: 10,
					Image:  []byte("not a real image"),
				},
			},
		})
		s.AddModel(&dc.DublinCore{
			Format:  "image/jpeg",
			Title:   xmp.NewAltString("demo"),
			Creator: xmp.NewStringList("Alexander Eichhorn"),
			Description: xmp.NewAltString(
				xmp.AltItem{Value: "Go-XMP Demo Model", Lang: "en", IsDefault: true},
				xmp.AltItem{Value: "Go-XMP Beispiel Modell", Lang: "de", IsDefault: false},
			),
		})
		b = marshal(s)
		s.Close()
	}

	if forig {
		out(b)
	}

	model := unmarshal(b)

	if fjson {
		b, err := json.MarshalIndent(model, "", "  ")
		if err != nil {
			fail(err)
		}
		out(b)
	}

	if fxmp {
		out(marshal(model))
	}

	if fpath {
		l, err := model.ListPaths()
		if err != nil {
			fail(err)
		}
		for _, v := range l {
			fmt.Printf("%s = %s\n", v.Path.String(), v.Value)
		}
	}

	model.Close()

	if debug {
		xmp.DumpStats()
	}
}
<|endoftext|>


<commit_before>package creational
<commit_msg>Add initial version of object pool<commit_after>package creational

import "sync"

// PoolObject represents the object to be stored in the Pool.
type PoolObject struct {
}

// Pool represents the pool of objects to use.
type Pool struct {
	*sync.Mutex
	inuse     []*PoolObject
	available []*PoolObject
}

// NewPool creates a new pool.
func NewPool() *Pool {
	return &Pool{}
}

// Acquire acquires a new PoolObject to use from the pool.
// Here acquire creates a new instance of a PoolObject if none available.
func (p *Pool) Acquire() *PoolObject {
	p.Lock()
	var object *PoolObject = nil
	if len(p.available) != 0 {
		object = p.available[0]
		p.available = append(p.available[:0], p.available[1:]...)
		p.inuse = append(p.inuse, object)
	} else {
		object := &PoolObject{}
		p.inuse = append(p.inuse, object)
	}
	p.Unlock()
	return object
}

// Release releases a PoolObject back to the Pool.
func (p *Pool) Release(object *PoolObject) {
	p.Lock()
	p.available = append(p.available, object)
	for i, v := range p.inuse {
		if v == object {
			p.inuse = append(p.inuse[:i], p.inuse[i+1:]...)
			break
		}
	}
	p.Unlock()
}
<|endoftext|>


<commit_before>package helpers

import (
	"encoding/json"
	"log"

	"github.com/byuoitav/av-api/status"
)

func GetVolume(address string) (status.Volume, error) {
	log.Printf("Getting volume for %v", address)
	parentResponse, err := getAudioInformation(address)
	if err != nil {
		return status.Volume{}, err
	}
	log.Printf("%v", parentResponse)

	var output status.Volume
	for _, outerResult := range parentResponse.Result {

		for _, result := range outerResult {

			if result.Target == "speaker" {

				output.Volume = result.Volume
			}
		}
	}
	log.Printf("Done")

	return output, nil
}

func getAudioInformation(address string) (SonyAudioResponse, error) {
	payload := SonyTVRequest{
		Params:  []map[string]interface{}{},
		Method:  "getVolumeInformation",
		Version: "1.0",
		ID:      1,
	}

	log.Printf("%+v", payload)

	resp, err := PostHTTP(address, payload, "audio")

	parentResponse := SonyAudioResponse{}

	log.Printf("%s", resp)

	err = json.Unmarshal(resp, &parentResponse)
	return parentResponse, err

}

func GetMute(address string) (status.MuteStatus, error) {
	log.Printf("Getting mute status for %v", address)
	parentResponse, err := getAudioInformation(address)
	if err != nil {
		return status.MuteStatus{}, err
	}
	var output status.MuteStatus
	for _, outerResult := range parentResponse.Result {
		for _, result := range outerResult {
			if result.Target == "speaker" {
				output.Muted = result.Mute
			}
		}
	}

	log.Printf("Done")

	return status.MuteStatus{}, nil
}
<commit_msg>fixing getting mute status<commit_after>package helpers

import (
	"encoding/json"
	"log"

	"github.com/byuoitav/av-api/status"
)

func GetVolume(address string) (status.Volume, error) {
	log.Printf("Getting volume for %v", address)
	parentResponse, err := getAudioInformation(address)
	if err != nil {
		return status.Volume{}, err
	}
	log.Printf("%v", parentResponse)

	var output status.Volume
	for _, outerResult := range parentResponse.Result {

		for _, result := range outerResult {

			if result.Target == "speaker" {

				output.Volume = result.Volume
			}
		}
	}
	log.Printf("Done")

	return output, nil
}

func getAudioInformation(address string) (SonyAudioResponse, error) {
	payload := SonyTVRequest{
		Params:  []map[string]interface{}{},
		Method:  "getVolumeInformation",
		Version: "1.0",
		ID:      1,
	}

	log.Printf("%+v", payload)

	resp, err := PostHTTP(address, payload, "audio")

	parentResponse := SonyAudioResponse{}

	log.Printf("%s", resp)

	err = json.Unmarshal(resp, &parentResponse)
	return parentResponse, err

}

func GetMute(address string) (status.MuteStatus, error) {
	log.Printf("Getting mute status for %v", address)
	parentResponse, err := getAudioInformation(address)
	if err != nil {
		return status.MuteStatus{}, err
	}
	var output status.MuteStatus
	for _, outerResult := range parentResponse.Result {
		for _, result := range outerResult {
			if result.Target == "speaker" {
				log.Printf("local mute: %v", result.Mute)
				output.Muted = result.Mute
			}
		}
	}

	log.Printf("Done")

	return output, nil
}
<|endoftext|>


<commit_before>package coal

import (
	"github.com/256dpi/xo"
	"go.mongodb.org/mongo-driver/bson"
)

// Tester provides facilities to work with coal models in tests.
type Tester struct {
	// The store to use for cleaning the database.
	Store *Store

	// The registered models.
	Models []Model
}

// NewTester returns a new tester. If no store is provided one will be created.
func NewTester(store *Store, models ...Model) *Tester {
	// ensure store
	if store == nil {
		store = MustOpen(nil, "test", xo.Panic)
	}

	// create tester
	tester := &Tester{
		Store:  store,
		Models: models,
	}

	// ensure collections
	for _, model := range models {
		_, _ = store.C(model).InsertOne(nil, GetMeta(model).Make())
	}

	// clean
	tester.Clean()

	return tester
}

// Insert will insert the specified model.
func (t *Tester) Insert(model Model) Model {
	// insert to collection
	err := t.Store.M(model).Insert(nil, model)
	if err != nil {
		panic(err)
	}

	return model
}

// FindAll will return all saved models.
func (t *Tester) FindAll(model Model, query ...bson.M) interface{} {
	// prepare query
	qry := bson.M{}
	if len(query) > 0 {
		qry = query[0]
	}

	// find all documents
	list := GetMeta(model).MakeSlice()
	err := t.Store.M(model).FindAll(nil, list, qry, []string{"_id"}, 0, 0, false, NoTransaction)
	if err != nil {
		panic(err)
	}

	return list
}

// FindLast will return the last saved model.
func (t *Tester) FindLast(model Model, query ...bson.M) Model {
	// prepare query
	qry := bson.M{}
	if len(query) > 0 {
		qry = query[0]
	}

	// find last document
	found, err := t.Store.M(model).FindFirst(nil, model, qry, []string{"-_id"}, 0, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Count will count all saved models.
func (t *Tester) Count(model Model, query ...bson.M) int {
	// prepare query
	qry := bson.M{}
	if len(query) > 0 {
		qry = query[0]
	}

	// count all documents
	n, err := t.Store.M(model).Count(nil, qry, 0, 0, false, NoTransaction)
	if err != nil {
		panic(err)
	}

	return int(n)
}

// Fetch will return the saved model.
func (t *Tester) Fetch(model Model, id ID) Model {
	// find model
	found, err := t.Store.M(model).Find(nil, model, id, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Replace will replace the specified model.
func (t *Tester) Replace(model Model) Model {
	// replace model
	found, err := t.Store.M(model).Replace(nil, model, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Update will update the specified model.
func (t *Tester) Update(model Model, update bson.M) Model {
	// replace model
	found, err := t.Store.M(model).Update(nil, model, model.ID(), update, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Delete will delete the specified model.
func (t *Tester) Delete(model Model) {
	// delete model
	found, err := t.Store.M(model).Delete(nil, nil, model.ID())
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}
}

// Clean will remove the collections of models that have been registered and
// reset the header map.
func (t *Tester) Clean() {
	for _, model := range t.Models {
		// remove all is faster than dropping the collection
		_, err := t.Store.M(model).DeleteAll(nil, bson.M{})
		if err != nil {
			panic(err)
		}
	}
}
<commit_msg>coal: added refresh method to tester<commit_after>package coal

import (
	"github.com/256dpi/xo"
	"go.mongodb.org/mongo-driver/bson"
)

// Tester provides facilities to work with coal models in tests.
type Tester struct {
	// The store to use for cleaning the database.
	Store *Store

	// The registered models.
	Models []Model
}

// NewTester returns a new tester. If no store is provided one will be created.
func NewTester(store *Store, models ...Model) *Tester {
	// ensure store
	if store == nil {
		store = MustOpen(nil, "test", xo.Panic)
	}

	// create tester
	tester := &Tester{
		Store:  store,
		Models: models,
	}

	// ensure collections
	for _, model := range models {
		_, _ = store.C(model).InsertOne(nil, GetMeta(model).Make())
	}

	// clean
	tester.Clean()

	return tester
}

// Insert will insert the specified model.
func (t *Tester) Insert(model Model) Model {
	// insert to collection
	err := t.Store.M(model).Insert(nil, model)
	if err != nil {
		panic(err)
	}

	return model
}

// FindAll will return all saved models.
func (t *Tester) FindAll(model Model, query ...bson.M) interface{} {
	// prepare query
	qry := bson.M{}
	if len(query) > 0 {
		qry = query[0]
	}

	// find all documents
	list := GetMeta(model).MakeSlice()
	err := t.Store.M(model).FindAll(nil, list, qry, []string{"_id"}, 0, 0, false, NoTransaction)
	if err != nil {
		panic(err)
	}

	return list
}

// FindLast will return the last saved model.
func (t *Tester) FindLast(model Model, query ...bson.M) Model {
	// prepare query
	qry := bson.M{}
	if len(query) > 0 {
		qry = query[0]
	}

	// find last document
	found, err := t.Store.M(model).FindFirst(nil, model, qry, []string{"-_id"}, 0, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Count will count all saved models.
func (t *Tester) Count(model Model, query ...bson.M) int {
	// prepare query
	qry := bson.M{}
	if len(query) > 0 {
		qry = query[0]
	}

	// count all documents
	n, err := t.Store.M(model).Count(nil, qry, 0, 0, false, NoTransaction)
	if err != nil {
		panic(err)
	}

	return int(n)
}

// Refresh will refresh the provided model.
func (t *Tester) Refresh(model Model) {
	// refresh model
	t.Fetch(model, model.ID())
}

// Fetch will return the saved model.
func (t *Tester) Fetch(model Model, id ID) Model {
	// find model
	found, err := t.Store.M(model).Find(nil, model, id, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Replace will replace the specified model.
func (t *Tester) Replace(model Model) Model {
	// replace model
	found, err := t.Store.M(model).Replace(nil, model, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Update will update the specified model.
func (t *Tester) Update(model Model, update bson.M) Model {
	// replace model
	found, err := t.Store.M(model).Update(nil, model, model.ID(), update, false)
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}

	return model
}

// Delete will delete the specified model.
func (t *Tester) Delete(model Model) {
	// delete model
	found, err := t.Store.M(model).Delete(nil, nil, model.ID())
	if err != nil {
		panic(err)
	} else if !found {
		panic("not found")
	}
}

// Clean will remove the collections of models that have been registered and
// reset the header map.
func (t *Tester) Clean() {
	for _, model := range t.Models {
		// remove all is faster than dropping the collection
		_, err := t.Store.M(model).DeleteAll(nil, bson.M{})
		if err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/ayoisaiah/stellar-photos-server/config"
	"github.com/ayoisaiah/stellar-photos-server/unsplash"
	"github.com/ayoisaiah/stellar-photos-server/utils"
)

const stellarPhotosCollectionID = 998309

const (
	standardRes = 2000
	highRes     = 4000
)

func getCollection() (unsplash.Collection, error) {
	unsplashAccessKey := config.Conf.Unsplash.AccessKey

	url := fmt.Sprintf(
		"%s/collections/%d?client_id=%s",
		unsplash.APIBaseURL,
		stellarPhotosCollectionID,
		unsplashAccessKey,
	)

	var c unsplash.Collection

	_, err := utils.SendGETRequest(url, &c)
	if err != nil {
		return c, err
	}

	return c, nil
}

func retrieveAllPhotos() (map[string]unsplash.Photo, error) {
	collection, err := getCollection()
	if err != nil {
		return nil, err
	}

	unsplashAccessKey := config.Conf.Unsplash.AccessKey

	var allPhotos = make([]unsplash.Photo, collection.TotalPhotos)

	page, perPage := 1, 30

	for {
		var photos []unsplash.Photo

		url := fmt.Sprintf(
			"%s/collections/%d/photos?page=%d&per_page=%d&client_id=%s",
			unsplash.APIBaseURL,
			stellarPhotosCollectionID,
			page,
			perPage,
			unsplashAccessKey,
		)

		_, err := utils.SendGETRequest(url, &photos)
		if err != nil {
			return nil, err
		}

		allPhotos = append(allPhotos, photos...)

		page++

		if len(photos) == 0 {
			break
		}
	}

	var m = make(map[string]unsplash.Photo)

	for i := range allPhotos {
		v := allPhotos[i]

		m[v.ID] = v
	}

	return m, nil
}

func downloadPhotos(photos map[string]unsplash.Photo) []error {
	var errs []error

	for k := range photos {
		v := photos[k]

		err := os.MkdirAll(filepath.Join("cached_images", k), os.ModePerm)
		if err != nil {
			errs = append(errs, err)
			continue
		}

		widths := []int{standardRes}

		if v.Width >= highRes {
			widths = append(widths, highRes, v.Width)
		} else {
			widths = append(widths, v.Width)
		}

		for _, width := range widths {
			imageURL := fmt.Sprintf("%s&w=%d", v.Urls.Raw, width)

			fileName := fmt.Sprintf("%d.txt", width)

			filePath := filepath.Join("cached_images", k, fileName)

			if _, err = os.Stat(filePath); err == nil ||
				errors.Is(err, os.ErrExist) {
				continue
			}

			var base64 string

			base64, err = utils.GetImageBase64(imageURL, fileName, k)
			if err != nil {
				errs = append(errs, err)
				continue
			}

			err = os.WriteFile(filePath, []byte(base64), os.ModePerm)
			if err != nil {
				errs = append(errs, err)
				continue
			}
		}

		fileName := k + ".json"

		filePath := filepath.Join("cached_images", k, fileName)

		if _, err = os.Stat(filePath); err == nil ||
			errors.Is(err, os.ErrExist) {
			continue
		}

		b, err := json.Marshal(v)
		if err != nil {
			errs = append(errs, err)
			continue
		}

		err = os.WriteFile(filePath, b, os.ModePerm)
		if err != nil {
			errs = append(errs, err)
			continue
		}
	}

	return errs
}

func cleanup(photos map[string]unsplash.Photo) {
	files, err := os.ReadDir("cached_images")
	if err != nil {
		utils.Logger().Errorw("Unable to read cached_images directory",
			"tag", "read_cached_images_dir_failure",
			"error", err,
		)

		return
	}

	cleaned := make(map[string]bool)

	for _, f := range files {
		fileName := f.Name()

		id := strings.Split(fileName[:len(fileName)-len(filepath.Ext(fileName))], "_")[0]

		if _, ok := cleaned[id]; ok {
			continue
		}

		if _, ok := photos[id]; !ok {
			err := os.RemoveAll(filepath.Join("cached_images", id))
			if err != nil {
				utils.Logger().
					Warnw("Unable to clean deleted photo from cached_images directory",
						"tag", "cache_clean_failure",
						"image_id", id,
						"error", err,
					)

				continue
			}

			cleaned[id] = true

			utils.Logger().
				Infow("Photo cleaned from cached_images directory successfully",
					"image_id", id,
				)
		}
	}
}

// Photos caches all Unsplash images in the default collection locally.
// It also cleans up images that were deleted from the collection.
func Photos() {
	l := utils.Logger()

	l.Infow("Pre-caching all images in default collection",
		"tag", "pre_caching_start",
	)

	photos, err := retrieveAllPhotos()
	if err != nil {
		l.Errorw("Unable to retrieve all images in default collection",
			"tag", "retrieve_all_photos_failure",
			"error", err,
		)

		return
	}

	errs := downloadPhotos(photos)
	if len(errs) != 0 {
		l.Errorw("Some downloads failed to complete",
			"tag", "download_photos_cache_failure",
			"error", errs,
		)

		return
	}

	cleanup(photos)

	l.Infow("Cached images updated successfully!",
		"tag", "pre_caching_end",
	)
}
<commit_msg>Delete empty ids from photos collection<commit_after>package cache

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/ayoisaiah/stellar-photos-server/config"
	"github.com/ayoisaiah/stellar-photos-server/unsplash"
	"github.com/ayoisaiah/stellar-photos-server/utils"
)

const stellarPhotosCollectionID = 998309

const (
	standardRes = 2000
	highRes     = 4000
)

func getCollection() (unsplash.Collection, error) {
	unsplashAccessKey := config.Conf.Unsplash.AccessKey

	url := fmt.Sprintf(
		"%s/collections/%d?client_id=%s",
		unsplash.APIBaseURL,
		stellarPhotosCollectionID,
		unsplashAccessKey,
	)

	var c unsplash.Collection

	_, err := utils.SendGETRequest(url, &c)
	if err != nil {
		return c, err
	}

	return c, nil
}

func retrieveAllPhotos() (map[string]unsplash.Photo, error) {
	collection, err := getCollection()
	if err != nil {
		return nil, err
	}

	unsplashAccessKey := config.Conf.Unsplash.AccessKey

	var allPhotos = make([]unsplash.Photo, collection.TotalPhotos)

	page, perPage := 1, 30

	for {
		var photos []unsplash.Photo

		url := fmt.Sprintf(
			"%s/collections/%d/photos?page=%d&per_page=%d&client_id=%s",
			unsplash.APIBaseURL,
			stellarPhotosCollectionID,
			page,
			perPage,
			unsplashAccessKey,
		)

		_, err := utils.SendGETRequest(url, &photos)
		if err != nil {
			return nil, err
		}

		if len(photos) == 0 {
			break
		}

		allPhotos = append(allPhotos, photos...)

		page++
	}

	var m = make(map[string]unsplash.Photo)

	for i := range allPhotos {
		v := allPhotos[i]

		m[v.ID] = v
	}

	delete(m, "")

	return m, nil
}

func downloadPhotos(photos map[string]unsplash.Photo) []error {
	var errs []error

	for k := range photos {
		v := photos[k]

		err := os.MkdirAll(filepath.Join("cached_images", k), os.ModePerm)
		if err != nil {
			errs = append(errs, err)
			continue
		}

		widths := []int{standardRes}

		if v.Width >= highRes {
			widths = append(widths, highRes, v.Width)
		} else {
			widths = append(widths, v.Width)
		}

		for _, width := range widths {
			imageURL := fmt.Sprintf("%s&w=%d", v.Urls.Raw, width)

			fileName := fmt.Sprintf("%d.txt", width)

			filePath := filepath.Join("cached_images", k, fileName)

			if _, err = os.Stat(filePath); err == nil ||
				errors.Is(err, os.ErrExist) {
				continue
			}

			var base64 string

			base64, err = utils.GetImageBase64(imageURL, fileName, k)
			if err != nil {
				errs = append(errs, err)
				continue
			}

			err = os.WriteFile(filePath, []byte(base64), os.ModePerm)
			if err != nil {
				errs = append(errs, err)
				continue
			}
		}

		fileName := k + ".json"

		filePath := filepath.Join("cached_images", k, fileName)

		if _, err = os.Stat(filePath); err == nil ||
			errors.Is(err, os.ErrExist) {
			continue
		}

		b, err := json.Marshal(v)
		if err != nil {
			errs = append(errs, err)
			continue
		}

		err = os.WriteFile(filePath, b, os.ModePerm)
		if err != nil {
			errs = append(errs, err)
			continue
		}
	}

	return errs
}

func cleanup(photos map[string]unsplash.Photo) {
	files, err := os.ReadDir("cached_images")
	if err != nil {
		utils.Logger().Errorw("Unable to read cached_images directory",
			"tag", "read_cached_images_dir_failure",
			"error", err,
		)

		return
	}

	cleaned := make(map[string]bool)

	for _, f := range files {
		fileName := f.Name()

		id := strings.Split(fileName[:len(fileName)-len(filepath.Ext(fileName))], "_")[0]

		if _, ok := cleaned[id]; ok {
			continue
		}

		if _, ok := photos[id]; !ok {
			err := os.RemoveAll(filepath.Join("cached_images", id))
			if err != nil {
				utils.Logger().
					Warnw("Unable to clean deleted photo from cached_images directory",
						"tag", "cache_clean_failure",
						"image_id", id,
						"error", err,
					)

				continue
			}

			cleaned[id] = true

			utils.Logger().
				Infow("Photo cleaned from cached_images directory successfully",
					"image_id", id,
				)
		}
	}
}

// Photos caches all Unsplash images in the default collection locally.
// It also cleans up images that were deleted from the collection.
func Photos() {
	l := utils.Logger()

	l.Infow("Pre-caching all images in default collection",
		"tag", "pre_caching_start",
	)

	photos, err := retrieveAllPhotos()
	if err != nil {
		l.Errorw("Unable to retrieve all images in default collection",
			"tag", "retrieve_all_photos_failure",
			"error", err,
		)

		return
	}

	errs := downloadPhotos(photos)
	if len(errs) != 0 {
		l.Errorw("Some downloads failed to complete",
			"tag", "download_photos_cache_failure",
			"error", errs,
		)

		return
	}

	cleanup(photos)

	l.Infow("Cached images updated successfully!",
		"tag", "pre_caching_end",
	)
}
<|endoftext|>