<commit_before>package main

import (
	cryptoRand "crypto/rand"
	"encoding/binary"
	"flag"
	"fmt"
	"github.com/HouzuoGuo/laitos/daemon/common"
	"github.com/HouzuoGuo/laitos/encarchive"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/toolbox"
	"io/ioutil"
	pseudoRand "math/rand"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"runtime"
	"runtime/pprof"
	"strconv"
	"sync"
	"syscall"
	"time"
)

const DaemonRestartIntervalSec = 10 // DaemonRestartIntervalSec is the interval to pause between daemon start attempts.

var logger = misc.Logger{ComponentName: "laitos", ComponentID: strconv.Itoa(os.Getpid())}

// Re-seed global pseudo random generator using cryptographic random number generator.
func ReseedPseudoRand() {
	numAttempts := 1
	for ; ; numAttempts++ {
		seedBytes := make([]byte, 8)
		_, err := cryptoRand.Read(seedBytes)
		if err != nil {
			logger.Panicf("ReseedPseudoRand", "", err, "failed to read from random generator")
		}
		seed, _ := binary.Varint(seedBytes)
		if seed == 0 {
			// If random entropy decodes into an integer that overflows, simply retry.
			continue
		} else {
			pseudoRand.Seed(seed)
			break
		}
	}
	logger.Printf("ReseedPseudoRand", "", nil, "succeeded after %d attempt(s)", numAttempts)
}

// Stop and disable daemons that may run into port usage conflicts with laitos.
func DisableConflictingDaemons() {
	if os.Getuid() != 0 {
		logger.Fatalf("DisableConflictingDaemons", "", nil, "you must run laitos as root user if you wish to automatically disable conflicting daemons")
	}
	list := []string{"apache", "apache2", "bind", "bind9", "httpd", "lighttpd", "named", "named-chroot", "postfix", "sendmail"}
	waitGroup := new(sync.WaitGroup)
	waitGroup.Add(len(list))
	for _, name := range list {
		go func(name string) {
			defer waitGroup.Done()
			var success bool
			// Disable+stop intensifies three times...
			for i := 0; i < 3; i++ {
				cmds := []string{
					// Some hosting providers still do not use systemd, an example is Amazon Elastic Beanstalk.
					fmt.Sprintf("/etc/init.d/%s stop", name),
					fmt.Sprintf("chkconfig %s off", name),
					fmt.Sprintf("chmod 0000 /etc/init.d/%s", name),

					fmt.Sprintf("systemctl stop %s", name),
					fmt.Sprintf("systemctl disable %s", name),
					fmt.Sprintf("systemctl mask %s", name),
				}
				for _, cmd := range cmds {
					if _, err := misc.InvokeShell(5, "/bin/sh", cmd); err == nil {
						success = true
						// Continue to run subsequent commands to further disable the service
					}
				}
				// Do not overwhelm system with too many consecutive commands
				time.Sleep(1 * time.Second)
			}
			if success {
				logger.Printf("DisableConflictingDaemons", name, nil, "the daemon has been successfully stopped and disabled")
			}
		}(name)
	}
	waitGroup.Wait()
}

func main() {
	// Lock all program memory into main memory to prevent sensitive data from leaking into swap.
	if os.Geteuid() == 0 {
		if err := syscall.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE); err != nil {
			logger.Fatalf("main", "", err, "failed to lock memory")
			return
		}
		logger.Warningf("main", "", nil, "program has been locked into memory for safety reasons")
	} else {
		logger.Warningf("main", "", nil, "program is not running as root (UID 0) hence memory is not locked, your private information will leak into swap.")
	}

	// Process command line flags
	var frontend string
	var disableConflicts, tuneSystem, debug bool
	var gomaxprocs int
	flag.StringVar(&misc.ConfigFilePath, "config", "", "(Mandatory) path to configuration file in JSON syntax")
	flag.StringVar(&frontend, "frontend", "", "(Mandatory) comma-separated frontend services to start (dnsd, httpd, insecurehttpd, mailcmd, maintenance, plainsocket, smtpd, sockd, telegram)")
	flag.BoolVar(&disableConflicts, "disableconflicts", false, "(Optional) automatically stop and disable other daemon programs that may cause port usage conflicts")
	flag.BoolVar(&tuneSystem, "tunesystem", false, "(Optional) tune operating system parameters for optimal performance")
	flag.BoolVar(&debug, "debug", false, "(Optional) print goroutine stack traces upon receiving interrupt signal")
	flag.IntVar(&gomaxprocs, "gomaxprocs", 0, "(Optional) set gomaxprocs")
	// Process launcher flags
	var sl bool
	var slPort int
	var slArchivePath string
	var slURL string
	flag.BoolVar(&sl, encarchive.MagicArg, false, "(Optional) trigger \"special-launch\"")
	flag.IntVar(&slPort, "slport", 80, "(Optional) special-launch: port number")
	flag.StringVar(&slArchivePath, "slarchive", "", "(Optional) special-launch: archive path")
	flag.StringVar(&slURL, "slurl", "", "(Optional) special-launch: url that must include prefix slash")
	// Process launcher utility mode flags
	var slu, sluDir, sluFile string
	flag.StringVar(&slu, "slu", "", "(Optional) special-launch: utility name (extract, archive)")
	flag.StringVar(&sluDir, "sludir", "", "(Optional) special-launch: source/target directory")
	flag.StringVar(&sluFile, "slufile", "", "(Optional) special-launch: source/target archive file")
	flag.Parse()

	// Dump goroutine stacktraces upon receiving interrupt signal
	if debug {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for range c {
				pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
			}
		}()
	}

	// Re-seed pseudo random number generator once a while
	ReseedPseudoRand()
	go func() {
		time.Sleep(2 * time.Minute)
		ReseedPseudoRand()
	}()

	// Configure gomaxprocs to help laitos daemons
	if gomaxprocs > 0 {
		oldGomaxprocs := runtime.GOMAXPROCS(gomaxprocs)
		logger.Warningf("main", "", nil, "GOMAXPROCS has been changed from %d to %d", oldGomaxprocs, gomaxprocs)
	} else {
		logger.Warningf("main", "", nil, "GOMAXPROCS is unchanged at %d", runtime.GOMAXPROCS(0))
	}

	// Stop certain daemons to increase chance of successful launch of laitos daemons
	if disableConflicts {
		DisableConflictingDaemons()
	}

	// Tune operating system parameters for optimal performance and better resource utilisation
	if tuneSystem {
		logger.Warningf("main", "", nil, "System tuning result is: \n%s", toolbox.TuneLinux())
	}

	// The "special-launch" launcher runs prior to the daemons
	if sl {
		encarchive.CLIStartWebServer(slPort, slURL, slArchivePath)
		return
	}
	// The launcher utilities do not run daemons at all
	if slu != "" {
		switch slu {
		case "extract":
			encarchive.CLIExtract(sluDir, sluFile)
		case "archive":
			encarchive.CLIArchive(sluDir, sluFile)
		default:
			logger.Fatalf("main", "", nil, "please provide mode of operation (extract|archive) for parameter slu")
		}
		return
	}

	if misc.ConfigFilePath == "" {
		logger.Fatalf("main", "", nil, "please provide a configuration file (-config)")
		return
	}
	var err error
	misc.ConfigFilePath, err = filepath.Abs(misc.ConfigFilePath)
	if err != nil {
		logger.Fatalf("main", "", err, "failed to determine absolute path of config file \"%s\"", misc.ConfigFilePath)
	}

	// Deserialise JSON configuration file
	var config Config
	configBytes, err := ioutil.ReadFile(misc.ConfigFilePath)
	if err != nil {
		logger.Fatalf("main", "", err, "failed to read config file \"%s\"", misc.ConfigFilePath)
		return
	}
	if err := config.DeserialiseFromJSON(configBytes); err != nil {
		logger.Fatalf("main", "", err, "failed to deserialise config file \"%s\"", misc.ConfigFilePath)
		return
	}

	// Figure out what daemons are to be started
	frontendList := regexp.MustCompile(`\w+`)
	frontends := frontendList.FindAllString(frontend, -1)
	if frontends == nil || len(frontends) == 0 {
		logger.Fatalf("main", "", nil, "please provide comma-separated list of frontend services to start (-frontend).")
		return
	}

	// Start each daemon
	for _, frontendName := range frontends {
		// Daemons are started asynchronously, the order of startup does not matter.
		switch frontendName {
		case "dnsd":
			go common.NewSupervisor(config.GetDNSD(), DaemonRestartIntervalSec, frontendName).Start()
		case "httpd":
			go common.NewSupervisor(config.GetHTTPD(), DaemonRestartIntervalSec, frontendName).Start()
		case "insecurehttpd":
			go common.NewSupervisor(config.GetInsecureHTTPD(), DaemonRestartIntervalSec, frontendName).Start()
		case "mailcmd":
			mailContent, err := ioutil.ReadAll(os.Stdin)
			if err != nil {
				logger.Fatalf("main", "", err, "failed to read mail from STDIN")
				return
			}
			if err := config.GetMailCommandRunner().Process(mailContent); err != nil {
				logger.Fatalf("main", "", err, "failed to process mail")
			}
			// Mail processor for standard input is not a daemon
			return
		case "maintenance":
			go common.NewSupervisor(config.GetMaintenance(), DaemonRestartIntervalSec, frontendName).Start()
		case "plainsocket":
			go common.NewSupervisor(config.GetPlainSocketDaemon(), DaemonRestartIntervalSec, frontendName).Start()
		case "smtpd":
			go common.NewSupervisor(config.GetMailDaemon(), DaemonRestartIntervalSec, frontendName).Start()
		case "sockd":
			go common.NewSupervisor(config.GetSockDaemon(), DaemonRestartIntervalSec, frontendName).Start()
		case "telegram":
			go common.NewSupervisor(config.GetTelegramBot(), DaemonRestartIntervalSec, frontendName).Start()
		default:
			logger.Fatalf("main", "", err, "unknown frontend name \"%s\"", frontendName)
		}
	}
	// Daemons are not really supposed to quit.
	// In rare circumstance, if all daemons fail to start without panicking, laitos will just hang right here.
	for {
		time.Sleep(time.Hour)
		logger.Printf("main", "", nil, "laitos has been up for %s", time.Now().Sub(misc.StartupTime))
	}
}
<commit_msg>optionally turn off swap files and partitions prior to starting daemons<commit_after>package main

import (
	cryptoRand "crypto/rand"
	"encoding/binary"
	"flag"
	"fmt"
	"github.com/HouzuoGuo/laitos/daemon/common"
	"github.com/HouzuoGuo/laitos/encarchive"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/toolbox"
	"io/ioutil"
	pseudoRand "math/rand"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"runtime"
	"runtime/pprof"
	"strconv"
	"sync"
	"syscall"
	"time"
)

const DaemonRestartIntervalSec = 10 // DaemonRestartIntervalSec is the interval to pause between daemon start attempts.

var logger = misc.Logger{ComponentName: "laitos", ComponentID: strconv.Itoa(os.Getpid())}

// Re-seed global pseudo random generator using cryptographic random number generator.
func ReseedPseudoRand() {
	numAttempts := 1
	for ; ; numAttempts++ {
		seedBytes := make([]byte, 8)
		_, err := cryptoRand.Read(seedBytes)
		if err != nil {
			logger.Panicf("ReseedPseudoRand", "", err, "failed to read from random generator")
		}
		seed, _ := binary.Varint(seedBytes)
		if seed == 0 {
			// If random entropy decodes into an integer that overflows, simply retry.
			continue
		} else {
			pseudoRand.Seed(seed)
			break
		}
	}
	logger.Printf("ReseedPseudoRand", "", nil, "succeeded after %d attempt(s)", numAttempts)
}

// Stop and disable daemons that may run into port usage conflicts with laitos.
func DisableConflictingDaemons() {
	if os.Getuid() != 0 {
		logger.Fatalf("DisableConflictingDaemons", "", nil, "you must run laitos as root user if you wish to automatically disable conflicting daemons")
	}
	list := []string{"apache", "apache2", "bind", "bind9", "httpd", "lighttpd", "named", "named-chroot", "postfix", "sendmail"}
	waitGroup := new(sync.WaitGroup)
	waitGroup.Add(len(list))
	for _, name := range list {
		go func(name string) {
			defer waitGroup.Done()
			var success bool
			// Disable+stop intensifies three times...
			for i := 0; i < 3; i++ {
				cmds := []string{
					// Some hosting providers still do not use systemd, an example is Amazon Elastic Beanstalk.
					fmt.Sprintf("/etc/init.d/%s stop", name),
					fmt.Sprintf("chkconfig %s off", name),
					fmt.Sprintf("chmod 0000 /etc/init.d/%s", name),

					fmt.Sprintf("systemctl stop %s", name),
					fmt.Sprintf("systemctl disable %s", name),
					fmt.Sprintf("systemctl mask %s", name),
				}
				for _, cmd := range cmds {
					if _, err := misc.InvokeShell(5, "/bin/sh", cmd); err == nil {
						success = true
						// Continue to run subsequent commands to further disable the service
					}
				}
				// Do not overwhelm system with too many consecutive commands
				time.Sleep(1 * time.Second)
			}
			if success {
				logger.Printf("DisableConflictingDaemons", name, nil, "the daemon has been successfully stopped and disabled")
			}
		}(name)
	}
	waitGroup.Wait()
}

// SwapOff turns off all swap files and partitions for improved system security.
func SwapOff() {
	out, err := misc.InvokeProgram(nil, 60, "swapoff", "-a")
	if err == nil {
		logger.Printf("SwapOff", "", nil, "swap is now off")
	} else {
		logger.Printf("SwapOff", "", err, "failed to turn off swap - %s", out)
	}
}

func main() {
	// Lock all program memory into main memory to prevent sensitive data from leaking into swap.
	if os.Geteuid() == 0 {
		if err := syscall.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE); err != nil {
			logger.Fatalf("main", "", err, "failed to lock memory")
			return
		}
		logger.Warningf("main", "", nil, "program has been locked into memory for safety reasons")
	} else {
		logger.Warningf("main", "", nil, "program is not running as root (UID 0) hence memory is not locked, your private information will leak into swap.")
	}

	// Process command line flags
	var frontend string
	var disableConflicts, tuneSystem, debug, swapOff bool
	var gomaxprocs int
	flag.StringVar(&misc.ConfigFilePath, "config", "", "(Mandatory) path to configuration file in JSON syntax")
	flag.StringVar(&frontend, "frontend", "", "(Mandatory) comma-separated frontend services to start (dnsd, httpd, insecurehttpd, mailcmd, maintenance, plainsocket, smtpd, sockd, telegram)")
	flag.BoolVar(&disableConflicts, "disableconflicts", false, "(Optional) automatically stop and disable other daemon programs that may cause port usage conflicts")
	flag.BoolVar(&swapOff, "swapoff", false, "(Optional) turn off all swap files and partitions for improved system security")
	flag.BoolVar(&tuneSystem, "tunesystem", false, "(Optional) tune operating system parameters for optimal performance")
	flag.BoolVar(&debug, "debug", false, "(Optional) print goroutine stack traces upon receiving interrupt signal")
	flag.IntVar(&gomaxprocs, "gomaxprocs", 0, "(Optional) set gomaxprocs")
	// Process launcher flags
	var sl bool
	var slPort int
	var slArchivePath string
	var slURL string
	flag.BoolVar(&sl, encarchive.MagicArg, false, "(Optional) trigger \"special-launch\"")
	flag.IntVar(&slPort, "slport", 80, "(Optional) special-launch: port number")
	flag.StringVar(&slArchivePath, "slarchive", "", "(Optional) special-launch: archive path")
	flag.StringVar(&slURL, "slurl", "", "(Optional) special-launch: url that must include prefix slash")
	// Process launcher utility mode flags
	var slu, sluDir, sluFile string
	flag.StringVar(&slu, "slu", "", "(Optional) special-launch: utility name (extract, archive)")
	flag.StringVar(&sluDir, "sludir", "", "(Optional) special-launch: source/target directory")
	flag.StringVar(&sluFile, "slufile", "", "(Optional) special-launch: source/target archive file")
	flag.Parse()

	// Dump goroutine stacktraces upon receiving interrupt signal
	if debug {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for range c {
				pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
			}
		}()
	}

	// The "special-launch" launcher runs prior to the daemons
	if sl {
		encarchive.CLIStartWebServer(slPort, slURL, slArchivePath)
		return
	}
	// The launcher utilities do not run daemons at all
	if slu != "" {
		switch slu {
		case "extract":
			encarchive.CLIExtract(sluDir, sluFile)
		case "archive":
			encarchive.CLIArchive(sluDir, sluFile)
		default:
			logger.Fatalf("main", "", nil, "please provide mode of operation (extract|archive) for parameter slu")
		}
		return
	}

	// Deserialise JSON configuration file
	if misc.ConfigFilePath == "" {
		logger.Fatalf("main", "", nil, "please provide a configuration file (-config)")
		return
	}
	var err error
	misc.ConfigFilePath, err = filepath.Abs(misc.ConfigFilePath)
	if err != nil {
		logger.Fatalf("main", "", err, "failed to determine absolute path of config file \"%s\"", misc.ConfigFilePath)
	}
	var config Config
	configBytes, err := ioutil.ReadFile(misc.ConfigFilePath)
	if err != nil {
		logger.Fatalf("main", "", err, "failed to read config file \"%s\"", misc.ConfigFilePath)
		return
	}
	if err := config.DeserialiseFromJSON(configBytes); err != nil {
		logger.Fatalf("main", "", err, "failed to deserialise config file \"%s\"", misc.ConfigFilePath)
		return
	}

	// Figure out what daemons are to be started
	frontendList := regexp.MustCompile(`\w+`)
	frontends := frontendList.FindAllString(frontend, -1)
	if frontends == nil || len(frontends) == 0 {
		logger.Fatalf("main", "", nil, "please provide comma-separated list of frontend services to start (-frontend).")
		return
	}

	/*
		To help daemons running better, do the following (some of which are optional)
		- Periodically re-seed pseudo random generator.
		- Configure gomaxprocs.
		- Stop system daemons that may run into conflict with laitos daemons.
		- Turn off all swap files and partitions for improved security.
		- Tune system kernel parameters.
	*/
	ReseedPseudoRand()
	go func() {
		time.Sleep(2 * time.Minute)
		ReseedPseudoRand()
	}()
	if gomaxprocs > 0 {
		oldGomaxprocs := runtime.GOMAXPROCS(gomaxprocs)
		logger.Warningf("main", "", nil, "GOMAXPROCS has been changed from %d to %d", oldGomaxprocs, gomaxprocs)
	} else {
		logger.Warningf("main", "", nil, "GOMAXPROCS is unchanged at %d", runtime.GOMAXPROCS(0))
	}
	if disableConflicts {
		DisableConflictingDaemons()
	}
	if swapOff {
		SwapOff()
	}
	if tuneSystem {
		logger.Warningf("main", "", nil, "System tuning result is: \n%s", toolbox.TuneLinux())
	}

	// Start each daemon
	for _, frontendName := range frontends {
		// Daemons are started asynchronously, the order of startup does not matter.
		switch frontendName {
		case "dnsd":
			go common.NewSupervisor(config.GetDNSD(), DaemonRestartIntervalSec, frontendName).Start()
		case "httpd":
			go common.NewSupervisor(config.GetHTTPD(), DaemonRestartIntervalSec, frontendName).Start()
		case "insecurehttpd":
			go common.NewSupervisor(config.GetInsecureHTTPD(), DaemonRestartIntervalSec, frontendName).Start()
		case "mailcmd":
			mailContent, err := ioutil.ReadAll(os.Stdin)
			if err != nil {
				logger.Fatalf("main", "", err, "failed to read mail from STDIN")
				return
			}
			if err := config.GetMailCommandRunner().Process(mailContent); err != nil {
				logger.Fatalf("main", "", err, "failed to process mail")
			}
			// Mail processor for standard input is not a daemon
			return
		case "maintenance":
			go common.NewSupervisor(config.GetMaintenance(), DaemonRestartIntervalSec, frontendName).Start()
		case "plainsocket":
			go common.NewSupervisor(config.GetPlainSocketDaemon(), DaemonRestartIntervalSec, frontendName).Start()
		case "smtpd":
			go common.NewSupervisor(config.GetMailDaemon(), DaemonRestartIntervalSec, frontendName).Start()
		case "sockd":
			go common.NewSupervisor(config.GetSockDaemon(), DaemonRestartIntervalSec, frontendName).Start()
		case "telegram":
			go common.NewSupervisor(config.GetTelegramBot(), DaemonRestartIntervalSec, frontendName).Start()
		default:
			logger.Fatalf("main", "", err, "unknown frontend name \"%s\"", frontendName)
		}
	}
	// Daemons are not really supposed to quit
	for {
		// In rare circumstance, if all daemons fail to start without panicking, laitos will just hang right here.
		time.Sleep(time.Hour)
		logger.Printf("main", "", nil, "laitos has been up for %s", time.Now().Sub(misc.StartupTime))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"fm.tul.cz/dupl/output/text"
	"fm.tul.cz/dupl/suffixtree"
	"fm.tul.cz/dupl/syntax"
	"fm.tul.cz/dupl/syntax/golang"
)

type char int

func (c char) Val() int {
	return int(c)
}

var (
	dir       = "."
	threshold = flag.Int("t", 15, "minimum token sequence as a clone")
)

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) > 0 {
		dir = args[0]
	}

	fchan := make(chan string)
	go func() {
		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") {
				fchan <- path
			}
			return nil
		})
		close(fchan)
	}()

	t := suffixtree.New()

	for {
		file, ok := <-fchan
		if !ok {
			break
		}
		syn, err := golang.Parse(file)
		if err != nil {
			log.Println(err)
			continue
		}
		stream := syntax.Serialize(syn)
		for _, item := range stream {
			t.Update(item)
		}
	}

	// finish stream
	t.Update(char(-1))

	printer := text.NewPrinter(os.Stdout)

	mchan := t.FindDuplOver(*threshold)
	cnt := 0
	for {
		m, ok := <-mchan
		if !ok {
			break
		}
		dup1, dup2 := syntax.FindSyntaxUnits(t, m)
		printer.Print(dup1, dup2)
		cnt++
	}
	fmt.Printf("\nFound total %d clones.\n", cnt)
}
<commit_msg>main.go: make app more concurrent<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"fm.tul.cz/dupl/output/text"
	"fm.tul.cz/dupl/suffixtree"
	"fm.tul.cz/dupl/syntax"
	"fm.tul.cz/dupl/syntax/golang"
)

type char int

func (c char) Val() int {
	return int(c)
}

var (
	dir       = "."
	threshold = flag.Int("t", 15, "minimum token sequence as a clone")
)

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) > 0 {
		dir = args[0]
	}

	// collecting files
	fchan := make(chan string)
	go func() {
		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") {
				fchan <- path
			}
			return nil
		})
		close(fchan)
	}()

	// AST parsing
	achan := make(chan *syntax.Node)
	go func() {
		for {
			file, ok := <-fchan
			if !ok {
				break
			}
			ast, err := golang.Parse(file)
			if err != nil {
				log.Println(err)
				continue
			}
			achan <- ast
		}
		close(achan)
	}()

	// serialization
	schan := make(chan []*syntax.Node)
	go func() {
		for {
			ast, ok := <-achan
			if !ok {
				break
			}
			seq := syntax.Serialize(ast)
			schan <- seq
		}
		close(schan)
	}()

	// suffix tree
	t := suffixtree.New()
	for {
		seq, ok := <-schan
		if !ok {
			break
		}
		for _, item := range seq {
			t.Update(item)
		}
	}

	// finish stream
	t.Update(char(-1))

	// printing the clones
	printer := text.NewPrinter(os.Stdout)
	mchan := t.FindDuplOver(*threshold)
	cnt := 0
	for {
		m, ok := <-mchan
		if !ok {
			break
		}
		dup1, dup2 := syntax.FindSyntaxUnits(t, m)
		printer.Print(dup1, dup2)
		cnt++
	}
	fmt.Printf("\nFound total %d clones.\n", cnt)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"hash/crc32"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"qiniupkg.com/http/httputil.v2"
	"qiniupkg.com/x/log.v7"
)

const (
	mutexCount = 9973
)

var (
	ErrUnmatchedInodeType = errors.New("unmatched inode type(file or dir)")
	ErrRefreshWithoutPath = httputil.NewError(400, "refresh without path")
	ErrInvalidPkgPath     = httputil.NewError(400, "invalid package path")
)

var (
	doxygenApp string

	dataRootDir string
	srcRootDir  string
	tmpRootDir  string

	refreshRootDir string

	mutexs [mutexCount]sync.Mutex
)

func handleHome(w http.ResponseWriter, req *http.Request) {

}

func handleUnknown(w http.ResponseWriter, req *http.Request) {

}

func handleRefresh(w http.ResponseWriter, req *http.Request) {

	pkg := req.PostFormValue("path")
	if pkg == "" {
		httputil.Error(w, ErrRefreshWithoutPath)
		return
	}

	log.Info("Refresh", pkg)

	err := refresh(pkg)
	if err != nil {
		httputil.Error(w, err)
		return
	}

	http.Redirect(w, req, "/" + pkg + "/", 301)
}

func refresh(pkg string) (err error) {

	dataDir := dataRootDir + pkg
	indexFile := dataDir + "/html/index.html"
	if isRefreshed(indexFile) {
		return nil
	}

	if strings.Index(pkg, "..") >= 0 {
		return ErrInvalidPkgPath
	}

	parts := strings.SplitN(pkg, "/", 4)
	if len(parts) != 3 {
		return ErrInvalidPkgPath
	}

	refreshDir := refreshRootDir + pkg
	refreshHtmlDir := refreshDir + "/html/"
	os.RemoveAll(refreshDir)
	err = genDoc(parts, pkg, refreshDir, refreshHtmlDir)
	if err != nil {
		return
	}

	os.RemoveAll(dataDir)
	return os.Rename(refreshDir, dataDir)
}

func isRefreshed(indexFile string) bool {

	fi, err := os.Stat(indexFile)
	if err != nil {
		return false
	}

	return time.Now().Sub(fi.ModTime()) < 10*time.Second
}

func handleMain(w http.ResponseWriter, req *http.Request) {

	path := req.URL.Path

	if path == "/" {
		handleHome(w, req)
		return
	}

	log.Info("View", path)

	if strings.Index(path, "..") >= 0 {
		handleUnknown(w, req)
		return
	}

	parts := strings.SplitN(path[1:], "/", 4)
	if parts[0] != "github.com" || len(parts) < 3 {
		handleUnknown(w, req)
		return
	}

	pkg := strings.Join(parts[:3], "/")
	dataDir := dataRootDir + pkg
	htmlDir := dataDir + "/html/"
	err := isEntryExists(htmlDir, true)
	if err != nil {
		err = genDoc(parts, pkg, dataDir, htmlDir)
		if err != nil {
			httputil.Error(w, err)
			return
		}
	}

	if len(parts) > 3 {
		file := htmlDir + parts[3]
		if strings.HasSuffix(file, "/") {
			file += "index.html"
		}
		f, err := os.Open(file)
		if err != nil {
			httputil.Error(w, err)
			return
		}
		defer f.Close()
		fi, err := f.Stat()
		if err != nil {
			httputil.Error(w, err)
			return
		}
		serveContent(w, req, pkg, fi.Name(), fi.ModTime(), f)
	} else {
		http.Redirect(w, req, path + "/", 301)
	}
}

func genDoc(parts []string, pkg, dataDir, htmlDir string) (err error) {

	srcDir := srcRootDir + pkg
	repo := "https://github.com/" + parts[1] + "/" + parts[2] + ".git"

	crc := crc32.ChecksumIEEE([]byte(pkg))
	mutex := &mutexs[crc % mutexCount]
	mutex.Lock()
	defer mutex.Unlock()

	err2 := isEntryExists(htmlDir, true)
	if err2 != nil {
		err = cloneRepo(srcDir, repo)
		if err != nil {
			return
		}

		err = os.MkdirAll(dataDir, 0755)
		if err != nil {
			return
		}

		doxyfile := tmpRootDir + "github.com!" + parts[1] + "!" + parts[2] + ".doxyfile"
		err = genDoxyfileFile(doxyfile, &doxyfileConf{
			ProjectName:  parts[2],
			OutputDir:    dataDir,
			InputDir:     srcDir,
			FilePatterns: "*.md *.java *.h *.hpp *.hxx *.py *.php *.rb *.cs *.js *.scala *.go *.lua *.asp",
		})
		if err != nil {
			return
		}

		err = runCmd(doxygenApp, doxyfile)
		if err != nil {
			return
		}
	}
	return
}

func cloneRepo(srcDir string, repo string) (err error) {

	os.RemoveAll(srcDir)
	err = os.MkdirAll(srcDir, 0755)
	if err != nil {
		return
	}

	return runCmd("git", "clone", "--depth=50", repo, srcDir)
}

func runCmd(command string, args ...string) (err error) {

	cmd := exec.Command(command, args...)

	var out bytes.Buffer
	cmd.Stderr = &out

	err = cmd.Run()
	if err == nil {
		return
	}

	emsg := out.String()
	if emsg != "" {
		return errors.New(emsg)
	}
	return err
}

func isEntryExists(entryPath string, isDir bool) (err error) {

	fi, err := os.Stat(entryPath)
	if err != nil {
		return
	}

	if fi.IsDir() != isDir {
		err = ErrUnmatchedInodeType
		return
	}
	return nil
}

var (
	bindHost = flag.String("http", ":8888", "address that doxygen.io server listen")
)

func main() {

	flag.Parse()

	rootDir := os.Getenv("HOME") + "/.doxygen.io/"
	doxygenApp = os.Getenv("DOXYGEN")
	if doxygenApp == "" {
		doxygenApp = "doxygen"
	}

	dataRootDir = rootDir + "data/"
	refreshRootDir = rootDir + "refresh/"
	srcRootDir = rootDir + "src/"
	tmpRootDir = rootDir + "tmp/"
	os.MkdirAll(tmpRootDir, 0755)

	http.HandleFunc("/-/refresh", handleRefresh)
	http.HandleFunc("/", handleMain)
	err := http.ListenAndServe(*bindHost, nil)
	if err != nil {
		log.Fatal(err)
	}
}

<commit_msg>mainpage: *.dox<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"hash/crc32"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"qiniupkg.com/http/httputil.v2"
	"qiniupkg.com/x/log.v7"
)

const (
	mutexCount = 9973
)

var (
	ErrUnmatchedInodeType = errors.New("unmatched inode type(file or dir)")
	ErrRefreshWithoutPath = httputil.NewError(400, "refresh without path")
	ErrInvalidPkgPath     = httputil.NewError(400, "invalid package path")
)

var (
	doxygenApp string

	dataRootDir string
	srcRootDir  string
	tmpRootDir  string

	refreshRootDir string

	mutexs [mutexCount]sync.Mutex
)

func handleHome(w http.ResponseWriter, req *http.Request) {

}

func handleUnknown(w http.ResponseWriter, req *http.Request) {

}

func handleRefresh(w http.ResponseWriter, req *http.Request) {

	pkg := req.PostFormValue("path")
	if pkg == "" {
		httputil.Error(w, ErrRefreshWithoutPath)
		return
	}

	log.Info("Refresh", pkg)

	err := refresh(pkg)
	if err != nil {
		httputil.Error(w, err)
		return
	}

	http.Redirect(w, req, "/" + pkg + "/", 301)
}

func refresh(pkg string) (err error) {

	dataDir := dataRootDir + pkg
	indexFile := dataDir + "/html/index.html"
	if isRefreshed(indexFile) {
		return nil
	}

	if strings.Index(pkg, "..") >= 0 {
		return ErrInvalidPkgPath
	}

	parts := strings.SplitN(pkg, "/", 4)
	if len(parts) != 3 {
		return ErrInvalidPkgPath
	}

	refreshDir := refreshRootDir + pkg
	refreshHtmlDir := refreshDir + "/html/"
	os.RemoveAll(refreshDir)
	err = genDoc(parts, pkg, refreshDir, refreshHtmlDir)
	if err != nil {
		return
	}

	os.RemoveAll(dataDir)
	return os.Rename(refreshDir, dataDir)
}

func isRefreshed(indexFile string) bool {

	fi, err := os.Stat(indexFile)
	if err != nil {
		return false
	}

	return time.Now().Sub(fi.ModTime()) < 10*time.Second
}

func handleMain(w http.ResponseWriter, req *http.Request) {

	path := req.URL.Path

	if path == "/" {
		handleHome(w, req)
		return
	}

	log.Info("View", path)

	if strings.Index(path, "..") >= 0 {
		handleUnknown(w, req)
		return
	}

	parts := strings.SplitN(path[1:], "/", 4)
	if parts[0] != "github.com" || len(parts) < 3 {
		handleUnknown(w, req)
		return
	}

	pkg := strings.Join(parts[:3], "/")
	dataDir := dataRootDir + pkg
	htmlDir := dataDir + "/html/"
	err := isEntryExists(htmlDir, true)
	if err != nil {
		err = genDoc(parts, pkg, dataDir, htmlDir)
		if err != nil {
			httputil.Error(w, err)
			return
		}
	}

	if len(parts) > 3 {
		file := htmlDir + parts[3]
		if strings.HasSuffix(file, "/") {
			file += "index.html"
		}
		f, err := os.Open(file)
		if err != nil {
			httputil.Error(w, err)
			return
		}
		defer f.Close()
		fi, err := f.Stat()
		if err != nil {
			httputil.Error(w, err)
			return
		}
		serveContent(w, req, pkg, fi.Name(), fi.ModTime(), f)
	} else {
		http.Redirect(w, req, path + "/", 301)
	}
}

func genDoc(parts []string, pkg, dataDir, htmlDir string) (err error) {

	srcDir := srcRootDir + pkg
	repo := "https://github.com/" + parts[1] + "/" + parts[2] + ".git"

	crc := crc32.ChecksumIEEE([]byte(pkg))
	mutex := &mutexs[crc % mutexCount]
	mutex.Lock()
	defer mutex.Unlock()

	err2 := isEntryExists(htmlDir, true)
	if err2 != nil {
		err = cloneRepo(srcDir, repo)
		if err != nil {
			return
		}

		err = os.MkdirAll(dataDir, 0755)
		if err != nil {
			return
		}

		doxyfile := tmpRootDir + "github.com!" + parts[1] + "!" + parts[2] + ".doxyfile"
		err = genDoxyfileFile(doxyfile, &doxyfileConf{
			ProjectName:  parts[2],
			OutputDir:    dataDir,
			InputDir:     srcDir,
			FilePatterns: "*.md *.dox *.java *.h *.hpp *.hxx *.py *.php *.rb *.cs *.js *.scala *.go *.lua *.asp",
		})
		if err != nil {
			return
		}

		err = runCmd(doxygenApp, doxyfile)
		if err != nil {
			return
		}
	}
	return
}

func cloneRepo(srcDir string, repo string) (err error) {

	os.RemoveAll(srcDir)
	err = os.MkdirAll(srcDir, 0755)
	if err != nil {
		return
	}

	return runCmd("git", "clone", "--depth=50", repo, srcDir)
}

func runCmd(command string, args ...string) (err error) {

	cmd := exec.Command(command, args...)

	var out bytes.Buffer
	cmd.Stderr = &out

	err = cmd.Run()
	if err == nil {
		return
	}

	emsg := out.String()
	if emsg != "" {
		return errors.New(emsg)
	}
	return err
}

func isEntryExists(entryPath string, isDir bool) (err error) {

	fi, err := os.Stat(entryPath)
	if err != nil {
		return
	}

	if fi.IsDir() != isDir {
		err = ErrUnmatchedInodeType
		return
	}
	return nil
}

var (
	bindHost = flag.String("http", ":8888", "address that doxygen.io server listen")
)

func main() {

	flag.Parse()

	rootDir := os.Getenv("HOME") + "/.doxygen.io/"
	doxygenApp = os.Getenv("DOXYGEN")
	if doxygenApp == "" {
		doxygenApp = "doxygen"
	}

	dataRootDir = rootDir + "data/"
	refreshRootDir = rootDir + "refresh/"
	srcRootDir = rootDir + "src/"
	tmpRootDir = rootDir + "tmp/"
	os.MkdirAll(tmpRootDir, 0755)

	http.HandleFunc("/-/refresh", handleRefresh)
	http.HandleFunc("/", handleMain)
	err := http.ListenAndServe(*bindHost, nil)
	if err != nil {
		log.Fatal(err)
	}
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"github.com/BenchR267/lbd/lexer"
)

func main() {
	c, err := lexer.StreamFromFile("example/main.lbd")
	if err != nil {
		log.Fatal(err)
	}

	l := lexer.NewLexer(c)
	l.Start()
	for t := range l.NextToken {
		fmt.Println(t)
	}
}
<commit_msg>Use repl in main<commit_after>package main

import "github.com/BenchR267/lbd/repl"

func main() {
	repl.Start()
}
<|endoftext|>


<commit_before>/*
Copyright © 2022 NAME HERE <EMAIL ADDRESS>

*/
package cmd

import (
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/zmap/zdns/internal/util"
	"github.com/zmap/zdns/pkg/zdns"
)

// alookupCmd represents the alookup command
var alookupCmd = &cobra.Command{
	Use:   "alookup",
	Short: "A record lookups that follow CNAME records",
	Long: `alookup will get the information that is typically desired, instead of just
the information that exists in a single record.

Specifically, alookup acts similar to nslookup and will follow CNAME records.`,
	Run: func(cmd *cobra.Command, args []string) {
		GC.Module = strings.ToUpper("mxlookup")
		//TODO: don't think these params exist.
		zdns.Run(GC, cmd.Flags(),
			&Timeout, &IterationTimeout,
			&Class_string, &Servers_string,
			&Config_file, &Localaddr_string,
			&Localif_string, &NanoSeconds)
	},
}

func init() {
	rootCmd.AddCommand(alookupCmd)

	alookupCmd.PersistentFlags().Bool("ipv4-lookup", false, "perform A lookups for each MX server")
	alookupCmd.PersistentFlags().Bool("ipv6-lookup", false, "perform AAAA record lookups for each MX server")

	util.BindFlags(alookupCmd, viper.GetViper(), util.EnvPrefix)
}
<commit_msg>fix alookup module<commit_after>/*
Copyright © 2022 NAME HERE <EMAIL ADDRESS>

*/
package cmd

import (
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/zmap/zdns/internal/util"
	"github.com/zmap/zdns/pkg/zdns"
)

// alookupCmd represents the alookup command
var alookupCmd = &cobra.Command{
	Use:   "alookup",
	Short: "A record lookups that follow CNAME records",
	Long: `alookup will get the information that is typically desired, instead of just
the information that exists in a single record.

Specifically, alookup acts similar to nslookup and will follow CNAME records.`,
	Run: func(cmd *cobra.Command, args []string) {
		GC.Module = strings.ToUpper("alookup")
		//TODO: don't think these params exist.
		zdns.Run(GC, cmd.Flags(),
			&Timeout, &IterationTimeout,
			&Class_string, &Servers_string,
			&Config_file, &Localaddr_string,
			&Localif_string, &NanoSeconds)
	},
}

func init() {
	rootCmd.AddCommand(alookupCmd)

	alookupCmd.PersistentFlags().Bool("ipv4-lookup", false, "perform A lookups for each MX server")
	alookupCmd.PersistentFlags().Bool("ipv6-lookup", false, "perform AAAA record lookups for each MX server")

	util.BindFlags(alookupCmd, viper.GetViper(), util.EnvPrefix)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/tylerb/graceful"
	"github.com/yosisa/sigm"
	"github.com/yosisa/webutil"
)

var (
	dataDir         = flag.String("data-dir", "./data", "Data directory")
	listen          = flag.String("listen", ":8000", "Listen address")
	gracefulTimeout = flag.Duration("graceful-timeout", 10*time.Second, "Wait until force shutdown")
	gcInterval      = flag.Duration("gc-interval", time.Hour, "GC interval for cleaning deleted files")
	accessLog       = flag.String("access-log", "-", "Path to access log file")
)

var (
	accessLogWriter = new(webutil.ConsoleLogWriter)
	middlewares     []*middleware
)

const tombstone = ".restfs-deleted"

type middleware struct {
	priority int
	wrap     func(h http.Handler) http.Handler
}

type byPriority []*middleware

func (x byPriority) Len() int           { return len(x) }
func (x byPriority) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
func (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority }

func registerMiddleware(priority int, wrap func(http.Handler) http.Handler) {
	middlewares = append(middlewares, &middleware{priority: priority, wrap: wrap})
}

type restfs struct {
	dir string
}

func (c *restfs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fullpath := path.Join(c.dir, r.URL.Path)
	var (
		fi  os.FileInfo
		err error
	)
	switch r.Method {
	case "GET":
		s := stat(fullpath)
		if s == nil {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
		if s.IsDir() {
			serveFileList(w, fullpath)
		} else {
			http.ServeFile(w, r, fullpath)
		}
		return
	case "PUT":
		fi, err = os.Stat(fullpath)
		if fi.IsDir() {
			http.Error(w, "Cannot overwrite directory", http.StatusBadRequest)
			return
		}
		err = c.saveFile(fullpath, r.Body)
		r.Body.Close()
	case "DELETE":
		fi, err = os.Stat(fullpath)
		if os.IsNotExist(err) {
			return
		}
		if fi.IsDir() {
			recursive, _ := strconv.ParseBool(r.URL.Query().Get("recursive"))
			if recursive {
				err = c.removeAll(fullpath)
			} else {
				http.Error(w, "Cannot remove directory; forgot recursive=true?", http.StatusBadRequest)
				return
			}
		} else {
			err = c.remove(fullpath)
		}
	default:
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

func (c *restfs) saveFile(fullpath string, r io.Reader) error {
	dir, _ := path.Split(fullpath)
	if err := os.MkdirAll(dir, 0777); err != nil {
		return err
	}
	f, err := os.OpenFile(fullpath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}
	if _, err := io.Copy(f, r); err != nil {
		return err
	}
	return nil
}

func (c *restfs) remove(fullpath string) error {
	f, err := os.Create(fullpath + tombstone)
	if err == nil {
		f.Close()
	}
	return err
}

func (c *restfs) removeAll(fullpath string) error {
	return filepath.Walk(fullpath, func(name string, stat os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if stat.IsDir() || strings.HasSuffix(name, tombstone) {
			return nil
		}
		return c.remove(name)
	})
}

type gc struct {
	dir    string
	invoke chan struct{}
}

func newGC(dir string) *gc {
	g := &gc{
		dir:    dir,
		invoke: make(chan struct{}, 1),
	}
	go g.loop()
	return g
}

func (g *gc) loop() {
	remove := func(s string) error {
		log.Printf("Remove %s", s)
		if err := os.Remove(s); err != nil && !os.IsNotExist(err) {
			return err
		}
		return nil
	}
	for range g.invoke {
		log.Print("GC started")
		start := time.Now()
		err := filepath.Walk(g.dir, func(name string, stat os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if stat.IsDir() || !strings.HasSuffix(name, tombstone) {
				return nil
			}
			fname := name[:len(name)-len(tombstone)]
			fstat, err := os.Stat(fname)
			if err == nil {
				if !fstat.ModTime().After(stat.ModTime()) {
					if err = remove(fname); err != nil {
						return err
					}
				}
				return remove(name)
			} else if os.IsNotExist(err) {
				return remove(name)
			}
			return err
		})
		took := time.Since(start)
		if err == nil {
			log.Printf("GC has finished in %v", took)
		} else {
			log.Printf("GC has aborted in %v with error: %v", took, err)
		}
	}
}

func (g *gc) Start() {
	select {
	case g.invoke <- struct{}{}:
	default:
	}
}

func stat(fullpath string) os.FileInfo {
	astat, err := os.Stat(fullpath)
	if err != nil {
		return nil
	}
	if astat.IsDir() {
		return astat
	}

	bstat, err := os.Stat(fullpath + tombstone)
	if err != nil {
		if os.IsNotExist(err) {
			return astat
		}
		log.Print(err)
		return nil
	}
	if astat.ModTime().After(bstat.ModTime()) {
		return astat
	}
	return nil
}

func serveFileList(w http.ResponseWriter, s string) {
	fis, err := ioutil.ReadDir(s)
	if err != nil {
		log.Print(err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	tombstones := make(map[string]os.FileInfo)
	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			name = name[:len(name)-len(tombstone)]
			tombstones[name] = fi
		}
	}

	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			continue
		}
		if fi.IsDir() {
			name += "/"
		} else if ts := tombstones[name]; ts != nil && !fi.ModTime().After(ts.ModTime()) {
			continue
		}
		fmt.Fprintf(w, "%s\n", name)
	}
}

func openAccessLog() {
	if *accessLog == "-" {
		accessLogWriter.Swap(os.Stdout)
		return
	}
	f, err := os.OpenFile(*accessLog, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		log.Print(err)
		return
	}
	if old := accessLogWriter.Swap(f); old != nil {
		if ic, ok := old.(io.Closer); ok {
			ic.Close()
		}
		log.Print("Reopen access log file")
	}
}

func main() {
	flag.Parse()

	log.Printf("Data directory: %s", *dataDir)
	var h http.Handler = &restfs{*dataDir}

	sort.Sort(sort.Reverse(byPriority(middlewares)))
	for _, m := range middlewares {
		h = m.wrap(h)
	}

	openAccessLog()
	h = webutil.Logger(h, accessLogWriter)
	sigm.Handle(syscall.SIGHUP, openAccessLog)

	g := newGC(*dataDir)
	g.Start()
	sigm.Handle(syscall.SIGUSR1, g.Start)
	if *gcInterval > 0 {
		log.Printf("GC runs every %s", *gcInterval)
		go func() {
			for range time.Tick(*gcInterval) {
				g.Start()
			}
		}()
	}

	log.Printf("Server started at %s", *listen)
	graceful.Run(*listen, *gracefulTimeout, webutil.Recoverer(h, os.Stderr))
}
<commit_msg>Fix panic when putting a new file<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/tylerb/graceful"
	"github.com/yosisa/sigm"
	"github.com/yosisa/webutil"
)

var (
	dataDir         = flag.String("data-dir", "./data", "Data directory")
	listen          = flag.String("listen", ":8000", "Listen address")
	gracefulTimeout = flag.Duration("graceful-timeout", 10*time.Second, "Wait until force shutdown")
	gcInterval      = flag.Duration("gc-interval", time.Hour, "GC interval for cleaning deleted files")
	accessLog       = flag.String("access-log", "-", "Path to access log file")
)

var (
	accessLogWriter = new(webutil.ConsoleLogWriter)
	middlewares     []*middleware
)

const tombstone = ".restfs-deleted"

type middleware struct {
	priority int
	wrap     func(h http.Handler) http.Handler
}

type byPriority []*middleware

func (x byPriority) Len() int           { return len(x) }
func (x byPriority) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
func (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority }

func registerMiddleware(priority int, wrap func(http.Handler) http.Handler) {
	middlewares = append(middlewares, &middleware{priority: priority, wrap: wrap})
}

type restfs struct {
	dir string
}

func (c *restfs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fullpath := path.Join(c.dir, r.URL.Path)
	var (
		fi  os.FileInfo
		err error
	)
	switch r.Method {
	case "GET":
		s := stat(fullpath)
		if s == nil {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
		if s.IsDir() {
			serveFileList(w, fullpath)
		} else {
			http.ServeFile(w, r, fullpath)
		}
		return
	case "PUT":
		fi, err = os.Stat(fullpath)
		if err == nil && fi.IsDir() {
			http.Error(w, "Cannot overwrite directory", http.StatusBadRequest)
			return
		}
		if err == nil || os.IsNotExist(err) {
			err = c.saveFile(fullpath, r.Body)
			r.Body.Close()
		}
	case "DELETE":
		fi, err = os.Stat(fullpath)
		if err == nil {
			if fi.IsDir() {
				recursive, _ := strconv.ParseBool(r.URL.Query().Get("recursive"))
				if recursive {
					err = c.removeAll(fullpath)
				} else {
					http.Error(w, "Cannot remove directory; forgot recursive=true?", http.StatusBadRequest)
					return
				}
			} else {
				err = c.remove(fullpath)
			}
		} else if os.IsNotExist(err) {
			return
		}
	default:
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

func (c *restfs) saveFile(fullpath string, r io.Reader) error {
	dir, _ := path.Split(fullpath)
	if err := os.MkdirAll(dir, 0777); err != nil {
		return err
	}
	f, err := os.OpenFile(fullpath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}
	if _, err := io.Copy(f, r); err != nil {
		return err
	}
	return nil
}

func (c *restfs) remove(fullpath string) error {
	f, err := os.Create(fullpath + tombstone)
	if err == nil {
		f.Close()
	}
	return err
}

func (c *restfs) removeAll(fullpath string) error {
	return filepath.Walk(fullpath, func(name string, stat os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if stat.IsDir() || strings.HasSuffix(name, tombstone) {
			return nil
		}
		return c.remove(name)
	})
}

type gc struct {
	dir    string
	invoke chan struct{}
}

func newGC(dir string) *gc {
	g := &gc{
		dir:    dir,
		invoke: make(chan struct{}, 1),
	}
	go g.loop()
	return g
}

func (g *gc) loop() {
	remove := func(s string) error {
		log.Printf("Remove %s", s)
		if err := os.Remove(s); err != nil && !os.IsNotExist(err) {
			return err
		}
		return nil
	}
	for range g.invoke {
		log.Print("GC started")
		start := time.Now()
		err := filepath.Walk(g.dir, func(name string, stat os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if stat.IsDir() || !strings.HasSuffix(name, tombstone) {
				return nil
			}
			fname := name[:len(name)-len(tombstone)]
			fstat, err := os.Stat(fname)
			if err == nil {
				if !fstat.ModTime().After(stat.ModTime()) {
					if err = remove(fname); err != nil {
						return err
					}
				}
				return remove(name)
			} else if os.IsNotExist(err) {
				return remove(name)
			}
			return err
		})
		took := time.Since(start)
		if err == nil {
			log.Printf("GC has finished in %v", took)
		} else {
			log.Printf("GC has aborted in %v with error: %v", took, err)
		}
	}
}

func (g *gc) Start() {
	select {
	case g.invoke <- struct{}{}:
	default:
	}
}

func stat(fullpath string) os.FileInfo {
	astat, err := os.Stat(fullpath)
	if err != nil {
		return nil
	}
	if astat.IsDir() {
		return astat
	}

	bstat, err := os.Stat(fullpath + tombstone)
	if err != nil {
		if os.IsNotExist(err) {
			return astat
		}
		log.Print(err)
		return nil
	}
	if astat.ModTime().After(bstat.ModTime()) {
		return astat
	}
	return nil
}

func serveFileList(w http.ResponseWriter, s string) {
	fis, err := ioutil.ReadDir(s)
	if err != nil {
		log.Print(err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	tombstones := make(map[string]os.FileInfo)
	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			name = name[:len(name)-len(tombstone)]
			tombstones[name] = fi
		}
	}

	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			continue
		}
		if fi.IsDir() {
			name += "/"
		} else if ts := tombstones[name]; ts != nil && !fi.ModTime().After(ts.ModTime()) {
			continue
		}
		fmt.Fprintf(w, "%s\n", name)
	}
}

func openAccessLog() {
	if *accessLog == "-" {
		accessLogWriter.Swap(os.Stdout)
		return
	}
	f, err := os.OpenFile(*accessLog, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		log.Print(err)
		return
	}
	if old := accessLogWriter.Swap(f); old != nil {
		if ic, ok := old.(io.Closer); ok {
			ic.Close()
		}
		log.Print("Reopen access log file")
	}
}

func main() {
	flag.Parse()

	log.Printf("Data directory: %s", *dataDir)
	var h http.Handler = &restfs{*dataDir}

	sort.Sort(sort.Reverse(byPriority(middlewares)))
	for _, m := range middlewares {
		h = m.wrap(h)
	}

	openAccessLog()
	h = webutil.Logger(h, accessLogWriter)
	sigm.Handle(syscall.SIGHUP, openAccessLog)

	g := newGC(*dataDir)
	g.Start()
	sigm.Handle(syscall.SIGUSR1, g.Start)
	if *gcInterval > 0 {
		log.Printf("GC runs every %s", *gcInterval)
		go func() {
			for range time.Tick(*gcInterval) {
				g.Start()
			}
		}()
	}

	log.Printf("Server started at %s", *listen)
	graceful.Run(*listen, *gracefulTimeout, webutil.Recoverer(h, os.Stderr))
}
<|endoftext|>


<commit_before>package env

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
)

///////////
// Types //
///////////

type (
	CatalogType int
	HaloType    int
	TreeType    int
)

const (
	Gotetra CatalogType = iota
	LGadget2
	ARTIO

	Rockstar HaloType = iota
	NilHalo

	ConsistentTrees TreeType = iota
	NilTree
)

type Environment struct {
	Catalogs
	Halos
	MemoDir string
}

//////////////////
// Info structs //
//////////////////

type ParticleInfo struct {
	SnapshotFormat string
	MemoDir        string

	SnapshotFormatMeanings []string
	ScaleFactorFile        string
	BlockMins, BlockMaxes  []int64
	SnapMin, SnapMax       int64
}

type HaloInfo struct {
	HaloDir, TreeDir   string
	HSnapMin, HSnapMax int64
}

func (info *ParticleInfo) GetColumn(
	i int,
) (col []interface{}, snapAligned bool, err error) {

	m := info.SnapshotFormatMeanings[i]
	switch {
	case m == "ScaleFactor":
		bs, err := ioutil.ReadFile(info.ScaleFactorFile)
		if err != nil {
			return nil, false, err
		}
		text := string(bs)
		lines := strings.Split(text, "\n")

		out := []string{}
		for i := range lines {
			line := strings.Trim(lines[i], " ")
			if len(line) != 0 {
				out = append(out, line)
			}
		}

		if len(out) != int(info.SnapMax-info.SnapMin)+1 {
			return nil, false, fmt.Errorf(
				"%s has %d non-empty lines, but SnapMax = %d and SnapMin = %d.",
				info.ScaleFactorFile, len(out), info.SnapMax, info.SnapMin,
			)
		}

		return anonymize(out), true, nil

	case m == "Snapshot":
		out := make([]int, int(info.SnapMax-info.SnapMin)+1)
		for i := range out {
			out[i] = i + int(info.SnapMin)
		}
		return anonymize(out), true, nil

	case len(m) > 5 && m[:5] == "Block":
		idx := 0
		if len(m) > 5 {
			var err error
			idx, err = strconv.Atoi(m[5:])
			if err != nil {
				return nil, false, err
			}
		}
		out := make([]int, info.BlockMaxes[idx]-info.BlockMins[idx]+1)
		for i := range out {
			out[i] = i + int(info.BlockMins[idx])
		}
		return anonymize(out), false, nil
	}
	panic("Impossible")
}

func anonymize(col interface{}) []interface{} {
	switch xs := col.(type) {
	case []int:
		out := make([]interface{}, len(xs))
		for i := range out {
			out[i] = xs[i]
		}
		return out
	case []string:
		out := make([]interface{}, len(xs))
		for i := range out {
			out[i] = xs[i]
		}
		return out
	}
	panic("Unknown type.")
}

//////////////
// Catalogs //
//////////////

type Catalogs struct {
	CatalogType
	snapMin int
	names   [][]string
}

func (cat *Catalogs) Blocks() int {
	return len(cat.names[0])
}

func (cat *Catalogs) ParticleCatalog(snap, block int) string {
	return cat.names[snap-cat.snapMin][block]
}

///////////
// Halos //
///////////

type Halos struct {
	HaloType
	TreeType
	snapMin    int
	snapOffset int
	names      []string
}

func (h *Halos) HaloCatalog(snap int) string {
	return h.names[snap-h.snapMin]
}

func (h *Halos) SnapOffset() int {
	return h.snapOffset
}

func (h *Halos) InitNilHalo(dir string, snapMin, snapMax int64) error {
	h.HaloType = NilHalo
	h.TreeType = NilTree

	return nil
}

//////////////////////////////////
// Format Argument Manipulation //
//////////////////////////////////

// interleave is a bit hard to explain. It will be easier to understand
// what it's supposed to do by reading the source than it would be to read prose
// trying to explain it precisely.
func interleave(cols [][]interface{}, snapAligned []bool) [][][]interface{} {
	// Check that all snap-aligned columns are same length.
	snaps, nonSnaps := -1, 1
	for i := range cols {
		if snapAligned[i] {
			if snaps == -1 {
				snaps = len(cols[i])
			} else if snaps != len(cols[i]) {
				// This is safe to think of as an internal error.
				panic(fmt.Sprintf("Column %d is snapAligned, but has height "+
					"%d instead of %d.", i, len(cols[i]), snaps))
			}
		} else {
			nonSnaps *= len(cols[i])
		}
	}

	// This is safe to think of as an internal error.
	if snaps == 1 {
		panic("All values in snapAligned are false.")
	}

	out := [][][]interface{}{}

	for snap := 0; snap < snaps; snap++ {
		snapRows := make([][]interface{}, nonSnaps)
		for i := range snapRows {
			snapRows[i] = make([]interface{}, len(cols))
		}

		divSize, modSize := 1, 1

		for col := range cols {
			if snapAligned[col] {
				for row := range snapRows {
					snapRows[row][col] = cols[col][snap]
				}
			} else {
				divSize *= modSize
				modSize = len(cols[col])

				for row := range snapRows {
					snapRows[row][col] = cols[col][(row/divSize)%modSize]
				}
			}
		}

		out = append(out, snapRows)
	}

	return out
}
<commit_msg>Fixed off by one error in cmd/env.GetColumn()<commit_after>package env

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
)

///////////
// Types //
///////////

type (
	CatalogType int
	HaloType    int
	TreeType    int
)

const (
	Gotetra CatalogType = iota
	LGadget2
	ARTIO

	Rockstar HaloType = iota
	NilHalo

	ConsistentTrees TreeType = iota
	NilTree
)

type Environment struct {
	Catalogs
	Halos
	MemoDir string
}

//////////////////
// Info structs //
//////////////////

type ParticleInfo struct {
	SnapshotFormat string
	MemoDir        string

	SnapshotFormatMeanings []string
	ScaleFactorFile        string
	BlockMins, BlockMaxes  []int64
	SnapMin, SnapMax       int64
}

type HaloInfo struct {
	HaloDir, TreeDir   string
	HSnapMin, HSnapMax int64
}

func (info *ParticleInfo) GetColumn(
	i int,
) (col []interface{}, snapAligned bool, err error) {

	m := info.SnapshotFormatMeanings[i]
	switch {
	case m == "ScaleFactor":
		bs, err := ioutil.ReadFile(info.ScaleFactorFile)
		if err != nil {
			return nil, false, err
		}
		text := string(bs)
		lines := strings.Split(text, "\n")

		out := []string{}
		for i := range lines {
			line := strings.Trim(lines[i], " ")
			if len(line) != 0 {
				out = append(out, line)
			}
		}

		if len(out) != int(info.SnapMax-info.SnapMin)+1 {
			return nil, false, fmt.Errorf(
				"%s has %d non-empty lines, but SnapMax = %d and SnapMin = %d.",
				info.ScaleFactorFile, len(out), info.SnapMax, info.SnapMin,
			)
		}

		return anonymize(out), true, nil

	case m == "Snapshot":
		out := make([]int, int(info.SnapMax-info.SnapMin)+1)
		for i := range out {
			out[i] = i + int(info.SnapMin)
		}
		return anonymize(out), true, nil

	case len(m) >= 5 && m[:5] == "Block":
		idx := 0
		if len(m) > 5 {
			var err error
			idx, err = strconv.Atoi(m[5:])
			if err != nil {
				return nil, false, err
			}
		}
		out := make([]int, info.BlockMaxes[idx]-info.BlockMins[idx]+1)
		for i := range out {
			out[i] = i + int(info.BlockMins[idx])
		}
		return anonymize(out), false, nil
	}
	panic("Impossible")
}

func anonymize(col interface{}) []interface{} {
	switch xs := col.(type) {
	case []int:
		out := make([]interface{}, len(xs))
		for i := range out {
			out[i] = xs[i]
		}
		return out
	case []string:
		out := make([]interface{}, len(xs))
		for i := range out {
			out[i] = xs[i]
		}
		return out
	}
	panic("Unknown type.")
}

//////////////
// Catalogs //
//////////////

type Catalogs struct {
	CatalogType
	snapMin int
	names   [][]string
}

func (cat *Catalogs) Blocks() int {
	return len(cat.names[0])
}

func (cat *Catalogs) ParticleCatalog(snap, block int) string {
	return cat.names[snap-cat.snapMin][block]
}

///////////
// Halos //
///////////

type Halos struct {
	HaloType
	TreeType
	snapMin    int
	snapOffset int
	names      []string
}

func (h *Halos) HaloCatalog(snap int) string {
	return h.names[snap-h.snapMin]
}

func (h *Halos) SnapOffset() int {
	return h.snapOffset
}

func (h *Halos) InitNilHalo(dir string, snapMin, snapMax int64) error {
	h.HaloType = NilHalo
	h.TreeType = NilTree

	return nil
}

//////////////////////////////////
// Format Argument Manipulation //
//////////////////////////////////

// interleave is a bit hard to explain. It will be easier to understand
// what it's supposed to do by reading the source than it would be to read prose
// trying to explain it precisely.
func interleave(cols [][]interface{}, snapAligned []bool) [][][]interface{} {
	// Check that all snap-aligned columns are same length.
	snaps, nonSnaps := -1, 1
	for i := range cols {
		if snapAligned[i] {
			if snaps == -1 {
				snaps = len(cols[i])
			} else if snaps != len(cols[i]) {
				// This is safe to think of as an internal error.
				panic(fmt.Sprintf("Column %d is snapAligned, but has height "+
					"%d instead of %d.", i, len(cols[i]), snaps))
			}
		} else {
			nonSnaps *= len(cols[i])
		}
	}

	// This is safe to think of as an internal error.
	if snaps == 1 {
		panic("All values in snapAligned are false.")
	}

	out := [][][]interface{}{}

	for snap := 0; snap < snaps; snap++ {
		snapRows := make([][]interface{}, nonSnaps)
		for i := range snapRows {
			snapRows[i] = make([]interface{}, len(cols))
		}

		divSize, modSize := 1, 1

		for col := range cols {
			if snapAligned[col] {
				for row := range snapRows {
					snapRows[row][col] = cols[col][snap]
				}
			} else {
				divSize *= modSize
				modSize = len(cols[col])

				for row := range snapRows {
					snapRows[row][col] = cols[col][(row/divSize)%modSize]
				}
			}
		}

		out = append(out, snapRows)
	}

	return out
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	TITLE  string `json:"title"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	id := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			generateName()
		}
	}
	db.Close()

	return id

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string, title string, expiry string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, title, hash, data, delkey from pastebin where hash=?", sha)
	if err != sql.ErrNoRows {
		for query.Next() {
			var id, title, hash, paste, delkey string
			err := query.Scan(&id, &title, &hash, &paste, &delkey)
			check(err)
			url := ADDRESS + "/p/" + id
			return []string{id, title, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	now := time.Now()
	var expiryTime string

	switch expiry {
	case "5 minutes":
		expiryTime = now.Add(time.Minute * 5).Format("2006-01-02 15:04:05")
		break

	case "1 hour":
		expiryTime = now.Add(time.Hour + 1).Format("2006-01-02 15:04:05")
		break

	case "1 day":
		expiryTime = now.Add(time.Hour * 24 * 1).Format("2006-01-02 15:04:05")
		break

	case "1 week":
		expiryTime = now.Add(time.Hour * 24 * 7).Format("2006-01-02 15:04:05")
		break

	case "1 month":
		expiryTime = now.Add(time.Hour * 24 * 30).Format("2006-01-02 15:04:05")
		break

	case "1 year":
		expiryTime = now.Add(time.Hour * 24 * 365).Format("2006-01-02 15:04:05")
		break

	case "forever":
		expiryTime = now.Add(time.Hour * 24 * (365 * 20)).Format("2006-01-02 15:04:05")
		break

	default:
		expiryTime = now.Add(time.Hour * 24 * (365 * 20)).Format("2006-01-02 15:04:05")
		break

	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, title, hash, data, delkey, expiry) values(?,?,?,?,?,?)")
	check(err)
	if title == "" {
		_, err = stmt.Exec(id, id, sha, paste, delKey, expiryTime)
		check(err)
	} else {
		_, err = stmt.Exec(id, html.EscapeString(title), sha, paste, delKey, expiryTime)
		check(err)
	}
	db.Close()
	return []string{id, title, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	_, err = res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		title := r.FormValue("title")
		expiry := r.FormValue("expiry")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		values := save(paste, lang, title, expiry)
		b := &Response{
			ID:     values[0],
			TITLE:  values[1],
			HASH:   values[2],
			URL:    values[3],
			SIZE:   len(values[4]),
			DELKEY: values[5],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "plain/text")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

func getPaste(paste string, lang string) (string, string) {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var title, s string
	var expiry string
	err = db.QueryRow("select title, data, expiry from pastebin where id=?", param1).Scan(&title, &s, &expiry)
	check(err)
	if time.Now().Format("2006-01-02 15:04:05") > expiry {
		stmt, err := db.Prepare("delete from pastebin where id=?")
		check(err)
		_, err = stmt.Exec(param1)
		check(err)
		return "Error invalid paste", ""
	}
	db.Close()

	if err == sql.ErrNoRows {
		return "Error invalid paste", ""
	} else {
		if lang == "" {
			return html.UnescapeString(s), html.UnescapeString(title)
		} else {
			high, err := highlight(s, lang)
			check(err)
			return high, html.UnescapeString(title)

		}
	}

}

var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s, title := getPaste(paste, lang)
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	if lang == "" {

		err := templates.ExecuteTemplate(w, "paste.html", p)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		fmt.Fprintf(w, string(syntax), p.Title, p.Title, s, p.Home, p.Download, p.Raw, p.Clone)

	}
}

func cloneHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, title := getPaste(paste, "")
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	err := templates.ExecuteTemplate(w, "clone.html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

}
func downloadHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := getPaste(paste, "")
	w.Header().Set("Content-Disposition", "attachment; filename="+paste)
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	io.WriteString(w, s)

}
func rawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := getPaste(paste, "")
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/raw/{pasteId}", rawHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", pasteHandler)
	router.HandleFunc("/clone/{pasteId}", cloneHandler)
	router.HandleFunc("/download/{pasteId}", downloadHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Only open db once<commit_after>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

var (
	db *db.SQL
)

type Response struct {
	ID     string `json:"id"`
	TITLE  string `json:"title"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

func setup() {
	var err error
	db, err = sql.Open("mysql", DATABASE)
	if err != nil {
		log.Fatal(err.Error())
	}

}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	id := uniuri.NewLen(LENGTH)

	query, err := db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			generateName()
		}
	}
	db.Close()

	return id

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string, title string, expiry string) []string {

	sha := hash(raw)
	query, err := db.Query("select id, title, hash, data, delkey from pastebin where hash=?", sha)
	if err != sql.ErrNoRows {
		for query.Next() {
			var id, title, hash, paste, delkey string
			err := query.Scan(&id, &title, &hash, &paste, &delkey)
			check(err)
			url := ADDRESS + "/p/" + id
			return []string{id, title, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	now := time.Now()
	var expiryTime string

	switch expiry {
	case "5 minutes":
		expiryTime = now.Add(time.Minute * 5).Format("2006-01-02 15:04:05")
		break

	case "1 hour":
		expiryTime = now.Add(time.Hour + 1).Format("2006-01-02 15:04:05")
		break

	case "1 day":
		expiryTime = now.Add(time.Hour * 24 * 1).Format("2006-01-02 15:04:05")
		break

	case "1 week":
		expiryTime = now.Add(time.Hour * 24 * 7).Format("2006-01-02 15:04:05")
		break

	case "1 month":
		expiryTime = now.Add(time.Hour * 24 * 30).Format("2006-01-02 15:04:05")
		break

	case "1 year":
		expiryTime = now.Add(time.Hour * 24 * 365).Format("2006-01-02 15:04:05")
		break

	case "forever":
		expiryTime = now.Add(time.Hour * 24 * (365 * 20)).Format("2006-01-02 15:04:05")
		break

	default:
		expiryTime = now.Add(time.Hour * 24 * (365 * 20)).Format("2006-01-02 15:04:05")
		break

	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, title, hash, data, delkey, expiry) values(?,?,?,?,?,?)")
	check(err)
	if title == "" {
		_, err = stmt.Exec(id, id, sha, paste, delKey, expiryTime)
		check(err)
	} else {
		_, err = stmt.Exec(id, html.EscapeString(title), sha, paste, delKey, expiryTime)
		check(err)
	}
	db.Close()
	return []string{id, title, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	_, err = res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		title := r.FormValue("title")
		expiry := r.FormValue("expiry")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		values := save(paste, lang, title, expiry)
		b := &Response{
			ID:     values[0],
			TITLE:  values[1],
			HASH:   values[2],
			URL:    values[3],
			SIZE:   len(values[4]),
			DELKEY: values[5],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "plain/text")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

func getPaste(paste string, lang string) (string, string) {
	param1 := html.EscapeString(paste)
	var title, s string
	var expiry string
	err := db.QueryRow("select title, data, expiry from pastebin where id=?", param1).Scan(&title, &s, &expiry)
	check(err)
	if time.Now().Format("2006-01-02 15:04:05") > expiry {
		stmt, err := db.Prepare("delete from pastebin where id=?")
		check(err)
		_, err = stmt.Exec(param1)
		check(err)
		return "Error invalid paste", ""
	}
	db.Close()

	if err == sql.ErrNoRows {
		return "Error invalid paste", ""
	} else {
		if lang == "" {
			return html.UnescapeString(s), html.UnescapeString(title)
		} else {
			high, err := highlight(s, lang)
			check(err)
			return high, html.UnescapeString(title)

		}
	}

}

var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s, title := getPaste(paste, lang)
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	if lang == "" {

		err := templates.ExecuteTemplate(w, "paste.html", p)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		fmt.Fprintf(w, string(syntax), p.Title, p.Title, s, p.Home, p.Download, p.Raw, p.Clone)

	}
}

func cloneHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, title := getPaste(paste, "")
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	err := templates.ExecuteTemplate(w, "clone.html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

}
func downloadHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := getPaste(paste, "")
	w.Header().Set("Content-Disposition", "attachment; filename="+paste)
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	io.WriteString(w, s)

}
func rawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := getPaste(paste, "")
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/raw/{pasteId}", rawHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", pasteHandler)
	router.HandleFunc("/clone/{pasteId}", cloneHandler)
	router.HandleFunc("/download/{pasteId}", downloadHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/snippet"
)

var historyCmd = &cobra.Command{
	Use:   "history",
	Short: "Search keyword history",
	Long:  "Search keyword history",
	Run: func(cmd *cobra.Command, args []string) {
		var snippets snippet.Snippets
		snippets.Load()
		for _, snip := range snippets.Snippets {
			fmt.Println(color.YellowString("url: " + snip.Url))
			fmt.Println(color.GreenString("title: " + snip.Title))
			fmt.Println(color.CyanString("keyword: " + snip.SearchKeyword))
			fmt.Println(color.WhiteString("-------------------------------"))
		}
	},
}

func init() {
	RootCmd.AddCommand(historyCmd)
}
<commit_msg>Modified history indent<commit_after>package cmd

import (
	"fmt"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/snippet"
)

var historyCmd = &cobra.Command{
	Use:   "history",
	Short: "Search keyword history",
	Long:  "Search keyword history",
	Run: func(cmd *cobra.Command, args []string) {
		var snippets snippet.Snippets
		snippets.Load()
		for _, snip := range snippets.Snippets {
			fmt.Println(color.YellowString("url:     " + snip.Url))
			fmt.Println(color.GreenString("title: 	 " + snip.Title))
			fmt.Println(color.CyanString("keyword: " + snip.SearchKeyword))
			fmt.Println(color.WhiteString("-------------------------------"))
		}
	},
}

func init() {
	RootCmd.AddCommand(historyCmd)
}
<|endoftext|>


<commit_before>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// ht generates HTTP requests and checks the received responses.
//
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/vdobler/ht/ht"
)

// A Command is one of the subcommands of ht.
type Command struct {
	// Run the command.
	// The args are the arguments after the command name.
	Run func(cmd *Command, suites []*ht.Suite)

	Usage       string       // must start with command name
	Description string       // short description for ' go help'
	Help        string       // the output of 'ht help <cmd>'
	Flag        flag.FlagSet // the flags for this command
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.Usage
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) usage() {
	fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.Usage)
	fmt.Fprintf(os.Stderr, "%s\n", c.Help)
	os.Exit(2)
}

// Commands lists the available commands and help topics.
// The order here is the order in which they are printed by 'go help'.
var commands = []*Command{
	cmdList,
	cmdRun,
	cmdExec,
	cmdBench,
	cmdPerf,
}

func usage() {
	formatedCmdList := ""

	for _, cmd := range commands {
		formatedCmdList += fmt.Sprintf("    %-8s %s\n",
			cmd.Name(), cmd.Description)
	}

	fmt.Printf(`Ht is a tool to generate http request and test the response.

Usage:

    ht <command> [flags...] <suite>...

The commands are:
%s
Run  ht help <command> to display the usage of <command>.

Tests IDs have the following format <suite>.<type><test> with <suite> and
<test> the sequential numbers of the suite and the test inside the suite.
Type is either empty, "u" for setUp test or "d" for tearDown tests. <test>
maybe a single number like "3" or a range like "3-7".
`, formatedCmdList)
	os.Exit(2)
}

// Variables which can be set via the command line. Statisfied flag.Value interface.
type cmdlVar map[string]string

func (v cmdlVar) String() string { return "" }
func (v cmdlVar) Set(s string) error {
	part := strings.SplitN(s, "=", 2)
	if len(part) != 2 {
		return fmt.Errorf("Bad argument '%s' to -D commandline parameter", s)
	}
	v[part[0]] = part[1]
	return nil
}

// Includepath which can be set via the command line. Statisfied flag.Value interface.
type cmdlIncl []string

func (i *cmdlIncl) String() string { return "" }
func (i *cmdlIncl) Set(s string) error {
	s = strings.TrimRight(s, "/")
	*i = append(*i, s)
	return nil
}

// The common flags.
var (
	variablesFlag cmdlVar = make(cmdlVar) // flag -D
	onlyFlag      string
	skipFlag      string
	verbosity     int
)

func addVariablesFlag(fs *flag.FlagSet) {
	fs.Var(variablesFlag, "D", "set `parameter=value`")
}

func addOnlyFlag(fs *flag.FlagSet) {
	fs.StringVar(&onlyFlag, "only", "", "run only tests given by `testID`")
}

func addSkipFlag(fs *flag.FlagSet) {
	fs.StringVar(&skipFlag, "skip", "", "skip tests identified by `testID`")
}

func addVerbosityFlag(fs *flag.FlagSet) {
	fs.IntVar(&verbosity, "verbosity", -99, "verbosity to `level`")
}

func main() {
	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		return
	}
	var suites []*ht.Suite
	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.Flag.Usage = func() { cmd.usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			if cmd.Name() == "run" {
				suites = loadTests(args)
			} else {
				suites = loadSuites(args)
			}
			cmd.Run(cmd, suites)
			return
		}
	}

	fmt.Fprintf(os.Stderr, "go: unknown subcommand %q\nRun 'go help' for usage.\n",
		args[0])
	os.Exit(2)
}

// The help command.
func help(args []string) {
	if len(args) == 0 {
		usage() // TODO: this is not a failure
	}
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "usage: ht help <command>\n\nToo many arguments given.\n")
		os.Exit(2)
	}

	arg := args[0]

	for _, cmd := range commands {
		if cmd.Name() == arg {
			fmt.Printf(`Usage:

    ht %s
%s
Flags:
`, cmd.Usage, cmd.Help)
			cmd.Flag.PrintDefaults()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic %#q.  Run 'ht help'.\n", arg)
	os.Exit(2) // failed at 'go help cmd'
}

func loadSuites(args []string) []*ht.Suite {
	var suites []*ht.Suite

	logger := log.New(os.Stdout, "", log.LstdFlags)

	// Handle -only and -skip flags.
	only, skip := splitTestIDs(onlyFlag), splitTestIDs(skipFlag)

	// Input and setup suites from command line arguments.
	for _, s := range args {
		suite, err := ht.LoadSuite(s)
		if err != nil {
			log.Fatalf("Cannot read suite %q: %s", s, err)
		}
		for varName, varVal := range variablesFlag {
			suite.Variables[varName] = varVal
		}
		suite.Log = logger
		err = suite.Prepare()
		if err != nil {
			log.Fatal(err.Error())
		}
		if verbosity != -99 {
			for i := range suite.Setup {
				suite.Setup[i].Verbosity = verbosity
			}
			for i := range suite.Tests {
				suite.Tests[i].Verbosity = verbosity
			}
			for i := range suite.Teardown {
				suite.Teardown[i].Verbosity = verbosity
			}
		}
		suites = append(suites, suite)
	}

	// Disable tests based on the -only and -skip flags.
	for sNo, suite := range suites {
		for tNo, test := range suite.Setup {
			shouldRun(test, fmt.Sprintf("%d.U%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Tests {
			shouldRun(test, fmt.Sprintf("%d.%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Teardown {
			shouldRun(test, fmt.Sprintf("%d.D%d", sNo+1, tNo+1), only, skip)
		}
	}

	return suites
}

// loadTests loads single Test and combines them into an artificial
// Suite, ready for execution. Unrolling happens, but only the first
// unrolled test gets included into the suite.
func loadTests(args []string) []*ht.Suite {
	logger := log.New(os.Stdout, "", log.LstdFlags)
	suite := &ht.Suite{
		Name: "Autogenerated suite for run",
		Log:  logger,
	}

	// Input and setup tests from command line arguments.
	for _, t := range args {
		tests, err := ht.LoadTest(t)
		if err != nil {
			log.Fatalf("Cannot read test %q: %s", t, err)
		}
		suite.Tests = append(suite.Tests, tests[0])
	}

	err := suite.Prepare()
	if err != nil {
		log.Fatal(err.Error())
	}
	if verbosity != -99 {
		for i := range suite.Tests {
			suite.Tests[i].Verbosity = verbosity
		}
	}

	for varName, varVal := range variablesFlag {
		suite.Variables[varName] = varVal
	}

	suites := []*ht.Suite{suite}
	return suites
}

// shouldRun disables t if needed.
func shouldRun(t *ht.Test, id string, only, skip map[string]struct{}) {
	if _, ok := skip[id]; ok {
		t.Poll.Max = -1
		log.Printf("Skipping test %s %q", id, t.Name)
		return
	}
	if _, ok := only[id]; !ok && len(only) > 0 {
		t.Poll.Max = -1
		log.Printf("Not running test %s %q", id, t.Name)
		return
	}
}

func splitTestIDs(f string) (ids map[string]struct{}) {
	ids = make(map[string]struct{})
	if len(f) == 0 {
		return
	}
	fp := strings.Split(f, ",")
	for _, x := range fp {
		xp := strings.SplitN(x, ".", 2)
		s, t := "1", xp[0]
		if len(xp) == 2 {
			s, t = xp[0], xp[1]
		}
		typ := ""
		switch t[0] {
		case 'U', 'u', 'S', 's':
			typ = "U"
			t = t[1:]
		case 'D', 'd', 'T', 't':
			typ = "D"
			t = t[1:]
		default:
			typ = ""
		}

		sNo := mustAtoi(s)
		beg, end := 1, 99
		if i := strings.Index(t, "-"); i > -1 {
			if i > 0 {
				beg = mustAtoi(t[:i])
			}
			if i < len(t)-1 {
				end = mustAtoi(t[i+1:])
			}
		} else {
			beg = mustAtoi(t)
			end = beg
		}
		for tNo := beg; tNo <= end; tNo++ {
			id := fmt.Sprintf("%d.%s%d", sNo, typ, tNo)
			ids[id] = struct{}{}
		}
	}
	return ids
}

func mustAtoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		log.Fatalf("%s", err.Error())
	}
	return n
}

// add current working direcory to end of include path slice if not already
// there.
func addCWD(i *cmdlIncl) {
	for _, p := range *i {
		if p == "." {
			return
		}
	}
	*i = append(*i, ".")
}
<commit_msg>cmd/ht: disable performance measurements<commit_after>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// ht generates HTTP requests and checks the received responses.
//
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/vdobler/ht/ht"
)

// A Command is one of the subcommands of ht.
type Command struct {
	// Run the command.
	// The args are the arguments after the command name.
	Run func(cmd *Command, suites []*ht.Suite)

	Usage       string       // must start with command name
	Description string       // short description for ' go help'
	Help        string       // the output of 'ht help <cmd>'
	Flag        flag.FlagSet // the flags for this command
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.Usage
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) usage() {
	fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.Usage)
	fmt.Fprintf(os.Stderr, "%s\n", c.Help)
	os.Exit(2)
}

// Commands lists the available commands and help topics.
// The order here is the order in which they are printed by 'go help'.
var commands = []*Command{
	cmdList,
	cmdRun,
	cmdExec,
	cmdBench,
	// cmdPerf,
}

func usage() {
	formatedCmdList := ""

	for _, cmd := range commands {
		formatedCmdList += fmt.Sprintf("    %-8s %s\n",
			cmd.Name(), cmd.Description)
	}

	fmt.Printf(`Ht is a tool to generate http request and test the response.

Usage:

    ht <command> [flags...] <suite>...

The commands are:
%s
Run  ht help <command> to display the usage of <command>.

Tests IDs have the following format <suite>.<type><test> with <suite> and
<test> the sequential numbers of the suite and the test inside the suite.
Type is either empty, "u" for setUp test or "d" for tearDown tests. <test>
maybe a single number like "3" or a range like "3-7".
`, formatedCmdList)
	os.Exit(2)
}

// Variables which can be set via the command line. Statisfied flag.Value interface.
type cmdlVar map[string]string

func (v cmdlVar) String() string { return "" }
func (v cmdlVar) Set(s string) error {
	part := strings.SplitN(s, "=", 2)
	if len(part) != 2 {
		return fmt.Errorf("Bad argument '%s' to -D commandline parameter", s)
	}
	v[part[0]] = part[1]
	return nil
}

// Includepath which can be set via the command line. Statisfied flag.Value interface.
type cmdlIncl []string

func (i *cmdlIncl) String() string { return "" }
func (i *cmdlIncl) Set(s string) error {
	s = strings.TrimRight(s, "/")
	*i = append(*i, s)
	return nil
}

// The common flags.
var (
	variablesFlag cmdlVar = make(cmdlVar) // flag -D
	onlyFlag      string
	skipFlag      string
	verbosity     int
)

func addVariablesFlag(fs *flag.FlagSet) {
	fs.Var(variablesFlag, "D", "set `parameter=value`")
}

func addOnlyFlag(fs *flag.FlagSet) {
	fs.StringVar(&onlyFlag, "only", "", "run only tests given by `testID`")
}

func addSkipFlag(fs *flag.FlagSet) {
	fs.StringVar(&skipFlag, "skip", "", "skip tests identified by `testID`")
}

func addVerbosityFlag(fs *flag.FlagSet) {
	fs.IntVar(&verbosity, "verbosity", -99, "verbosity to `level`")
}

func main() {
	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		return
	}
	var suites []*ht.Suite
	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.Flag.Usage = func() { cmd.usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			if cmd.Name() == "run" {
				suites = loadTests(args)
			} else {
				suites = loadSuites(args)
			}
			cmd.Run(cmd, suites)
			return
		}
	}

	fmt.Fprintf(os.Stderr, "go: unknown subcommand %q\nRun 'go help' for usage.\n",
		args[0])
	os.Exit(2)
}

// The help command.
func help(args []string) {
	if len(args) == 0 {
		usage() // TODO: this is not a failure
	}
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "usage: ht help <command>\n\nToo many arguments given.\n")
		os.Exit(2)
	}

	arg := args[0]

	for _, cmd := range commands {
		if cmd.Name() == arg {
			fmt.Printf(`Usage:

    ht %s
%s
Flags:
`, cmd.Usage, cmd.Help)
			cmd.Flag.PrintDefaults()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic %#q.  Run 'ht help'.\n", arg)
	os.Exit(2) // failed at 'go help cmd'
}

func loadSuites(args []string) []*ht.Suite {
	var suites []*ht.Suite

	logger := log.New(os.Stdout, "", log.LstdFlags)

	// Handle -only and -skip flags.
	only, skip := splitTestIDs(onlyFlag), splitTestIDs(skipFlag)

	// Input and setup suites from command line arguments.
	for _, s := range args {
		suite, err := ht.LoadSuite(s)
		if err != nil {
			log.Fatalf("Cannot read suite %q: %s", s, err)
		}
		for varName, varVal := range variablesFlag {
			suite.Variables[varName] = varVal
		}
		suite.Log = logger
		err = suite.Prepare()
		if err != nil {
			log.Fatal(err.Error())
		}
		if verbosity != -99 {
			for i := range suite.Setup {
				suite.Setup[i].Verbosity = verbosity
			}
			for i := range suite.Tests {
				suite.Tests[i].Verbosity = verbosity
			}
			for i := range suite.Teardown {
				suite.Teardown[i].Verbosity = verbosity
			}
		}
		suites = append(suites, suite)
	}

	// Disable tests based on the -only and -skip flags.
	for sNo, suite := range suites {
		for tNo, test := range suite.Setup {
			shouldRun(test, fmt.Sprintf("%d.U%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Tests {
			shouldRun(test, fmt.Sprintf("%d.%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Teardown {
			shouldRun(test, fmt.Sprintf("%d.D%d", sNo+1, tNo+1), only, skip)
		}
	}

	return suites
}

// loadTests loads single Test and combines them into an artificial
// Suite, ready for execution. Unrolling happens, but only the first
// unrolled test gets included into the suite.
func loadTests(args []string) []*ht.Suite {
	logger := log.New(os.Stdout, "", log.LstdFlags)
	suite := &ht.Suite{
		Name: "Autogenerated suite for run",
		Log:  logger,
	}

	// Input and setup tests from command line arguments.
	for _, t := range args {
		tests, err := ht.LoadTest(t)
		if err != nil {
			log.Fatalf("Cannot read test %q: %s", t, err)
		}
		suite.Tests = append(suite.Tests, tests[0])
	}

	err := suite.Prepare()
	if err != nil {
		log.Fatal(err.Error())
	}
	if verbosity != -99 {
		for i := range suite.Tests {
			suite.Tests[i].Verbosity = verbosity
		}
	}

	for varName, varVal := range variablesFlag {
		suite.Variables[varName] = varVal
	}

	suites := []*ht.Suite{suite}
	return suites
}

// shouldRun disables t if needed.
func shouldRun(t *ht.Test, id string, only, skip map[string]struct{}) {
	if _, ok := skip[id]; ok {
		t.Poll.Max = -1
		log.Printf("Skipping test %s %q", id, t.Name)
		return
	}
	if _, ok := only[id]; !ok && len(only) > 0 {
		t.Poll.Max = -1
		log.Printf("Not running test %s %q", id, t.Name)
		return
	}
}

func splitTestIDs(f string) (ids map[string]struct{}) {
	ids = make(map[string]struct{})
	if len(f) == 0 {
		return
	}
	fp := strings.Split(f, ",")
	for _, x := range fp {
		xp := strings.SplitN(x, ".", 2)
		s, t := "1", xp[0]
		if len(xp) == 2 {
			s, t = xp[0], xp[1]
		}
		typ := ""
		switch t[0] {
		case 'U', 'u', 'S', 's':
			typ = "U"
			t = t[1:]
		case 'D', 'd', 'T', 't':
			typ = "D"
			t = t[1:]
		default:
			typ = ""
		}

		sNo := mustAtoi(s)
		beg, end := 1, 99
		if i := strings.Index(t, "-"); i > -1 {
			if i > 0 {
				beg = mustAtoi(t[:i])
			}
			if i < len(t)-1 {
				end = mustAtoi(t[i+1:])
			}
		} else {
			beg = mustAtoi(t)
			end = beg
		}
		for tNo := beg; tNo <= end; tNo++ {
			id := fmt.Sprintf("%d.%s%d", sNo, typ, tNo)
			ids[id] = struct{}{}
		}
	}
	return ids
}

func mustAtoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		log.Fatalf("%s", err.Error())
	}
	return n
}

// add current working direcory to end of include path slice if not already
// there.
func addCWD(i *cmdlIncl) {
	for _, p := range *i {
		if p == "." {
			return
		}
	}
	*i = append(*i, ".")
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/spf13/cobra"

	"github.com/ory/hydra/cmd/cli"
	"github.com/ory/x/configx"
)

func NewJanitorCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "janitor <database-url>",
		Short: "Clean the database of old tokens and login/consent requests",
		Long: `This command will cleanup any expired oauth2 tokens as well as login/consent requests.

### Warning ###

This command is in beta. Proceed with caution!

This is a destructive command and will purge data directly from the database.
Please use this command with caution if you need to keep historic data for any reason.

###############

Janitor can be used in several ways.

1. By passing the database connection string (DSN) as an argument
Pass the database url (dsn) as an argument to janitor. E.g. janitor <database-url>

2. By passing the DSN as an environment variable
	export DSN=...
	janitor -e

3. By passing a configuration file containing the DSN
janitor -c /path/to/conf.yml

4. Extra *optional* parameters can also be added such as
janitor <database-url> --keep-if-younger 23h --access-lifespan 1h --refresh-lifespan 40h --consent-request-lifespan 10m

5. Running only a certain cleanup
janitor <database-url> --tokens

or

janitor <database-url> --requests

or both

janitor <database-url> --tokens --requests
`,
		RunE: cli.NewHandler().Janitor.RunE,
		Args: cli.NewHandler().Janitor.Args,
	}
	cmd.Flags().Duration(cli.KeepIfYounger, 0, "Keep database records that are younger than a specified duration e.g. 1s, 1m, 1h.")
	cmd.Flags().Duration(cli.AccessLifespan, 0, "Set the access token lifespan e.g. 1s, 1m, 1h.")
	cmd.Flags().Duration(cli.RefreshLifespan, 0, "Set the refresh token lifespan e.g. 1s, 1m, 1h.")
	cmd.Flags().Duration(cli.ConsentRequestLifespan, 0, "Set the login/consent request lifespan e.g. 1s, 1m, 1h")
	cmd.Flags().Bool(cli.OnlyRequests, false, "This will only run the cleanup on requests and will skip token cleanup.")
	cmd.Flags().Bool(cli.OnlyTokens, false, "This will only run the cleanup on tokens and will skip requests cleanup.")
	cmd.Flags().BoolP(cli.ReadFromEnv, "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
	configx.RegisterFlags(cmd.PersistentFlags())
	return cmd

}
<commit_msg>fix: update janitor help<commit_after>package cmd

import (
	"github.com/spf13/cobra"

	"github.com/ory/hydra/cmd/cli"
	"github.com/ory/x/configx"
)

func NewJanitorCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "janitor [<database-url>]",
		Short: "Clean the database of old tokens and login/consent requests",
		Long: `This command will cleanup any expired oauth2 tokens as well as login/consent requests.

### Warning ###

This command is in beta. Proceed with caution!

This is a destructive command and will purge data directly from the database.
Please use this command with caution if you need to keep historic data for any reason.

###############

Janitor can be used in several ways.

1. By passing the database connection string (DSN) as an argument
   Pass the database url (dsn) as an argument to janitor. E.g. janitor <database-url>
2. By passing the DSN as an environment variable

		export DSN=...
		janitor -e

3. By passing a configuration file containing the DSN
   janitor -c /path/to/conf.yml
4. Extra *optional* parameters can also be added such as

		janitor --keep-if-younger 23h --access-lifespan 1h --refresh-lifespan 40h --consent-request-lifespan 10m ...

5. Running only a certain cleanup

		janitor --tokens ...

   or

		janitor --requests ...

   or both

		janitor --tokens --requests ...
`,
		RunE: cli.NewHandler().Janitor.RunE,
		Args: cli.NewHandler().Janitor.Args,
	}
	cmd.Flags().Duration(cli.KeepIfYounger, 0, "Keep database records that are younger than a specified duration e.g. 1s, 1m, 1h.")
	cmd.Flags().Duration(cli.AccessLifespan, 0, "Set the access token lifespan e.g. 1s, 1m, 1h.")
	cmd.Flags().Duration(cli.RefreshLifespan, 0, "Set the refresh token lifespan e.g. 1s, 1m, 1h.")
	cmd.Flags().Duration(cli.ConsentRequestLifespan, 0, "Set the login/consent request lifespan e.g. 1s, 1m, 1h")
	cmd.Flags().Bool(cli.OnlyRequests, false, "This will only run the cleanup on requests and will skip token cleanup.")
	cmd.Flags().Bool(cli.OnlyTokens, false, "This will only run the cleanup on tokens and will skip requests cleanup.")
	cmd.Flags().BoolP(cli.ReadFromEnv, "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
	configx.RegisterFlags(cmd.PersistentFlags())
	return cmd

}
<|endoftext|>