<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/rfjakob/gocryptfs/cluefs_frontend"
	"github.com/rfjakob/gocryptfs/pathfs_frontend"
	"github.com/rfjakob/gocryptfs/cryptfs"

	bazilfuse "bazil.org/fuse"
	bazilfusefs "bazil.org/fuse/fs"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

const (
	USE_CLUEFS   = false // Use cluefs or pathfs FUSE frontend
	USE_OPENSSL  = true // 3x speed increase
	PATHFS_DEBUG = false

	PROGRAM_NAME = "gocryptfs"

	// Exit codes
	ERREXIT_USAGE  = 1
	ERREXIT_NEWFS  = 2
	ERREXIT_MOUNT  = 3
	ERREXIT_SERVE  = 4
	ERREXIT_MOUNT2 = 5
	ERREXIT_CIPHERDIR = 6
	ERREXIT_INIT = 7
	ERREXIT_LOADCONF = 8
)

func main() {
	// Parse command line arguments
	var debug bool
	var init bool
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&init, "init", false, "Initialize encrypted directory")
	flag.Parse()
	if debug {
		cryptfs.Debug.Enable()
		cryptfs.Debug.Printf("Debug output enabled\n")
	}
	if init {
		if flag.NArg() != 1 {
			fmt.Printf("usage: %s --init CIPHERDIR\n", PROGRAM_NAME)
			os.Exit(ERREXIT_USAGE)
		}
		dir, _ := filepath.Abs(flag.Arg(0))
		filename := filepath.Join(dir, cryptfs.ConfDefaultName)
		err := cryptfs.CreateConfFile(filename, "test")
		if err != nil {
			fmt.Println(err)
			os.Exit(ERREXIT_INIT)
		}
		os.Exit(0)
	}
	if flag.NArg() < 2 {
		fmt.Printf("usage: %s CIPHERDIR MOUNTPOINT\n", PROGRAM_NAME)
		os.Exit(ERREXIT_USAGE)
	}
	cipherdir, _ := filepath.Abs(flag.Arg(0))
	mountpoint, _ := filepath.Abs(flag.Arg(1))
	cryptfs.Debug.Printf("cipherdir=%s\nmountpoint=%s\n", cipherdir, mountpoint)

	_, err := os.Stat(cipherdir)
	if err != nil {
		fmt.Printf("Cipherdir: %s\n", err.Error())
		os.Exit(ERREXIT_CIPHERDIR)
	}

	cfname := filepath.Join(cipherdir, cryptfs.ConfDefaultName)
	_, err = os.Stat(cfname)
	if err != nil {
		fmt.Printf("Error: %s not found in CIPHERDIR\n", cryptfs.ConfDefaultName)
		fmt.Printf("Please run \"%s --init %s\" first\n", PROGRAM_NAME, cipherdir)
		os.Exit(ERREXIT_LOADCONF)
	}
	key, err := cryptfs.LoadConfFile(cfname, "test")
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_LOADCONF)
	}

	if USE_CLUEFS {
		cluefsFrontend(key, cipherdir, mountpoint)
	} else {
		pathfsFrontend(key, cipherdir, mountpoint, debug)
	}
}

func cluefsFrontend(key []byte, cipherdir string, mountpoint string) {
	cfs, err := cluefs_frontend.NewFS(key, cipherdir, USE_OPENSSL)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_NEWFS)
	}

	// Mount the file system
	mountOpts := []bazilfuse.MountOption{
		bazilfuse.FSName(PROGRAM_NAME),
		bazilfuse.Subtype(PROGRAM_NAME),
		bazilfuse.VolumeName(PROGRAM_NAME),
		bazilfuse.LocalVolume(),
		bazilfuse.MaxReadahead(1024 * 1024),
	}
	conn, err := bazilfuse.Mount(mountpoint, mountOpts...)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_MOUNT)
	}
	defer conn.Close()

	// Start serving requests
	if err = bazilfusefs.Serve(conn, cfs); err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_SERVE)
	}

	// Check for errors when mounting the file system
	<-conn.Ready
	if err = conn.MountError; err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_MOUNT2)
	}

	// We are done
	os.Exit(0)
}

func pathfsFrontend(key []byte, cipherdir string, mountpoint string, debug bool){

	finalFs := pathfs_frontend.NewFS(key, cipherdir, USE_OPENSSL)

	opts := &nodefs.Options{
		// These options are to be compatible with libfuse defaults,
		// making benchmarking easier.
		NegativeTimeout: time.Second,
		AttrTimeout:     time.Second,
		EntryTimeout:    time.Second,
	}
	pathFs := pathfs.NewPathNodeFs(finalFs, nil)
	conn := nodefs.NewFileSystemConnector(pathFs.Root(), opts)
	mOpts := &fuse.MountOptions{
		AllowOther: false,
	}
	state, err := fuse.NewServer(conn.RawFS(), mountpoint, mOpts)
	if err != nil {
		fmt.Printf("Mount fail: %v\n", err)
		os.Exit(1)
	}
	state.SetDebug(debug)

	fmt.Println("Mounted!")
	state.Serve()
}
<commit_msg>Implement password handling<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"time"
	"encoding/hex"

	"github.com/rfjakob/gocryptfs/cluefs_frontend"
	"github.com/rfjakob/gocryptfs/pathfs_frontend"
	"github.com/rfjakob/gocryptfs/cryptfs"

	"golang.org/x/crypto/ssh/terminal"

	bazilfuse "bazil.org/fuse"
	bazilfusefs "bazil.org/fuse/fs"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

const (
	USE_CLUEFS   = false // Use cluefs or pathfs FUSE frontend
	USE_OPENSSL  = true // 3x speed increase
	PATHFS_DEBUG = false

	PROGRAM_NAME = "gocryptfs"

	// Exit codes
	ERREXIT_USAGE  = 1
	ERREXIT_NEWFS  = 2
	ERREXIT_MOUNT  = 3
	ERREXIT_SERVE  = 4
	ERREXIT_MOUNT2 = 5
	ERREXIT_CIPHERDIR = 6
	ERREXIT_INIT = 7
	ERREXIT_LOADCONF = 8
	ERREXIT_PASSWORD = 9
)

func main() {
	// Parse command line arguments
	var debug bool
	var init bool
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&init, "init", false, "Initialize encrypted directory")
	flag.Parse()
	if debug {
		cryptfs.Debug.Enable()
		cryptfs.Debug.Printf("Debug output enabled\n")
	}
	if init {
		if flag.NArg() != 1 {
			fmt.Printf("usage: %s --init CIPHERDIR\n", PROGRAM_NAME)
			os.Exit(ERREXIT_USAGE)
		}
		dir, _ := filepath.Abs(flag.Arg(0))
		filename := filepath.Join(dir, cryptfs.ConfDefaultName)
		fmt.Printf("Choose a password for protecting your files.\n")
		password := readPasswordTwice()
		err := cryptfs.CreateConfFile(filename, password)
		if err != nil {
			fmt.Println(err)
			os.Exit(ERREXIT_INIT)
		}
		fmt.Printf("The filesystem is now ready for mounting.\n")
		os.Exit(0)
	}
	if flag.NArg() < 2 {
		fmt.Printf("usage: %s CIPHERDIR MOUNTPOINT\n", PROGRAM_NAME)
		os.Exit(ERREXIT_USAGE)
	}
	cipherdir, _ := filepath.Abs(flag.Arg(0))
	mountpoint, _ := filepath.Abs(flag.Arg(1))
	cryptfs.Debug.Printf("cipherdir=%s\nmountpoint=%s\n", cipherdir, mountpoint)

	_, err := os.Stat(cipherdir)
	if err != nil {
		fmt.Printf("Cipherdir: %s\n", err.Error())
		os.Exit(ERREXIT_CIPHERDIR)
	}

	cfname := filepath.Join(cipherdir, cryptfs.ConfDefaultName)
	_, err = os.Stat(cfname)
	if err != nil {
		fmt.Printf("Error: %s not found in CIPHERDIR\n", cryptfs.ConfDefaultName)
		fmt.Printf("Please run \"%s --init %s\" first\n", PROGRAM_NAME, flag.Arg(0))
		os.Exit(ERREXIT_LOADCONF)
	}

	fmt.Printf("Password: ")
	password := readPassword()
	fmt.Printf("\nDecrypting master key... ")
	key, err := cryptfs.LoadConfFile(cfname, password)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_LOADCONF)
	}
	fmt.Printf("Success\n")
	printMasterKey(key)

	if USE_CLUEFS {
		cluefsFrontend(key, cipherdir, mountpoint)
	} else {
		pathfsFrontend(key, cipherdir, mountpoint, debug)
	}
}

// printMasterKey - remind the user that he should store the master key in
// a safe place
func printMasterKey(key []byte) {
	h := hex.EncodeToString(key)
	// Make it less scary by splitting it up in chunks
	h = h[0:8] + "-" + h[8:16] + "-" + h[16:24] + "-" + h[24:32]

	fmt.Printf(`
WARNING:
  If the gocryptfs config file becomes corrupted or you ever
  forget your password, there is only one hope for recovery:
  The master key. Print it to a piece of paper and store it in a drawer.

  Master key: %s

`, h)
}

func readPasswordTwice() string {
	fmt.Printf("Password: ")
	p1 := readPassword()
	fmt.Printf("\nRepeat: ")
	p2 := readPassword()
	fmt.Printf("\n")
	if p1 != p2 {
		fmt.Printf("Passwords do not match\n")
		os.Exit(ERREXIT_PASSWORD)
	}
	return p1
}

// Get password from terminal
func readPassword() string {
	fd := int(os.Stdin.Fd())
	p, err := terminal.ReadPassword(fd)
	if err != nil {
		fmt.Printf("Error: Could not read password: %s\n")
		os.Exit(ERREXIT_PASSWORD)
	}
	return string(p)
}

func cluefsFrontend(key []byte, cipherdir string, mountpoint string) {
	cfs, err := cluefs_frontend.NewFS(key, cipherdir, USE_OPENSSL)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_NEWFS)
	}

	// Mount the file system
	mountOpts := []bazilfuse.MountOption{
		bazilfuse.FSName(PROGRAM_NAME),
		bazilfuse.Subtype(PROGRAM_NAME),
		bazilfuse.VolumeName(PROGRAM_NAME),
		bazilfuse.LocalVolume(),
		bazilfuse.MaxReadahead(1024 * 1024),
	}
	conn, err := bazilfuse.Mount(mountpoint, mountOpts...)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_MOUNT)
	}
	defer conn.Close()

	// Start serving requests
	if err = bazilfusefs.Serve(conn, cfs); err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_SERVE)
	}

	// Check for errors when mounting the file system
	<-conn.Ready
	if err = conn.MountError; err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_MOUNT2)
	}

	// We are done
	os.Exit(0)
}

func pathfsFrontend(key []byte, cipherdir string, mountpoint string, debug bool){

	finalFs := pathfs_frontend.NewFS(key, cipherdir, USE_OPENSSL)

	opts := &nodefs.Options{
		// These options are to be compatible with libfuse defaults,
		// making benchmarking easier.
		NegativeTimeout: time.Second,
		AttrTimeout:     time.Second,
		EntryTimeout:    time.Second,
	}
	pathFs := pathfs.NewPathNodeFs(finalFs, nil)
	conn := nodefs.NewFileSystemConnector(pathFs.Root(), opts)
	mOpts := &fuse.MountOptions{
		AllowOther: false,
	}
	state, err := fuse.NewServer(conn.RawFS(), mountpoint, mOpts)
	if err != nil {
		fmt.Printf("Mount fail: %v\n", err)
		os.Exit(1)
	}
	state.SetDebug(debug)

	fmt.Println("Mounted.")
	state.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
)

type config struct {
	owner string
	repo  string
	token string
}

func main() {
	cfg := new(config)

	flag.StringVar(&cfg.owner, "owner", "", "GitHub owner")
	flag.StringVar(&cfg.repo, "repo", "", "GitHub repo")
	flag.StringVar(&cfg.token, "token", "", "GitHub token")

	flag.Parse()

	if cfg.owner == "" {
		fmt.Println("Missing GitHub owner")
		os.Exit(-1)
	}

	if cfg.repo == "" {
		fmt.Println("Missing GitHub repo")
		os.Exit(-1)
	}

	if cfg.token == "" {
		fmt.Println("Missing GitHub token")
		os.Exit(-1)
	}
}
<commit_msg>List all forks<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"os"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type config struct {
	owner string
	repo  string
	token string
}

func main() {
	cfg := new(config)

	flag.StringVar(&cfg.owner, "owner", "", "GitHub owner")
	flag.StringVar(&cfg.repo, "repo", "", "GitHub repo")
	flag.StringVar(&cfg.token, "token", "", "GitHub token")

	flag.Parse()

	if cfg.owner == "" {
		fmt.Println("Missing GitHub owner")
		os.Exit(-1)
	}

	if cfg.repo == "" {
		fmt.Println("Missing GitHub repo")
		os.Exit(-1)
	}

	if cfg.token == "" {
		fmt.Println("Missing GitHub token")
		os.Exit(-1)
	}

	ctx := context.Background()
	sts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: cfg.token})
	client := github.NewClient(oauth2.NewClient(ctx, sts))

	forks, _, err := client.Repositories.ListForks(ctx, cfg.owner, cfg.repo, nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}

	for _, fork := range forks {
		fmt.Println(*fork.SSHURL)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
)

type Page struct {
	IsProductionMode bool
}

// Compilation levels supported by Closure Compiler
const (
	JS_COMPILATION_LEVEL_ADVANCED_OPTIMIZATIONS = "ADVANCED_OPTIMIZATIONS"
	JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS   = "SIMPLE_OPTIMIZATIONS"
	JS_COMPILATION_LEVEL_WHITESPACE_ONLY        = "WHITESPACE_ONLY"
)

// Command-line flags
var (
	httpPort           = flag.String("port", "8080", "HTTP port the web server listens to.")
	isProductionMode   = flag.Bool("production", false, "Whether the server should run in production mode.")
	jsCompilationLevel = flag.String("js-compilation-level", JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS, "Either WHITESPACE_ONLY, SIMPLE_OPTIMIZATIONS or ADVANCED_OPTIMIZATIONS. See https://developers.google.com/closure/compiler/docs/compilation_levels. Advanced optimizations can break your code. Only used in production mode.")
	verbose            = flag.Bool("verbose", false, "Whether additional information should be displayed.")
)

// RegEx patterns
var (
	assetUrlPattern   = regexp.MustCompile("^/(?:css|images|js)/")
	whitespacePattern = regexp.MustCompile(">[ \f\n\r\t]+<")
)

func main() {
	// Set maximum number of CPUs that can be executing simultaneously
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Parse command-line flags
	flag.Parse()

	log.Println("Production mode:", *isProductionMode)

	if *isProductionMode {
		compileCss()
		compileJavaScript()
	}

	http.HandleFunc("/", handleRequest)
	log.Println("Web server is running at 127.0.0.1:" + *httpPort + ".")

	if error := http.ListenAndServe(":"+*httpPort, nil); error != nil {
		log.Fatal("Could not start web server: ", error)
	}
}

func handleRequest(responseWriter http.ResponseWriter, request *http.Request) {
	if request.URL.Path == "/" {
		fileContent, error := ioutil.ReadFile("views/layouts/default.html")

		if error != nil {
			http.NotFound(responseWriter, request)
			log.Println("Error:", error)
			return
		}

		cleanedFileContent := whitespacePattern.ReplaceAllString(string(fileContent), "><")
		parsedTemplate, error := template.New("default").Parse(cleanedFileContent)

		error = parsedTemplate.Execute(responseWriter, &Page{IsProductionMode: *isProductionMode})

		if error != nil {
			http.Error(responseWriter, error.Error(), http.StatusInternalServerError)
			log.Println("Error:", error)
			return
		}

		return
	}

	if assetUrlPattern.MatchString(request.URL.Path) {
		file, error := os.Open("webroot" + request.URL.Path)

		if error != nil {
			http.NotFound(responseWriter, request)
			log.Println("Error:", error)
			return
		}

		fileInfo, error := file.Stat()

		if error != nil {
			http.NotFound(responseWriter, request)
			log.Println("Error:", error)
			return
		}

		http.ServeContent(responseWriter, request, "webroot"+request.URL.Path, fileInfo.ModTime(), file)
		return
	}

	http.NotFound(responseWriter, request)
}

func compileCss() {
	log.Println("Compiling CSS ...")

	workingDirectory, error := os.Getwd()

	if error != nil {
		log.Fatal("Could not determine working directory: ", error)
	}

	command := exec.Command(
		"java",
		"-jar", workingDirectory+"/libraries/closure-stylesheets-20111230/closure-stylesheets-20111230.jar",
		"--allowed-non-standard-function", "color-stop",
		"--allowed-non-standard-function", "progid:DXImageTransform.Microsoft.gradient",
		"--allowed-unrecognized-property", "tap-highlight-color",
		"--allowed-unrecognized-property", "text-size-adjust",
		"--output-file", workingDirectory+"/webroot/css/compiled.css",

		// Stylesheet order is important: Succeeding rules overwrite preceding ones
		"./webroot/css/reset.gss",
		"./webroot/css/general.gss",
		"./webroot/css/form.gss",
		"./webroot/css/subreddit-picker.gss",
		"./webroot/css/board.gss",
		"./webroot/css/board-item.gss",
	)

	stderrPipe, error := command.StderrPipe()

	if error != nil {
		log.Fatal("Could not create stderr pipe for Closure Stylesheets: ", error)
	}

	if error := command.Start(); error != nil {
		log.Fatal("Could not start Closure Stylesheets: ", error)
	}

	stderrOutput, error := ioutil.ReadAll(stderrPipe)

	if error != nil {
		log.Fatal("Could not read from Closure Stylesheets' stderr pipe: ", error)
	}

	if error := command.Wait(); error != nil {
		log.Println("Could not compile CSS:", string(stderrOutput))
		log.Fatal("Closure Stylesheets finished with: ", error)
	}

	log.Println("Compiled CSS.")
}

func compileJavaScript() {
	*jsCompilationLevel = strings.ToUpper(*jsCompilationLevel)

	switch *jsCompilationLevel {
	case JS_COMPILATION_LEVEL_ADVANCED_OPTIMIZATIONS:
		log.Println("Compiling JavaScript with advanced optimizations ...")
	case JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS:
		log.Println("Compiling JavaScript with simple optimizations ...")
	case JS_COMPILATION_LEVEL_WHITESPACE_ONLY:
		log.Println("Compiling JavaScript with whitespace-only optimizations ...")
	default:
		log.Printf("JavaScript compilation level '%s' not recognized. Using '%s'.\n", *jsCompilationLevel, JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS)
		log.Println("Compiling JavaScript with simple optimizations ...")
		*jsCompilationLevel = JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS
	}

	workingDirectory, error := os.Getwd()

	if error != nil {
		log.Fatal("Could not determine working directory: ", error)
	}

	command := exec.Command(
		workingDirectory+"/libraries/closure-library-20120710-r2029/closure/bin/build/closurebuilder.py",
		"--compiler_flags=--compilation_level="+*jsCompilationLevel,
		"--compiler_flags=--warning_level=VERBOSE",
		"--compiler_jar="+workingDirectory+"/libraries/closure-compiler-20120917-r2180/compiler.jar",
		"--namespace=panoptikos.Panoptikos",
		"--output_file="+workingDirectory+"/webroot/js/compiled.js",
		"--output_mode=compiled",
		"--root="+workingDirectory,
	)

	stderrPipe, error := command.StderrPipe()

	if error != nil {
		log.Fatal("Could not create stderr pipe for Closure Builder: ", error)
	}

	if error := command.Start(); error != nil {
		log.Fatal("Could not start Closure Builder: ", error)
	}

	stderrOutput, error := ioutil.ReadAll(stderrPipe)

	if error != nil {
		log.Fatal("Could not read from Closure Builder's stderr pipe: ", error)
	}

	if error := command.Wait(); error != nil {
		log.Println("Could not compile JavaScript:", string(stderrOutput))
		log.Fatal("Closure Builder finished with: ", error)
	}

	if *verbose {
		// All Closure Builder output runs over stderr, even if no error occurred
		log.Println(string(stderrOutput))
	}

	log.Println("Compiled JavaScript.")
}
<commit_msg>Fixed problem where stylesheets were not compiled when server was started in development mode.<commit_after>package main

import (
	"flag"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
)

type Page struct {
	IsProductionMode bool
}

// Compilation levels supported by Closure Compiler
const (
	JS_COMPILATION_LEVEL_ADVANCED_OPTIMIZATIONS = "ADVANCED_OPTIMIZATIONS"
	JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS   = "SIMPLE_OPTIMIZATIONS"
	JS_COMPILATION_LEVEL_WHITESPACE_ONLY        = "WHITESPACE_ONLY"
)

// Command-line flags
var (
	httpPort           = flag.String("port", "8080", "HTTP port the web server listens to.")
	isProductionMode   = flag.Bool("production", false, "Whether the server should run in production mode.")
	jsCompilationLevel = flag.String("js-compilation-level", JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS, "Either WHITESPACE_ONLY, SIMPLE_OPTIMIZATIONS or ADVANCED_OPTIMIZATIONS. See https://developers.google.com/closure/compiler/docs/compilation_levels. Advanced optimizations can break your code. Only used in production mode.")
	verbose            = flag.Bool("verbose", false, "Whether additional information should be displayed.")
)

// RegEx patterns
var (
	assetUrlPattern   = regexp.MustCompile("^/(?:css|images|js)/")
	whitespacePattern = regexp.MustCompile(">[ \f\n\r\t]+<")
)

func main() {
	// Set maximum number of CPUs that can be executing simultaneously
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Parse command-line flags
	flag.Parse()

	log.Println("Production mode:", *isProductionMode)
	compileCss()

	if *isProductionMode {
		compileJavaScript()
	}

	http.HandleFunc("/", handleRequest)
	log.Println("Web server is running at 127.0.0.1:" + *httpPort + ".")

	if error := http.ListenAndServe(":"+*httpPort, nil); error != nil {
		log.Fatal("Could not start web server: ", error)
	}
}

func handleRequest(responseWriter http.ResponseWriter, request *http.Request) {
	if request.URL.Path == "/" {
		fileContent, error := ioutil.ReadFile("views/layouts/default.html")

		if error != nil {
			http.NotFound(responseWriter, request)
			log.Println("Error:", error)
			return
		}

		cleanedFileContent := whitespacePattern.ReplaceAllString(string(fileContent), "><")
		parsedTemplate, error := template.New("default").Parse(cleanedFileContent)

		error = parsedTemplate.Execute(responseWriter, &Page{IsProductionMode: *isProductionMode})

		if error != nil {
			http.Error(responseWriter, error.Error(), http.StatusInternalServerError)
			log.Println("Error:", error)
			return
		}

		return
	}

	if assetUrlPattern.MatchString(request.URL.Path) {
		file, error := os.Open("webroot" + request.URL.Path)

		if error != nil {
			http.NotFound(responseWriter, request)
			log.Println("Error:", error)
			return
		}

		fileInfo, error := file.Stat()

		if error != nil {
			http.NotFound(responseWriter, request)
			log.Println("Error:", error)
			return
		}

		http.ServeContent(responseWriter, request, "webroot"+request.URL.Path, fileInfo.ModTime(), file)
		return
	}

	http.NotFound(responseWriter, request)
}

func compileCss() {
	log.Println("Compiling CSS ...")

	workingDirectory, error := os.Getwd()

	if error != nil {
		log.Fatal("Could not determine working directory: ", error)
	}

	command := exec.Command(
		"java",
		"-jar", workingDirectory+"/libraries/closure-stylesheets-20111230/closure-stylesheets-20111230.jar",
		"--allowed-non-standard-function", "color-stop",
		"--allowed-non-standard-function", "progid:DXImageTransform.Microsoft.gradient",
		"--allowed-unrecognized-property", "tap-highlight-color",
		"--allowed-unrecognized-property", "text-size-adjust",
		"--output-file", workingDirectory+"/webroot/css/compiled.css",

		// Stylesheet order is important: Succeeding rules overwrite preceding ones
		"./webroot/css/reset.gss",
		"./webroot/css/general.gss",
		"./webroot/css/form.gss",
		"./webroot/css/subreddit-picker.gss",
		"./webroot/css/board.gss",
		"./webroot/css/board-item.gss",
	)

	stderrPipe, error := command.StderrPipe()

	if error != nil {
		log.Fatal("Could not create stderr pipe for Closure Stylesheets: ", error)
	}

	if error := command.Start(); error != nil {
		log.Fatal("Could not start Closure Stylesheets: ", error)
	}

	stderrOutput, error := ioutil.ReadAll(stderrPipe)

	if error != nil {
		log.Fatal("Could not read from Closure Stylesheets' stderr pipe: ", error)
	}

	if error := command.Wait(); error != nil {
		log.Println("Could not compile CSS:", string(stderrOutput))
		log.Fatal("Closure Stylesheets finished with: ", error)
	}

	log.Println("Compiled CSS.")
}

func compileJavaScript() {
	*jsCompilationLevel = strings.ToUpper(*jsCompilationLevel)

	switch *jsCompilationLevel {
	case JS_COMPILATION_LEVEL_ADVANCED_OPTIMIZATIONS:
		log.Println("Compiling JavaScript with advanced optimizations ...")
	case JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS:
		log.Println("Compiling JavaScript with simple optimizations ...")
	case JS_COMPILATION_LEVEL_WHITESPACE_ONLY:
		log.Println("Compiling JavaScript with whitespace-only optimizations ...")
	default:
		log.Printf("JavaScript compilation level '%s' not recognized. Using '%s'.\n", *jsCompilationLevel, JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS)
		log.Println("Compiling JavaScript with simple optimizations ...")
		*jsCompilationLevel = JS_COMPILATION_LEVEL_SIMPLE_OPTIMIZATIONS
	}

	workingDirectory, error := os.Getwd()

	if error != nil {
		log.Fatal("Could not determine working directory: ", error)
	}

	command := exec.Command(
		workingDirectory+"/libraries/closure-library-20120710-r2029/closure/bin/build/closurebuilder.py",
		"--compiler_flags=--compilation_level="+*jsCompilationLevel,
		"--compiler_flags=--warning_level=VERBOSE",
		"--compiler_jar="+workingDirectory+"/libraries/closure-compiler-20120917-r2180/compiler.jar",
		"--namespace=panoptikos.Panoptikos",
		"--output_file="+workingDirectory+"/webroot/js/compiled.js",
		"--output_mode=compiled",
		"--root="+workingDirectory,
	)

	stderrPipe, error := command.StderrPipe()

	if error != nil {
		log.Fatal("Could not create stderr pipe for Closure Builder: ", error)
	}

	if error := command.Start(); error != nil {
		log.Fatal("Could not start Closure Builder: ", error)
	}

	stderrOutput, error := ioutil.ReadAll(stderrPipe)

	if error != nil {
		log.Fatal("Could not read from Closure Builder's stderr pipe: ", error)
	}

	if error := command.Wait(); error != nil {
		log.Println("Could not compile JavaScript:", string(stderrOutput))
		log.Fatal("Closure Builder finished with: ", error)
	}

	if *verbose {
		// All Closure Builder output runs over stderr, even if no error occurred
		log.Println(string(stderrOutput))
	}

	log.Println("Compiled JavaScript.")
}
<|endoftext|>


<commit_before>package email

import (
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"

	"appengine"
)

// Required to be able to pass different kind of headers in the following functions
type emailHeader interface {
	Get(key string) string
}

// extracts the body of an email
func extractBody(c appengine.Context, header emailHeader, bodyReader io.Reader) (*email, error) {
	contentType := header.Get("Content-Type")
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return nil, err
	}

	if mediaType[:4] == "text" {
		c.Infof("extractBody: found text")
		return extractTextBody(c, header, bodyReader)
	}

	if mediaType[:9] == "multipart" {
		c.Infof("extractBody: multipart")
		return extractMimeBody(c, params["boundary"], bodyReader)
	}

	return nil, fmt.Errorf("Unsupported content type: %s", contentType)
}

// read through the varios multiple parts
func extractMimeBody(c appengine.Context, boundary string, bodyReader io.Reader) (*email, error) {
	var withError *email // stores an email parse with error

	mimeReader := multipart.NewReader(bodyReader, boundary)

	for {
		part, err := mimeReader.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		defer part.Close()

		result, err := extractBody(c, part.Header, part)

		// this means we tried to decode it, but are not sure
		// lets save this result and try the other parts before return this result
		if result != nil && err != nil {
			withError = result
			c.Infof("extractMimeBody: email guess with error %v", err)
			continue
		}

		if result != nil && result.ContentType[:4] == "text" {
			return result, nil
		}
	}

	if withError != nil {
		return withError, nil
	}

	return nil, fmt.Errorf("Could not parse any of the multiple parts:")
}

// Decode body text and store it in a string
func extractTextBody(c appengine.Context, header emailHeader, bodyReader io.Reader) (*email, error) {
	var returnee email

	s, err := ioutil.ReadAll(bodyReader)
	if err != nil {
		return nil, err
	}

	encoding := header.Get("Content-Transfer-Encoding")
	c.Infof("extractTextBody encoding: %v", encoding)

	if encoding == "base64" {
		b, err := base64.StdEncoding.DecodeString(string(s))
		if err != nil {
			return nil, err
		}
		returnee.Body = string(b)
		returnee.ContentType = header.Get("Content-Type")
		return &returnee, nil
	}

	if encoding == "7bit" {
		// that is just US ASCII (7bit)
		// https://stackoverflow.com/questions/25710599/content-transfer-encoding-7bit-or-8-bit
		returnee.Body = string(s)
		returnee.ContentType = header.Get("Content-Type")
		return &returnee, nil
	}

	if encoding == "quoted-printable" {
		// https://stackoverflow.com/questions/24883742/how-to-decode-mail-body-in-go
		// looks like it will be in go 1.5
		// maybe wait until then?
		// TODO
	}

	// ok, let's guess this is just plain text and put it into a string
	returnee.Body = string(s)
	returnee.ContentType = header.Get("Content-Type")

	return &returnee, fmt.Errorf("Unsupported Content-Transfer-Encoding: %v", encoding)
}
<commit_msg>Adds support for quoted printables. Fixes #108.<commit_after>package email

import (
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"

	"appengine"

	"gopkg.in/alexcesaro/quotedprintable.v3"
)

// Required to be able to pass different kind of headers in the following functions
type emailHeader interface {
	Get(key string) string
}

// extracts the body of an email
func extractBody(c appengine.Context, header emailHeader, bodyReader io.Reader) (*email, error) {
	contentType := header.Get("Content-Type")
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return nil, err
	}

	if mediaType[:4] == "text" {
		c.Infof("extractBody: found text")
		return extractTextBody(c, header, bodyReader)
	}

	if mediaType[:9] == "multipart" {
		c.Infof("extractBody: multipart")
		return extractMimeBody(c, params["boundary"], bodyReader)
	}

	return nil, fmt.Errorf("Unsupported content type: %s", contentType)
}

// read through the varios multiple parts
func extractMimeBody(c appengine.Context, boundary string, bodyReader io.Reader) (*email, error) {
	var withError *email // stores an email parse with error

	mimeReader := multipart.NewReader(bodyReader, boundary)

	for {
		part, err := mimeReader.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		defer part.Close()

		result, err := extractBody(c, part.Header, part)

		// this means we tried to decode it, but are not sure
		// lets save this result and try the other parts before return this result
		if result != nil && err != nil {
			withError = result
			c.Infof("extractMimeBody: email guess with error %v", err)
			continue
		}

		if result != nil && result.ContentType[:4] == "text" {
			return result, nil
		}
	}

	if withError != nil {
		return withError, nil
	}

	return nil, fmt.Errorf("Could not parse any of the multiple parts:")
}

// Decode body text and store it in a string
func extractTextBody(c appengine.Context, header emailHeader, bodyReader io.Reader) (*email, error) {
	var returnee email

	s, err := ioutil.ReadAll(bodyReader)
	if err != nil {
		return nil, err
	}

	encoding := header.Get("Content-Transfer-Encoding")
	c.Infof("extractTextBody encoding: %v", encoding)

	if encoding == "base64" {
		b, err := base64.StdEncoding.DecodeString(string(s))
		if err != nil {
			return nil, err
		}
		returnee.Body = string(b)
		returnee.ContentType = header.Get("Content-Type")
		return &returnee, nil
	}

	if encoding == "7bit" {
		// that is just US ASCII (7bit)
		// https://stackoverflow.com/questions/25710599/content-transfer-encoding-7bit-or-8-bit
		returnee.Body = string(s)
		returnee.ContentType = header.Get("Content-Type")
		return &returnee, nil
	}

	if encoding == "quoted-printable" {
		// https://stackoverflow.com/questions/24883742/how-to-decode-mail-body-in-go
		dec := new(quotedprintable.WordDecoder)
		b, err := dec.Decode(string(s))
		if err != nil {
			return nil, err
		}
		returnee.Body = string(b)
		returnee.ContentType = header.Get("Content-Type")
		return &returnee, nil
	}

	// ok, let's guess this is just plain text and put it into a string
	returnee.Body = string(s)
	returnee.ContentType = header.Get("Content-Type")

	return &returnee, fmt.Errorf("Unsupported Content-Transfer-Encoding: %v", encoding)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/mediocregopher/lever"
	"github.com/mediocregopher/radix/v3"

	_ "net/http/pprof"
)

// Prefix is a Markov chain prefix of one or more words.
type Prefix []string

// String returns the Prefix as a string (for use as a map key).
func (p Prefix) String() string {
	return strings.Join(p, " ")
}

// Shift removes the first word from the Prefix and appends the given word.
func (p Prefix) Shift(word string) {
	copy(p, p[1:])
	p[len(p)-1] = word
}

var p *radix.Pool

func prefixKey(chain string, prefix Prefix) string {
	return fmt.Sprintf("markov:%s:%s", chain, prefix.String())
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	log.SetFlags(log.Llongfile)

	l := lever.New("markov", nil)
	l.Add(lever.Param{
		Name:        "-prefixLen",
		Default:     "2",
		Description: "Prefix length for the markov chain algorithm",
	})
	l.Add(lever.Param{
		Name:        "-listenAddr",
		Default:     ":8080",
		Description: "Address to listen for calls to the http interface on",
	})
	l.Add(lever.Param{
		Name:        "-redisAddr",
		Default:     "127.0.0.1:6379",
		Description: "Address for an instance of redis",
	})
	l.Add(lever.Param{
		Name:        "-timeout",
		Default:     "720",
		Description: "Hours a suffix is allowed to stay untouched before it is cleaned up",
	})
	l.Parse()

	redisAddr, _ := l.ParamStr("-redisAddr")
	var err error

	p, err = radix.NewPool("tcp", redisAddr, 10)
	if err != nil {
		log.Fatal(err)
	}

	prefixLen, _ := l.ParamInt("-prefixLen")
	timeout, _ := l.ParamInt("-timeout")
	go clydeTheCleaner(int64(timeout))

	http.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
		var suffixes []string
		for {
			var s string
			if _, err := fmt.Fscan(r.Body, &s); err != nil {
				break
			}
			suffixes = append(suffixes, strings.TrimSpace(s))
		}

		prefix := make(Prefix, prefixLen)
		ts := time.Now().Unix()
		for _, suffix := range suffixes {
			key := prefixKey(r.FormValue("chainName"), prefix)
			if err := p.Do(radix.FlatCmd(nil, "ZADD", key, ts, suffix)); err != nil {
				log.Fatal(err)
			}
			prefix.Shift(suffix)
		}
	})

	http.HandleFunc("/generate", func(w http.ResponseWriter, r *http.Request) {
		numPartsStr := r.FormValue("numParts")
		if numPartsStr == "" {
			http.Error(w, "numParts argument must be specified", 400)
			return
		}

		numParts, err := strconv.Atoi(numPartsStr)
		if err != nil {
			http.Error(w, fmt.Sprintf("invalid value of numParts: %s", err), 400)
			return
		}

		prefix := make(Prefix, prefixLen)
		var words []string
		for {
			key := prefixKey(r.FormValue("chainName"), prefix)
			var suffixes []string
			if err := p.Do(radix.Cmd(&suffixes, "ZRANGE", key, "0", "-1")); err != nil {
				log.Fatal(err)
			} else if len(suffixes) == 0 {
				break
			}

			i := rand.Intn(len(suffixes))
			next := suffixes[i]
			words = append(words, next)
			prefix.Shift(next)

			if len(next) == 0 {
				continue
			}

			lastChar := next[len(next)-1]

			if lastChar == '!' ||
				lastChar == '?' ||
				lastChar == '.' ||
				(numParts == 1 &&
					(lastChar == ',' ||
						lastChar == ':' ||
						lastChar == ';')) {
				numParts--
			}

			switch lastChar {
			case '!', '?', '.', ',', ':', ';':
				numParts--
			}

			if numParts <= 0 {
				break
			}
		}

		fmt.Fprintln(w, strings.Join(words, " "))
	})

	listenAddr, _ := l.ParamStr("-listenAddr")
	log.Fatal(http.ListenAndServe(listenAddr, nil))
}

func clydeTheCleaner(timeout int64) {
	tick := time.Tick(30 * time.Second)
	for {
		expire := time.Now().Unix() - (timeout * 3600)
		scanner := radix.NewScanner(p, radix.ScanOpts{
			Command: "SCAN",
			Pattern: "markov:*",
		})
		var key string
		for scanner.Next(&key) {
			if err := p.Do(radix.FlatCmd(nil, "ZREMRANGEBYSCORE", key, "0", expire)); err != nil {
				log.Fatal(err)
			}
		}
		if err := scanner.Close(); err != nil {
			log.Fatal(err)
		}

		<-tick
	}
}
<commit_msg>make generate track prefices it's used already so it doesn't get into infinite loops<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/mediocregopher/lever"
	"github.com/mediocregopher/radix/v3"

	_ "net/http/pprof"
)

// Prefix is a Markov chain prefix of one or more words.
type Prefix []string

// String returns the Prefix as a string (for use as a map key).
func (p Prefix) String() string {
	return strings.Join(p, " ")
}

// Shift returns a copy of the Prefix with the first word removed and the given
// one appended.
func (p Prefix) Shift(word string) Prefix {
	p2 := make(Prefix, len(p))
	copy(p2, p[1:])
	p2[len(p2)-1] = word
	return p2
}

var p *radix.Pool

func prefixKey(chain string, prefix Prefix) string {
	return fmt.Sprintf("markov:%s:%s", chain, prefix.String())
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	log.SetFlags(log.Llongfile)

	l := lever.New("markov", nil)
	l.Add(lever.Param{
		Name:        "-prefixLen",
		Default:     "2",
		Description: "Prefix length for the markov chain algorithm",
	})
	l.Add(lever.Param{
		Name:        "-listenAddr",
		Default:     ":8080",
		Description: "Address to listen for calls to the http interface on",
	})
	l.Add(lever.Param{
		Name:        "-redisAddr",
		Default:     "127.0.0.1:6379",
		Description: "Address for an instance of redis",
	})
	l.Add(lever.Param{
		Name:        "-timeout",
		Default:     "720",
		Description: "Hours a suffix is allowed to stay untouched before it is cleaned up",
	})
	l.Parse()

	redisAddr, _ := l.ParamStr("-redisAddr")
	var err error

	p, err = radix.NewPool("tcp", redisAddr, 10)
	if err != nil {
		log.Fatal(err)
	}

	prefixLen, _ := l.ParamInt("-prefixLen")
	timeout, _ := l.ParamInt("-timeout")
	go clydeTheCleaner(int64(timeout))

	http.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
		var suffixes []string
		for {
			var s string
			if _, err := fmt.Fscan(r.Body, &s); err != nil {
				break
			}
			suffixes = append(suffixes, strings.TrimSpace(s))
		}

		prefix := make(Prefix, prefixLen)
		ts := time.Now().Unix()
		for _, suffix := range suffixes {
			key := prefixKey(r.FormValue("chainName"), prefix)
			if err := p.Do(radix.FlatCmd(nil, "ZADD", key, ts, suffix)); err != nil {
				log.Fatal(err)
			}
			prefix = prefix.Shift(suffix)
		}
	})

	http.HandleFunc("/generate", func(w http.ResponseWriter, r *http.Request) {
		numPartsStr := r.FormValue("numParts")
		if numPartsStr == "" {
			http.Error(w, "numParts argument must be specified", 400)
			return
		}

		numParts, err := strconv.Atoi(numPartsStr)
		if err != nil {
			http.Error(w, fmt.Sprintf("invalid value of numParts: %s", err), 400)
			return
		}

		// for tracking if a prefix has been used already or not.
		prefixM := map[string]bool{}

		prefix := make(Prefix, prefixLen)
		var words []string
		for {
			key := prefixKey(r.FormValue("chainName"), prefix)
			var suffixes []string
			if err := p.Do(radix.Cmd(&suffixes, "ZRANGE", key, "0", "-1")); err != nil {
				log.Fatal(err)
			} else if len(suffixes) == 0 {
				break
			}

			// try each possible suffix (randomly) trying to find one that
			// generates a prefix which hasn't been used already. If none do
			// then break.
			var next string
			var ok bool
			for _, i := range rand.Perm(len(suffixes)) {
				next = suffixes[i]
				words = append(words, next)
				newPrefix := prefix.Shift(next)
				newPrefixStr := newPrefix.String()
				if prefixM[newPrefixStr] {
					continue
				}

				prefixM[newPrefixStr] = true
				prefix = newPrefix
				ok = true
				break
			}

			if !ok {
				break
			} else if len(next) == 0 {
				continue
			}

			lastChar := next[len(next)-1]

			if lastChar == '!' ||
				lastChar == '?' ||
				lastChar == '.' ||
				(numParts == 1 &&
					(lastChar == ',' ||
						lastChar == ':' ||
						lastChar == ';')) {
				numParts--
			}

			switch lastChar {
			case '!', '?', '.', ',', ':', ';':
				numParts--
			}

			if numParts <= 0 {
				break
			}
		}

		fmt.Fprintln(w, strings.Join(words, " "))
	})

	listenAddr, _ := l.ParamStr("-listenAddr")
	log.Fatal(http.ListenAndServe(listenAddr, nil))
}

func clydeTheCleaner(timeout int64) {
	tick := time.Tick(30 * time.Second)
	for {
		expire := time.Now().Unix() - (timeout * 3600)
		scanner := radix.NewScanner(p, radix.ScanOpts{
			Command: "SCAN",
			Pattern: "markov:*",
		})
		var key string
		for scanner.Next(&key) {
			if err := p.Do(radix.FlatCmd(nil, "ZREMRANGEBYSCORE", key, "0", expire)); err != nil {
				log.Fatal(err)
			}
		}
		if err := scanner.Close(); err != nil {
			log.Fatal(err)
		}

		<-tick
	}
}
<|endoftext|>


<commit_before>package httpfs

import (
	"context"
	"errors"
	"io"
	"net/http"
	"os"
	"strings"
	"time"

	"zenhack.net/go/sandstorm-filesystem/filesystem"
	"zenhack.net/go/sandstorm/exp/util/bytestream"

	"zombiezen.com/go/capnproto2"
)

var (
	InvalidArgument = errors.New("Invalid argument")
	TooManyEntries  = errors.New("Stream received too many entries")
)

// Copy StatInfo. Useful to when the original is going to be reclaimed.
func cloneInfo(info filesystem.StatInfo) filesystem.StatInfo {
	msg, _, err := capnp.NewMessage(capnp.SingleSegment(nil))
	if err != nil {
		panic(err)
	}
	msg.SetRoot(info.Struct.ToPtr())
	newInfo, err := filesystem.ReadRootStatInfo(msg)
	if err != nil {
		panic(err)
	}
	return newInfo
}

type FileSystem struct {
	Dir filesystem.Directory
}

type File struct {
	Node filesystem.Node
	Name string
	Info *FileInfo
	pos  int64
}

func (f *File) Close() error {
	return nil
}

func (f *File) Read(buf []byte) (n int, err error) {
	if f.Info.IsDir() {
		return 0, InvalidArgument
	}
	r, w := io.Pipe()
	file := filesystem.File{Client: f.Node.Client}
	file.Read(context.TODO(), func(p filesystem.File_read_Params) error {
		p.SetStartAt(f.pos)
		p.SetAmount(uint64(len(buf)))
		p.SetSink(bytestream.FromWriteCloser(w, nil))
		return nil
	})
	n, err = io.ReadFull(r, buf)
	if err == io.ErrUnexpectedEOF {
		// ReadFull expects to read the full buffer, but
		// a short read is OK for Read in general.
		err = nil
	}
	r.Close()
	return n, err
}

func (f *File) Seek(offset int64, whence int) (int64, error) {
	if f.Info.IsDir() {
		return 0, InvalidArgument
	}
	oldPos := f.pos
	switch whence {
	case io.SeekStart:
		f.pos = offset
	case io.SeekCurrent:
		f.pos += offset
	case io.SeekEnd:
		f.pos = f.Info.Size() + offset
	default:
		return f.pos, InvalidArgument
	}
	if f.pos < 0 {
		f.pos = oldPos
		return f.pos, InvalidArgument
	}
	return f.pos, nil
}

func (f *File) Stat() (os.FileInfo, error) {
	return f.Info, nil
}

type fiStream struct {
	isClosed bool
	limit    bool
	have     int
	buf      []os.FileInfo
	done     chan struct{}
	err      error
}

func (s *fiStream) Close() error {
	if !s.isClosed {
		s.isClosed = true
		s.err = io.ErrUnexpectedEOF
		s.done <- struct{}{}
	}
	return nil
}

func (s *fiStream) Done(ctx context.Context, p filesystem.Directory_Entry_Stream_done) error {
	if !s.isClosed {
		s.isClosed = true
		s.done <- struct{}{}
	}
	return nil
}

func (s *fiStream) Push(ctx context.Context, p filesystem.Directory_Entry_Stream_push) error {
	if s.isClosed {
		return s.err
	}
	entries, err := p.Args().Entries()
	if err != nil {
		return err
	}
	if s.limit && cap(s.buf)-s.have < entries.Len() {
		s.err = TooManyEntries
		s.isClosed = true
		s.done <- struct{}{}
		return s.err
	}
	for i := 0; i < entries.Len(); i++ {
		entry := entries.At(i)
		name, err := entry.Name()
		if err != nil {
			return err
		}
		info, err := entry.Info()
		if err != nil {
			return err
		}
		s.buf = append(s.buf, &FileInfo{
			name: name,
			info: cloneInfo(info),
		})
		s.have++
	}
	return nil
}

func (f *File) Readdir(count int) ([]os.FileInfo, error) {
	if !f.Info.IsDir() {
		return nil, InvalidArgument
	}
	ret := &fiStream{
		buf:  []os.FileInfo{},
		done: make(chan struct{}, 1),
	}
	if count > 0 {
		ret.buf = make([]os.FileInfo, 0, count)
		ret.limit = true
	}
	// FIXME: the remote could very easily cause us to hang here.
	filesystem.Directory{Client: f.Node.Client}.List(
		context.TODO(),
		func(p filesystem.Directory_list_Params) error {
			p.SetStream(filesystem.Directory_Entry_Stream_ServerToClient(ret, nil))
			return nil
		})
	<-ret.done
	return ret.buf, ret.err
}

type FileInfo struct {
	name string
	info filesystem.StatInfo
}

func (fi *FileInfo) Name() string {
	return fi.name
}

func (fi *FileInfo) Size() int64 {
	if fi.info.Which() == filesystem.StatInfo_Which_file {
		return fi.info.File().Size()
	} else {
		return 0
	}
}

func (fi *FileInfo) Mode() os.FileMode {
	mode := os.FileMode(0400)
	if fi.info.Executable() {
		mode |= 0100
	}
	if fi.info.Writable() {
		mode |= 0200
	}
	if fi.info.Which() == filesystem.StatInfo_Which_dir {
		mode |= os.ModeDir
	}
	return mode
}

func (fi *FileInfo) ModTime() (mtime time.Time) {
	// TODO: right now the schema doesn't carry this information;
	// might want to fix that.
	return
}

func (fi *FileInfo) IsDir() bool {
	return fi.Mode().IsDir()
}

func (fi *FileInfo) Sys() interface{} {
	return fi.info
}

func (fs *FileSystem) Open(name string) (http.File, error) {
	parts := strings.Split(name, "/")
	parts = parts[1:] // remove the empty string at the start
	if len(parts) != 0 && parts[0] == "fs" {
		// TODO(cleanup): This logic ought to go elsewhere.

		// strip off the path prefix
		parts = parts[1:]
	}
	toRelease := make([]capnp.ReleaseFunc, 0, len(parts))

	var node filesystem.Node
	var dir filesystem.Directory
	node = filesystem.Node{Client: fs.Dir.Client}
	dir.Client = node.Client

	defer func() {
		for _, release := range toRelease {
			release()
		}
	}()

	for _, nodeName := range parts {
		res, release := dir.Walk(context.TODO(), func(p filesystem.Directory_walk_Params) error {
			p.SetName(nodeName)
			return nil
		})
		node := res.Node()
		toRelease = append(toRelease, release)
		dir = filesystem.Directory{node.Client}
	}
	ret, _ := node.Stat(context.TODO(), func(p filesystem.Node_stat_Params) error {
		return nil
	})

	info, err := ret.Info().Struct()
	if err != nil {
		return nil, err
	}

	var retName string
	if len(parts) == 0 {
		retName = ""
	} else {
		retName = parts[len(parts)-1]
	}
	return &File{
		Node: node,
		Info: &FileInfo{
			name: retName,
			info: cloneInfo(info),
		},
	}, nil
}
<commit_msg>httpfs: Don't shadow `node` variable.<commit_after>package httpfs

import (
	"context"
	"errors"
	"io"
	"net/http"
	"os"
	"strings"
	"time"

	"zenhack.net/go/sandstorm-filesystem/filesystem"
	"zenhack.net/go/sandstorm/exp/util/bytestream"

	"zombiezen.com/go/capnproto2"
)

var (
	InvalidArgument = errors.New("Invalid argument")
	TooManyEntries  = errors.New("Stream received too many entries")
)

// Copy StatInfo. Useful to when the original is going to be reclaimed.
func cloneInfo(info filesystem.StatInfo) filesystem.StatInfo {
	msg, _, err := capnp.NewMessage(capnp.SingleSegment(nil))
	if err != nil {
		panic(err)
	}
	msg.SetRoot(info.Struct.ToPtr())
	newInfo, err := filesystem.ReadRootStatInfo(msg)
	if err != nil {
		panic(err)
	}
	return newInfo
}

type FileSystem struct {
	Dir filesystem.Directory
}

type File struct {
	Node filesystem.Node
	Name string
	Info *FileInfo
	pos  int64
}

func (f *File) Close() error {
	return nil
}

func (f *File) Read(buf []byte) (n int, err error) {
	if f.Info.IsDir() {
		return 0, InvalidArgument
	}
	r, w := io.Pipe()
	file := filesystem.File{Client: f.Node.Client}
	file.Read(context.TODO(), func(p filesystem.File_read_Params) error {
		p.SetStartAt(f.pos)
		p.SetAmount(uint64(len(buf)))
		p.SetSink(bytestream.FromWriteCloser(w, nil))
		return nil
	})
	n, err = io.ReadFull(r, buf)
	if err == io.ErrUnexpectedEOF {
		// ReadFull expects to read the full buffer, but
		// a short read is OK for Read in general.
		err = nil
	}
	r.Close()
	return n, err
}

func (f *File) Seek(offset int64, whence int) (int64, error) {
	if f.Info.IsDir() {
		return 0, InvalidArgument
	}
	oldPos := f.pos
	switch whence {
	case io.SeekStart:
		f.pos = offset
	case io.SeekCurrent:
		f.pos += offset
	case io.SeekEnd:
		f.pos = f.Info.Size() + offset
	default:
		return f.pos, InvalidArgument
	}
	if f.pos < 0 {
		f.pos = oldPos
		return f.pos, InvalidArgument
	}
	return f.pos, nil
}

func (f *File) Stat() (os.FileInfo, error) {
	return f.Info, nil
}

type fiStream struct {
	isClosed bool
	limit    bool
	have     int
	buf      []os.FileInfo
	done     chan struct{}
	err      error
}

func (s *fiStream) Close() error {
	if !s.isClosed {
		s.isClosed = true
		s.err = io.ErrUnexpectedEOF
		s.done <- struct{}{}
	}
	return nil
}

func (s *fiStream) Done(ctx context.Context, p filesystem.Directory_Entry_Stream_done) error {
	if !s.isClosed {
		s.isClosed = true
		s.done <- struct{}{}
	}
	return nil
}

func (s *fiStream) Push(ctx context.Context, p filesystem.Directory_Entry_Stream_push) error {
	if s.isClosed {
		return s.err
	}
	entries, err := p.Args().Entries()
	if err != nil {
		return err
	}
	if s.limit && cap(s.buf)-s.have < entries.Len() {
		s.err = TooManyEntries
		s.isClosed = true
		s.done <- struct{}{}
		return s.err
	}
	for i := 0; i < entries.Len(); i++ {
		entry := entries.At(i)
		name, err := entry.Name()
		if err != nil {
			return err
		}
		info, err := entry.Info()
		if err != nil {
			return err
		}
		s.buf = append(s.buf, &FileInfo{
			name: name,
			info: cloneInfo(info),
		})
		s.have++
	}
	return nil
}

func (f *File) Readdir(count int) ([]os.FileInfo, error) {
	if !f.Info.IsDir() {
		return nil, InvalidArgument
	}
	ret := &fiStream{
		buf:  []os.FileInfo{},
		done: make(chan struct{}, 1),
	}
	if count > 0 {
		ret.buf = make([]os.FileInfo, 0, count)
		ret.limit = true
	}
	// FIXME: the remote could very easily cause us to hang here.
	filesystem.Directory{Client: f.Node.Client}.List(
		context.TODO(),
		func(p filesystem.Directory_list_Params) error {
			p.SetStream(filesystem.Directory_Entry_Stream_ServerToClient(ret, nil))
			return nil
		})
	<-ret.done
	return ret.buf, ret.err
}

type FileInfo struct {
	name string
	info filesystem.StatInfo
}

func (fi *FileInfo) Name() string {
	return fi.name
}

func (fi *FileInfo) Size() int64 {
	if fi.info.Which() == filesystem.StatInfo_Which_file {
		return fi.info.File().Size()
	} else {
		return 0
	}
}

func (fi *FileInfo) Mode() os.FileMode {
	mode := os.FileMode(0400)
	if fi.info.Executable() {
		mode |= 0100
	}
	if fi.info.Writable() {
		mode |= 0200
	}
	if fi.info.Which() == filesystem.StatInfo_Which_dir {
		mode |= os.ModeDir
	}
	return mode
}

func (fi *FileInfo) ModTime() (mtime time.Time) {
	// TODO: right now the schema doesn't carry this information;
	// might want to fix that.
	return
}

func (fi *FileInfo) IsDir() bool {
	return fi.Mode().IsDir()
}

func (fi *FileInfo) Sys() interface{} {
	return fi.info
}

func (fs *FileSystem) Open(name string) (http.File, error) {
	parts := strings.Split(name, "/")
	parts = parts[1:] // remove the empty string at the start
	if len(parts) != 0 && parts[0] == "fs" {
		// TODO(cleanup): This logic ought to go elsewhere.

		// strip off the path prefix
		parts = parts[1:]
	}
	toRelease := make([]capnp.ReleaseFunc, 0, len(parts))

	var node filesystem.Node
	var dir filesystem.Directory
	node = filesystem.Node{Client: fs.Dir.Client}
	dir.Client = node.Client

	defer func() {
		for _, release := range toRelease {
			release()
		}
	}()

	for _, nodeName := range parts {
		res, release := dir.Walk(context.TODO(), func(p filesystem.Directory_walk_Params) error {
			p.SetName(nodeName)
			return nil
		})
		node = res.Node()
		toRelease = append(toRelease, release)
		dir = filesystem.Directory{node.Client}
	}
	ret, _ := node.Stat(context.TODO(), func(p filesystem.Node_stat_Params) error {
		return nil
	})

	info, err := ret.Info().Struct()
	if err != nil {
		return nil, err
	}

	var retName string
	if len(parts) == 0 {
		retName = ""
	} else {
		retName = parts[len(parts)-1]
	}
	return &File{
		Node: node,
		Info: &FileInfo{
			name: retName,
			info: cloneInfo(info),
		},
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/gorilla/schema"
	"github.com/phyber/negroni-gzip/gzip"
	"net/http"
)

var (
	decoder  = schema.NewDecoder()
	client   = &http.Client{}
	outbound = make(chan *http.Request)
)

func MuxHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Printf("Raw Payload: %v\n", r)
	err := r.ParseForm()

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	herokuWebhookPayload := new(HerokuWebhookPayload)
	err = decoder.Decode(herokuWebhookPayload, r.PostForm)
	if err != nil {
		http.Error(w, err.Error(), 422)
		return
	}

	fmt.Printf("Recieved Heroku Deploy Webhook: %v\n", herokuWebhookPayload)

	if NewRelicIsConfigured() {
		go func() {
			handleOutboundRequest("NewRelic", NewRelicRequest(herokuWebhookPayload))
		}()
	}

	if HoneybadgerIsConfigured() {
		go func() {
			handleOutboundRequest("Honeybadger", HoneybadgerRequest(herokuWebhookPayload))
		}()
	}

	if SlackIsConfigured() {
		go func() {
			handleOutboundRequest("Slack", SlackRequest(herokuWebhookPayload))
		}()
	}

	w.WriteHeader(http.StatusAccepted)
}

func handleOutboundRequest(service string, req *http.Request) {
	resp, err := client.Do(req)
	if err != nil {
		fmt.Printf("Error: %s %v\n", service, err)
	} else {
		fmt.Printf("OK: %s %v\n", service, resp)
	}
}

func main() {

	r := mux.NewRouter()
	r.HandleFunc("/"+config.Secret, MuxHandler).Methods("POST")
	http.Handle("/", r)

	n := negroni.Classic()
	n.Use(gzip.Gzip(gzip.DefaultCompression))
	n.UseHandler(r)
	n.Run(config.ListenAddress)
}
<commit_msg>More heroku debugging<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/gorilla/schema"
	"github.com/phyber/negroni-gzip/gzip"
	"net/http"
)

var (
	decoder  = schema.NewDecoder()
	client   = &http.Client{}
	outbound = make(chan *http.Request)
)

func MuxHandler(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	herokuWebhookPayload := new(HerokuWebhookPayload)
	err = decoder.Decode(herokuWebhookPayload, r.PostForm)
	if err != nil {
		fmt.Printf("Raw Payload: %v\n", r.PostForm)
		http.Error(w, err.Error(), 422)
		return
	}

	fmt.Printf("Recieved Heroku Deploy Webhook: %v\n", herokuWebhookPayload)

	if NewRelicIsConfigured() {
		go func() {
			handleOutboundRequest("NewRelic", NewRelicRequest(herokuWebhookPayload))
		}()
	}

	if HoneybadgerIsConfigured() {
		go func() {
			handleOutboundRequest("Honeybadger", HoneybadgerRequest(herokuWebhookPayload))
		}()
	}

	if SlackIsConfigured() {
		go func() {
			handleOutboundRequest("Slack", SlackRequest(herokuWebhookPayload))
		}()
	}

	w.WriteHeader(http.StatusAccepted)
}

func handleOutboundRequest(service string, req *http.Request) {
	resp, err := client.Do(req)
	if err != nil {
		fmt.Printf("Error: %s %v\n", service, err)
	} else {
		fmt.Printf("OK: %s %v\n", service, resp)
	}
}

func main() {

	r := mux.NewRouter()
	r.HandleFunc("/"+config.Secret, MuxHandler).Methods("POST")
	http.Handle("/", r)

	n := negroni.Classic()
	n.Use(gzip.Gzip(gzip.DefaultCompression))
	n.UseHandler(r)
	n.Run(config.ListenAddress)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"
	"github.com/yuya-takeyama/argf"
)

func usage() {
	os.Stderr.WriteString(`
Usage: csvp [OPTION]... [FILE]...
Print selected parts of CSV from each FILE to standard output.

Options:
  -i, --indexes=LIST       select only these indexes
  -h, --headers=LIST       select only these headers
  -d, --delimiter=STRING   use STRING as the output delimiter (default: \t)
      --help               display this help text and exit
      --version            display version information and exit
`[1:])
}

func version() {
	os.Stderr.WriteString(`
v0.3.0
`[1:])
}

type Option struct {
	IndexesList string `short:"i" long:"indexes"`
	HeadersList string `short:"h" long:"headers"`
	Delimiter   string `short:"d" long:"delimiter" default:"\t"`
	IsHelp      bool   `          long:"help"`
	IsVersion   bool   `          long:"version"`
	Files       []string
}

func parseOption(args []string) (opt *Option, err error) {
	opt = &Option{}
	flag := flags.NewParser(opt, flags.PassDoubleDash)

	opt.Files, err = flag.ParseArgs(args)
	if err != nil && !opt.IsHelp && !opt.IsVersion {
		return nil, err
	}
	return opt, nil
}

func newCSVScannerFromOption(opt *Option) (c *CSVScanner, err error) {
	var selector Selector
	switch {
	case opt.IndexesList != "" && opt.HeadersList != "":
		return nil, fmt.Errorf("only one type of list may be specified")
	case opt.IndexesList != "":
		indexes, err := parseIndexesList(opt.IndexesList)
		if err != nil {
			return nil, err
		}
		selector = NewIndexes(indexes)
	case opt.HeadersList != "":
		headers, err := parseHeadersList(opt.HeadersList)
		if err != nil {
			return nil, err
		}
		selector = NewHeaders(headers)
	default:
		selector = NewAll()
	}

	reader, err := argf.From(opt.Files)
	if err != nil {
		return nil, err
	}

	c = NewCSVScanner(selector, reader)
	c.Delimiter = opt.Delimiter
	return c, nil
}

func do(c *CSVScanner) error {
	for c.Scan() {
		fmt.Println(c.Text())
	}
	return c.Err()
}

func printErr(err error) {
	fmt.Fprintln(os.Stderr, "cspv:", err)
}

func guideToHelp() {
	os.Stderr.WriteString(`
Try 'cspv --help' for more information.
`[1:])
}

func _main() int {
	opt, err := parseOption(os.Args[1:])
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	switch {
	case opt.IsHelp:
		usage()
		return 0
	case opt.IsVersion:
		version()
		return 0
	}

	c, err := newCSVScannerFromOption(opt)
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	if err = do(c); err != nil {
		printErr(err)
		return 1
	}
	return 0
}

func main() {
	e := _main()
	os.Exit(e)
}
<commit_msg>Version up for default selector<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"
	"github.com/yuya-takeyama/argf"
)

func usage() {
	os.Stderr.WriteString(`
Usage: csvp [OPTION]... [FILE]...
Print selected parts of CSV from each FILE to standard output.

Options:
  -i, --indexes=LIST       select only these indexes
  -h, --headers=LIST       select only these headers
  -d, --delimiter=STRING   use STRING as the output delimiter (default: \t)
      --help               display this help text and exit
      --version            display version information and exit
`[1:])
}

func version() {
	os.Stderr.WriteString(`
v0.4.0
`[1:])
}

type Option struct {
	IndexesList string `short:"i" long:"indexes"`
	HeadersList string `short:"h" long:"headers"`
	Delimiter   string `short:"d" long:"delimiter" default:"\t"`
	IsHelp      bool   `          long:"help"`
	IsVersion   bool   `          long:"version"`
	Files       []string
}

func parseOption(args []string) (opt *Option, err error) {
	opt = &Option{}
	flag := flags.NewParser(opt, flags.PassDoubleDash)

	opt.Files, err = flag.ParseArgs(args)
	if err != nil && !opt.IsHelp && !opt.IsVersion {
		return nil, err
	}
	return opt, nil
}

func newCSVScannerFromOption(opt *Option) (c *CSVScanner, err error) {
	var selector Selector
	switch {
	case opt.IndexesList != "" && opt.HeadersList != "":
		return nil, fmt.Errorf("only one type of list may be specified")
	case opt.IndexesList != "":
		indexes, err := parseIndexesList(opt.IndexesList)
		if err != nil {
			return nil, err
		}
		selector = NewIndexes(indexes)
	case opt.HeadersList != "":
		headers, err := parseHeadersList(opt.HeadersList)
		if err != nil {
			return nil, err
		}
		selector = NewHeaders(headers)
	default:
		selector = NewAll()
	}

	reader, err := argf.From(opt.Files)
	if err != nil {
		return nil, err
	}

	c = NewCSVScanner(selector, reader)
	c.Delimiter = opt.Delimiter
	return c, nil
}

func do(c *CSVScanner) error {
	for c.Scan() {
		fmt.Println(c.Text())
	}
	return c.Err()
}

func printErr(err error) {
	fmt.Fprintln(os.Stderr, "cspv:", err)
}

func guideToHelp() {
	os.Stderr.WriteString(`
Try 'cspv --help' for more information.
`[1:])
}

func _main() int {
	opt, err := parseOption(os.Args[1:])
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	switch {
	case opt.IsHelp:
		usage()
		return 0
	case opt.IsVersion:
		version()
		return 0
	}

	c, err := newCSVScannerFromOption(opt)
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	if err = do(c); err != nil {
		printErr(err)
		return 1
	}
	return 0
}

func main() {
	e := _main()
	os.Exit(e)
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package crdclient

import (
	"context"
	"fmt"
	"reflect"
	"testing"
	"time"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"

	"istio.io/istio/pilot/pkg/model"
	"istio.io/istio/pilot/pkg/serviceregistry/kube/controller"
	"istio.io/istio/pkg/config"
	"istio.io/istio/pkg/config/schema/collection"
	"istio.io/istio/pkg/config/schema/collections"
	"istio.io/istio/pkg/kube"
	"istio.io/istio/pkg/test/util/retry"
)

func makeClient(t *testing.T, schemas collection.Schemas) model.ConfigStoreCache {
	fake := kube.NewFakeClient()
	for _, s := range schemas.All() {
		fake.Ext().ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), &v1beta1.CustomResourceDefinition{
			ObjectMeta: metav1.ObjectMeta{
				Name: fmt.Sprintf("%s.%s", s.Resource().Plural(), s.Resource().Group()),
			},
		}, metav1.CreateOptions{})
	}
	stop := make(chan struct{})
	config, err := New(fake, &model.DisabledLedger{}, "", controller.Options{})
	if err != nil {
		t.Fatal(err)
	}
	go config.Run(stop)
	fake.RunAndWait(stop)
	cache.WaitForCacheSync(stop, config.HasSynced)
	t.Cleanup(func() {
		close(stop)
	})
	return config
}

// Ensure that the client can run without CRDs present
func TestClientNoCRDs(t *testing.T) {
	schema := collection.NewSchemasBuilder().MustAdd(collections.IstioNetworkingV1Alpha3Sidecars).Build()
	store := makeClient(t, schema)
	retry.UntilSuccessOrFail(t, func() error {
		if !store.HasSynced() {
			return fmt.Errorf("store has not synced yet")
		}
		return nil
	}, retry.Timeout(time.Second))
	r := collections.IstioNetworkingV1Alpha3Virtualservices.Resource()
	configMeta := config.Meta{
		Name:             "name",
		Namespace:        "ns",
		GroupVersionKind: r.GroupVersionKind(),
	}
	pb, err := r.NewInstance()
	if err != nil {
		t.Fatal(err)
	}

	if _, err := store.Create(config.Config{
		Meta: configMeta,
		Spec: pb,
	}); err != nil {
		t.Fatalf("Create => got %v", err)
	}
	retry.UntilSuccessOrFail(t, func() error {
		l, err := store.List(r.GroupVersionKind(), configMeta.Namespace)
		// List should actually not return an error in this case; this allows running with missing CRDs
		// Instead, we just return an empty list.
		if err != nil {
			return fmt.Errorf("expected no error, but got %v", err)
		}
		if len(l) != 0 {
			return fmt.Errorf("expected no items returned for unknown CRD")
		}
		return nil
	}, retry.Timeout(time.Second*5), retry.Converge(5))
	retry.UntilSuccessOrFail(t, func() error {
		l := store.Get(r.GroupVersionKind(), configMeta.Name, configMeta.Namespace)
		if l != nil {
			return fmt.Errorf("expected no items returned for unknown CRD, got %v", l)
		}
		return nil
	}, retry.Timeout(time.Second*5), retry.Converge(5))
}

// CheckIstioConfigTypes validates that an empty store can do CRUD operators on all given types
func TestClient(t *testing.T) {
	store := makeClient(t, collections.PilotServiceApi)
	configName := "name"
	configNamespace := "namespace"
	timeout := retry.Timeout(time.Millisecond * 200)
	for _, c := range collections.PilotServiceApi.All() {
		name := c.Resource().Kind()
		t.Run(name, func(t *testing.T) {
			r := c.Resource()
			configMeta := config.Meta{
				GroupVersionKind: r.GroupVersionKind(),
				Name:             configName,
			}
			if !r.IsClusterScoped() {
				configMeta.Namespace = configNamespace
			}

			pb, err := r.NewInstance()
			if err != nil {
				t.Fatal(err)
			}

			if _, err := store.Create(config.Config{
				Meta: configMeta,
				Spec: pb,
			}); err != nil {
				t.Fatalf("Create(%v) => got %v", name, err)
			}
			// Kubernetes is eventually consistent, so we allow a short time to pass before we get
			retry.UntilSuccessOrFail(t, func() error {
				cfg := store.Get(r.GroupVersionKind(), configName, configMeta.Namespace)
				if cfg == nil || !reflect.DeepEqual(cfg.Meta, configMeta) {
					return fmt.Errorf("get(%v) => got unexpected object %v", name, cfg)
				}
				return nil
			}, timeout)

			// Validate it shows up in List
			retry.UntilSuccessOrFail(t, func() error {
				cfgs, err := store.List(r.GroupVersionKind(), configNamespace)
				if err != nil {
					return err
				}
				if len(cfgs) != 1 {
					return fmt.Errorf("expected 1 config, got %v", len(cfgs))
				}
				for _, cfg := range cfgs {
					if !reflect.DeepEqual(cfg.Meta, configMeta) {
						return fmt.Errorf("get(%v) => got %v", name, cfg)
					}
				}
				return nil
			}, timeout)

			// Check we can remove items
			if err := store.Delete(r.GroupVersionKind(), configName, configNamespace); err != nil {
				t.Fatalf("failed to delete: %v", err)
			}
			retry.UntilSuccessOrFail(t, func() error {
				cfg := store.Get(r.GroupVersionKind(), configName, configNamespace)
				if cfg != nil {
					return fmt.Errorf("get(%v) => got %v, expected item to be deleted", name, cfg)
				}
				return nil
			}, timeout)
		})
	}
}
<commit_msg>initial (#27684)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package crdclient

import (
	"context"
	"fmt"
	"reflect"
	"testing"
	"time"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"

	"istio.io/istio/pilot/pkg/model"
	"istio.io/istio/pilot/pkg/serviceregistry/kube/controller"
	"istio.io/istio/pkg/config"
	"istio.io/istio/pkg/config/schema/collection"
	"istio.io/istio/pkg/config/schema/collections"
	"istio.io/istio/pkg/kube"
	"istio.io/istio/pkg/test/util/retry"
)

func makeClient(t *testing.T, schemas collection.Schemas) model.ConfigStoreCache {
	fake := kube.NewFakeClient()
	for _, s := range schemas.All() {
		fake.Ext().ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), &v1beta1.CustomResourceDefinition{
			ObjectMeta: metav1.ObjectMeta{
				Name: fmt.Sprintf("%s.%s", s.Resource().Plural(), s.Resource().Group()),
			},
		}, metav1.CreateOptions{})
	}
	stop := make(chan struct{})
	config, err := New(fake, &model.DisabledLedger{}, "", controller.Options{})
	if err != nil {
		t.Fatal(err)
	}
	go config.Run(stop)
	fake.RunAndWait(stop)
	cache.WaitForCacheSync(stop, config.HasSynced)
	t.Cleanup(func() {
		close(stop)
	})
	return config
}

// Ensure that the client can run without CRDs present
func TestClientNoCRDs(t *testing.T) {
	schema := collection.NewSchemasBuilder().MustAdd(collections.IstioNetworkingV1Alpha3Sidecars).Build()
	store := makeClient(t, schema)
	retry.UntilSuccessOrFail(t, func() error {
		if !store.HasSynced() {
			return fmt.Errorf("store has not synced yet")
		}
		return nil
	}, retry.Timeout(time.Second))
	r := collections.IstioNetworkingV1Alpha3Virtualservices.Resource()
	configMeta := config.Meta{
		Name:             "name",
		Namespace:        "ns",
		GroupVersionKind: r.GroupVersionKind(),
	}
	pb, err := r.NewInstance()
	if err != nil {
		t.Fatal(err)
	}

	if _, err := store.Create(config.Config{
		Meta: configMeta,
		Spec: pb,
	}); err != nil {
		t.Fatalf("Create => got %v", err)
	}
	retry.UntilSuccessOrFail(t, func() error {
		l, err := store.List(r.GroupVersionKind(), configMeta.Namespace)
		// List should actually not return an error in this case; this allows running with missing CRDs
		// Instead, we just return an empty list.
		if err != nil {
			return fmt.Errorf("expected no error, but got %v", err)
		}
		if len(l) != 0 {
			return fmt.Errorf("expected no items returned for unknown CRD")
		}
		return nil
	}, retry.Timeout(time.Second*5), retry.Converge(5))
	retry.UntilSuccessOrFail(t, func() error {
		l := store.Get(r.GroupVersionKind(), configMeta.Name, configMeta.Namespace)
		if l != nil {
			return fmt.Errorf("expected no items returned for unknown CRD, got %v", l)
		}
		return nil
	}, retry.Timeout(time.Second*5), retry.Converge(5))
}

// CheckIstioConfigTypes validates that an empty store can do CRUD operators on all given types
func TestClient(t *testing.T) {
	store := makeClient(t, collections.PilotServiceApi)
	configName := "name"
	configNamespace := "namespace"
	timeout := retry.Timeout(time.Millisecond * 200)
	for _, c := range collections.PilotServiceApi.All() {
		name := c.Resource().Kind()
		t.Run(name, func(t *testing.T) {
			r := c.Resource()
			configMeta := config.Meta{
				GroupVersionKind: r.GroupVersionKind(),
				Name:             configName,
			}
			if !r.IsClusterScoped() {
				configMeta.Namespace = configNamespace
			}

			pb, err := r.NewInstance()
			if err != nil {
				t.Fatal(err)
			}

			if _, err := store.Create(config.Config{
				Meta: configMeta,
				Spec: pb,
			}); err != nil {
				t.Fatalf("Create(%v) => got %v", name, err)
			}
			// Kubernetes is eventually consistent, so we allow a short time to pass before we get
			retry.UntilSuccessOrFail(t, func() error {
				cfg := store.Get(r.GroupVersionKind(), configName, configMeta.Namespace)
				if cfg == nil || !reflect.DeepEqual(cfg.Meta, configMeta) {
					return fmt.Errorf("get(%v) => got unexpected object %v", name, cfg)
				}
				return nil
			}, timeout)

			// Validate it shows up in List
			retry.UntilSuccessOrFail(t, func() error {
				cfgs, err := store.List(r.GroupVersionKind(), configNamespace)
				if err != nil {
					return err
				}
				if len(cfgs) != 1 {
					return fmt.Errorf("expected 1 config, got %v", len(cfgs))
				}
				for _, cfg := range cfgs {
					if !reflect.DeepEqual(cfg.Meta, configMeta) {
						return fmt.Errorf("get(%v) => got %v", name, cfg)
					}
				}
				return nil
			}, timeout)

			// check we can update object metadata
			annotations := map[string]string{
				"foo": "bar",
			}
			configMeta.Annotations = annotations
			if _, err := store.Update(config.Config{
				Meta: configMeta,
				Spec: pb,
			}); err != nil {
				t.Errorf("Unexpected Error in Update -> %v", err)
			}
			// validate it is updated
			retry.UntilSuccessOrFail(t, func() error {
				cfg := store.Get(r.GroupVersionKind(), configName, configMeta.Namespace)
				if cfg == nil || !reflect.DeepEqual(cfg.Meta, configMeta) {
					return fmt.Errorf("get(%v) => got unexpected object %v", name, cfg)
				}
				return nil
			})

			// Check we can remove items
			if err := store.Delete(r.GroupVersionKind(), configName, configNamespace); err != nil {
				t.Fatalf("failed to delete: %v", err)
			}
			retry.UntilSuccessOrFail(t, func() error {
				cfg := store.Get(r.GroupVersionKind(), configName, configNamespace)
				if cfg != nil {
					return fmt.Errorf("get(%v) => got %v, expected item to be deleted", name, cfg)
				}
				return nil
			}, timeout)
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/boltdb/bolt"
	"github.com/cloudfoundry-community/firehose-to-syslog/caching"
	"github.com/cloudfoundry-community/firehose-to-syslog/events"
	"github.com/cloudfoundry-community/firehose-to-syslog/firehose"
	"github.com/cloudfoundry-community/firehose-to-syslog/logging"
	"github.com/cloudfoundry-community/go-cfclient"
	"gopkg.in/alecthomas/kingpin.v2"
	"log"
	"os"
	"time"
)

var (
	debug             = kingpin.Flag("debug", "Enable debug mode. This disables forwarding to syslog").Default("false").Bool()
	apiEndpoint       = kingpin.Flag("api-address", "Api endpoint address.").Default("https://api.10.244.0.34.xip.io").String()
	dopplerAddress    = kingpin.Flag("doppler-address", "Overwrite default doppler endpoint return by /v2/info").String()
	syslogServer      = kingpin.Flag("syslog-server", "Syslog server.").String()
	subscriptionId    = kingpin.Flag("subscription-id", "Id for the subscription.").Default("firehose").String()
	user              = kingpin.Flag("user", "Admin user.").Default("admin").String()
	password          = kingpin.Flag("password", "Admin password.").Default("admin").String()
	skipSSLValidation = kingpin.Flag("skip-ssl-validation", "Please don't").Default("false").Bool()
	wantedEvents      = kingpin.Flag("events", fmt.Sprintf("Comma seperated list of events you would like. Valid options are %s", events.GetListAuthorizedEventEvents())).Default("LogMessage").String()
	boltDatabasePath  = kingpin.Flag("boltdb-path", "Bolt Database path ").Default("my.db").String()
	tickerTime        = kingpin.Flag("cc-pull-time", "CloudController Pooling time in sec").Default("60s").Duration()
)

const (
	version = "0.1.2-dev"
)

func main() {
	kingpin.Version(version)
	kingpin.Parse()
	logging.LogStd(fmt.Sprintf("Starting firehose-to-syslog %s ", version), true)

	logging.SetupLogging(*syslogServer, *debug)

	c := cfclient.Config{
		ApiAddress:        *apiEndpoint,
		Username:          *user,
		Password:          *password,
		SkipSslValidation: *skipSSLValidation,
	}
	cfClient := cfclient.NewClient(&c)

	dopplerEndpoint := cfClient.Endpoint.DopplerAddress
	if len(*dopplerAddress) > 0 {
		dopplerEndpoint = *dopplerAddress
	}
	logging.LogStd(fmt.Sprintf("Using %s as doppler endpoint", dopplerEndpoint), true)

	//Use bolt for in-memory  - file caching
	db, err := bolt.Open(*boltDatabasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal("Error opening bolt db: ", err)
		os.Exit(1)

	}
	defer db.Close()

	caching.SetCfClient(cfClient)
	caching.SetAppDb(db)
	caching.CreateBucket()

	//Let's Update the database the first time
	logging.LogStd("Start filling app/space/org cache.", true)
	apps := caching.GetAllApp()
	logging.LogStd(fmt.Sprintf("Done filling cache! Found [%d] Apps", len(apps)), true)

	logging.LogStd("Setting up event routing!", true)
	events.SetupEventRouting(*wantedEvents)

	// Ticker Pooling the CC every X sec
	ccPooling := time.NewTicker(*tickerTime)

	go func() {
		for range ccPooling.C {
			apps = caching.GetAllApp()
		}
	}()

	if logging.Connect() || *debug {

		logging.LogStd("Connected to Syslog Server! Connecting to Firehose...", true)

		firehose := firehose.CreateFirehoseChan(dopplerEndpoint, cfClient.GetToken(), *subscriptionId, *skipSSLValidation)
		if firehose != nil {
			logging.LogStd("Firehose Subscription Succesfull! Routing events...", true)
			events.RouteEvents(firehose)
		} else {
			logging.LogError("Failed connecting to Firehose...Please check settings and try again!", "")
		}

	} else {
		logging.LogError("Failed connecting to the Syslog Server...Please check settings and try again!", "")
	}

}
<commit_msg>Make api-endpoint required.<commit_after>package main

import (
	"fmt"
	"github.com/boltdb/bolt"
	"github.com/cloudfoundry-community/firehose-to-syslog/caching"
	"github.com/cloudfoundry-community/firehose-to-syslog/events"
	"github.com/cloudfoundry-community/firehose-to-syslog/firehose"
	"github.com/cloudfoundry-community/firehose-to-syslog/logging"
	"github.com/cloudfoundry-community/go-cfclient"
	"gopkg.in/alecthomas/kingpin.v2"
	"log"
	"os"
	"time"
)

var (
	debug             = kingpin.Flag("debug", "Enable debug mode. This disables forwarding to syslog").Default("false").Bool()
	apiEndpoint       = kingpin.Flag("api-address", "Api endpoint address. For bosh-lite installation of CF: https://api.10.244.0.34.xip.io").Required().String()
	dopplerAddress    = kingpin.Flag("doppler-address", "Overwrite default doppler endpoint return by /v2/info").String()
	syslogServer      = kingpin.Flag("syslog-server", "Syslog server.").String()
	subscriptionId    = kingpin.Flag("subscription-id", "Id for the subscription.").Default("firehose").String()
	user              = kingpin.Flag("user", "Admin user.").Default("admin").String()
	password          = kingpin.Flag("password", "Admin password.").Default("admin").String()
	skipSSLValidation = kingpin.Flag("skip-ssl-validation", "Please don't").Default("false").Bool()
	wantedEvents      = kingpin.Flag("events", fmt.Sprintf("Comma seperated list of events you would like. Valid options are %s", events.GetListAuthorizedEventEvents())).Default("LogMessage").String()
	boltDatabasePath  = kingpin.Flag("boltdb-path", "Bolt Database path ").Default("my.db").String()
	tickerTime        = kingpin.Flag("cc-pull-time", "CloudController Pooling time in sec").Default("60s").Duration()
)

const (
	version = "0.1.2-dev"
)

func main() {
	kingpin.Version(version)
	kingpin.Parse()
	logging.LogStd(fmt.Sprintf("Starting firehose-to-syslog %s ", version), true)

	logging.SetupLogging(*syslogServer, *debug)

	c := cfclient.Config{
		ApiAddress:        *apiEndpoint,
		Username:          *user,
		Password:          *password,
		SkipSslValidation: *skipSSLValidation,
	}
	cfClient := cfclient.NewClient(&c)

	dopplerEndpoint := cfClient.Endpoint.DopplerAddress
	if len(*dopplerAddress) > 0 {
		dopplerEndpoint = *dopplerAddress
	}
	logging.LogStd(fmt.Sprintf("Using %s as doppler endpoint", dopplerEndpoint), true)

	//Use bolt for in-memory  - file caching
	db, err := bolt.Open(*boltDatabasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal("Error opening bolt db: ", err)
		os.Exit(1)

	}
	defer db.Close()

	caching.SetCfClient(cfClient)
	caching.SetAppDb(db)
	caching.CreateBucket()

	//Let's Update the database the first time
	logging.LogStd("Start filling app/space/org cache.", true)
	apps := caching.GetAllApp()
	logging.LogStd(fmt.Sprintf("Done filling cache! Found [%d] Apps", len(apps)), true)

	logging.LogStd("Setting up event routing!", true)
	events.SetupEventRouting(*wantedEvents)

	// Ticker Pooling the CC every X sec
	ccPooling := time.NewTicker(*tickerTime)

	go func() {
		for range ccPooling.C {
			apps = caching.GetAllApp()
		}
	}()

	if logging.Connect() || *debug {

		logging.LogStd("Connected to Syslog Server! Connecting to Firehose...", true)

		firehose := firehose.CreateFirehoseChan(dopplerEndpoint, cfClient.GetToken(), *subscriptionId, *skipSSLValidation)
		if firehose != nil {
			logging.LogStd("Firehose Subscription Succesfull! Routing events...", true)
			events.RouteEvents(firehose)
		} else {
			logging.LogError("Failed connecting to Firehose...Please check settings and try again!", "")
		}

	} else {
		logging.LogError("Failed connecting to the Syslog Server...Please check settings and try again!", "")
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	"runtime"

	"github.com/gorilla/mux"
	"github.com/op/go-logging"
)

// Set the default, min and max width to resize processed images to.
const (
	DefaultWidth = uint(180)
	MinWidth     = uint(8)
	MaxWidth     = uint(300)

	ImgdVersion = "2.9.1"
)

var (
	config        = &Configuration{}
	cache         Cache
	stats         *StatusCollector
	signalHandler *SignalHandler
)

var log = logging.MustGetLogger("imgd")
var format = "[%{time:15:04:05.000000}] %{level:.4s} %{message}"

func setupConfig() {
	err := config.load()
	if err != nil {
		fmt.Printf("Error loading config: %s\n", err)
		return
	}
}

func setupCache() {
	cache = MakeCache(config.Server.Cache)
	err := cache.setup()
	if err != nil {
		log.Critical("Unable to setup Cache. (" + fmt.Sprintf("%v", err) + ")")
		os.Exit(1)
	}
}

func setupLog(logBackend *logging.LogBackend) {
	logging.SetBackend(logBackend)
	logging.SetFormatter(logging.MustStringFormatter(format))
	logLevel, err := logging.LogLevel(config.Server.Logging)
	logging.SetLevel(logLevel, "")
	if err != nil {
		log.Error("Invalid log type: %s", config.Server.Logging)
		// If error it sets the logging to ERROR, let's change it to INFO
		logging.SetLevel(4, "")
	}
	log.Notice("Log level set to %s", logging.GetLevel(""))
}

func startServer() {
	r := Router{Mux: mux.NewRouter()}
	r.Bind()
	http.Handle("/", imgdHandler(r.Mux))
	log.Notice("imgd %s starting on %s", ImgdVersion, config.Server.Address)
	err := http.ListenAndServe(config.Server.Address, nil)
	if err != nil {
		log.Critical("ListenAndServe: \"%s\"", err.Error())
		os.Exit(1)
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	logBackend := logging.NewLogBackend(os.Stdout, "", 0)

	signalHandler = MakeSignalHandler()
	stats = MakeStatsCollector()
	setupConfig()
	setupLog(logBackend)
	setupCache()
	startServer()
}
<commit_msg>Bump version to 2.9.2<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"runtime"

	"github.com/gorilla/mux"
	"github.com/op/go-logging"
)

// Set the default, min and max width to resize processed images to.
const (
	DefaultWidth = uint(180)
	MinWidth     = uint(8)
	MaxWidth     = uint(300)

	ImgdVersion = "2.9.2"
)

var (
	config        = &Configuration{}
	cache         Cache
	stats         *StatusCollector
	signalHandler *SignalHandler
)

var log = logging.MustGetLogger("imgd")
var format = "[%{time:15:04:05.000000}] %{level:.4s} %{message}"

func setupConfig() {
	err := config.load()
	if err != nil {
		fmt.Printf("Error loading config: %s\n", err)
		return
	}
}

func setupCache() {
	cache = MakeCache(config.Server.Cache)
	err := cache.setup()
	if err != nil {
		log.Critical("Unable to setup Cache. (" + fmt.Sprintf("%v", err) + ")")
		os.Exit(1)
	}
}

func setupLog(logBackend *logging.LogBackend) {
	logging.SetBackend(logBackend)
	logging.SetFormatter(logging.MustStringFormatter(format))
	logLevel, err := logging.LogLevel(config.Server.Logging)
	logging.SetLevel(logLevel, "")
	if err != nil {
		log.Error("Invalid log type: %s", config.Server.Logging)
		// If error it sets the logging to ERROR, let's change it to INFO
		logging.SetLevel(4, "")
	}
	log.Notice("Log level set to %s", logging.GetLevel(""))
}

func startServer() {
	r := Router{Mux: mux.NewRouter()}
	r.Bind()
	http.Handle("/", imgdHandler(r.Mux))
	log.Notice("imgd %s starting on %s", ImgdVersion, config.Server.Address)
	err := http.ListenAndServe(config.Server.Address, nil)
	if err != nil {
		log.Critical("ListenAndServe: \"%s\"", err.Error())
		os.Exit(1)
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	logBackend := logging.NewLogBackend(os.Stdout, "", 0)

	signalHandler = MakeSignalHandler()
	stats = MakeStatsCollector()
	setupConfig()
	setupLog(logBackend)
	setupCache()
	startServer()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/jessevdk/go-flags"
)

// GlobalOptions contains all global options.
type GlobalOptions struct {
	Verbose      bool   `short:"v" long:"verbose"                     description:"Be verbose"`
	Config       string `short:"C" long:"config"                      description:"Read config from this file"`
	DryRun       bool   `short:"n" long:"dry-run"                     description:"Only print what commands would be executed without actually runnig them"`
	PollInterval uint   `short:"i" long:"interval"    default:"5"     description:"Number of seconds between polls, set to zero to disable polling"`
	Pause        uint   `short:"p" long:"pause"       default:"2"     description:"Number of seconds to pause after a change was executed"`
	Logfile      string `short:"l" long:"logfile"                     description:"Write log to file"`

	cfg     *Config
	log     *log.Logger
	logfile *log.Logger
}

func (gopts *GlobalOptions) ReadConfigfile() {
	if gopts.cfg != nil {
		return
	}

	cfg, err := readConfig(gopts.Config)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error reading config file: %v\n", err)
		os.Exit(1)
	}

	gopts.cfg = &cfg
}

// RunCommand runs the given command or prints the arguments to stdout if
// globalOpts.DryRun is true.
func RunCommand(cmd *exec.Cmd) error {
	if globalOpts.DryRun {
		s := fmt.Sprintf("%s", cmd.Args)
		fmt.Printf("%s\n", s[1:len(s)-1])
		return nil
	}

	V("running command %v %v\n", cmd.Path, strings.Join(cmd.Args, " "))
	cmd.Stderr = os.Stderr
	if globalOpts.Verbose {
		cmd.Stdout = os.Stdout
	}
	return cmd.Run()
}

var globalOpts = GlobalOptions{}
var parser = flags.NewParser(&globalOpts, flags.Default)

func V(s string, data ...interface{}) {
	if globalOpts.Verbose && globalOpts.log == nil {
		globalOpts.log = log.New(os.Stdout, "grobi: ", log.Lmicroseconds|log.Ltime)
	}

	if globalOpts.log != nil {
		globalOpts.log.Printf(s, data...)
	}

	if globalOpts.Logfile != "" && globalOpts.logfile == nil {
		f, err := os.OpenFile(globalOpts.Logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to open logfile: %v\n", err)
			os.Exit(23)
		}
		globalOpts.logfile = log.New(f, "", log.Lmicroseconds|log.Ltime)
	}

	if globalOpts.logfile != nil {
		globalOpts.logfile.Printf(s, data...)
	}
}

func main() {
	_, err := parser.Parse()
	if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
		os.Exit(0)
	}

	if err != nil {
		os.Exit(1)
	}
}
<commit_msg>Set default pause to zero<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/jessevdk/go-flags"
)

// GlobalOptions contains all global options.
type GlobalOptions struct {
	Verbose      bool   `short:"v" long:"verbose"                     description:"Be verbose"`
	Config       string `short:"C" long:"config"                      description:"Read config from this file"`
	DryRun       bool   `short:"n" long:"dry-run"                     description:"Only print what commands would be executed without actually runnig them"`
	PollInterval uint   `short:"i" long:"interval"    default:"5"     description:"Number of seconds between polls, set to zero to disable polling"`
	Pause        uint   `short:"p" long:"pause"       default:"0"     description:"Number of seconds to pause after a change was executed"`
	Logfile      string `short:"l" long:"logfile"                     description:"Write log to file"`

	cfg     *Config
	log     *log.Logger
	logfile *log.Logger
}

func (gopts *GlobalOptions) ReadConfigfile() {
	if gopts.cfg != nil {
		return
	}

	cfg, err := readConfig(gopts.Config)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error reading config file: %v\n", err)
		os.Exit(1)
	}

	gopts.cfg = &cfg
}

// RunCommand runs the given command or prints the arguments to stdout if
// globalOpts.DryRun is true.
func RunCommand(cmd *exec.Cmd) error {
	if globalOpts.DryRun {
		s := fmt.Sprintf("%s", cmd.Args)
		fmt.Printf("%s\n", s[1:len(s)-1])
		return nil
	}

	V("running command %v %v\n", cmd.Path, strings.Join(cmd.Args, " "))
	cmd.Stderr = os.Stderr
	if globalOpts.Verbose {
		cmd.Stdout = os.Stdout
	}
	return cmd.Run()
}

var globalOpts = GlobalOptions{}
var parser = flags.NewParser(&globalOpts, flags.Default)

func V(s string, data ...interface{}) {
	if globalOpts.Verbose && globalOpts.log == nil {
		globalOpts.log = log.New(os.Stdout, "grobi: ", log.Lmicroseconds|log.Ltime)
	}

	if globalOpts.log != nil {
		globalOpts.log.Printf(s, data...)
	}

	if globalOpts.Logfile != "" && globalOpts.logfile == nil {
		f, err := os.OpenFile(globalOpts.Logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to open logfile: %v\n", err)
			os.Exit(23)
		}
		globalOpts.logfile = log.New(f, "", log.Lmicroseconds|log.Ltime)
	}

	if globalOpts.logfile != nil {
		globalOpts.logfile.Printf(s, data...)
	}
}

func main() {
	_, err := parser.Parse()
	if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
		os.Exit(0)
	}

	if err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/kelseyhightower/envconfig"
)

const (
	Version = "1.1.0"
)

type config struct {
	// Flickr
	PhotoAlbum      string `envconfig:"FLICKR_ALBUM" required:"true"`
	Key             string `envconfig:"FLICKR_KEY" required:"true"`
	RefreshInterval int    `envconfig:"REFRESH_INTERVAL" default:15"` // in minutes

	// API Server
	Host string `envconfig:"HOST" default:""`
	Port int    `envconfig:"PORT" default:"3000"`
}

func main() {
	c, err := getConfig()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading configuration from environment (%s).\n", err)
		os.Exit(1)
	}

	StartUpdatingPhotos(c)
	err = StartApiServer(c)
	fmt.Fprintf(os.Stderr, "Error starting http server: %s\n", err)
}

func getConfig() (config, error) {
	var c config
	err := envconfig.Process("PHOTO_SERVICE", &c)
	return c, err
}
<commit_msg>Bump version number<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/kelseyhightower/envconfig"
)

const (
	Version = "1.1.1"
)

type config struct {
	// Flickr
	PhotoAlbum      string `envconfig:"FLICKR_ALBUM" required:"true"`
	Key             string `envconfig:"FLICKR_KEY" required:"true"`
	RefreshInterval int    `envconfig:"REFRESH_INTERVAL" default:15"` // in minutes

	// API Server
	Host string `envconfig:"HOST" default:""`
	Port int    `envconfig:"PORT" default:"3000"`
}

func main() {
	c, err := getConfig()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading configuration from environment (%s).\n", err)
		os.Exit(1)
	}

	StartUpdatingPhotos(c)
	err = StartApiServer(c)
	fmt.Fprintf(os.Stderr, "Error starting http server: %s\n", err)
}

func getConfig() (config, error) {
	var c config
	err := envconfig.Process("PHOTO_SERVICE", &c)
	return c, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"
)

const (
	version         = "0.2"
	cacheExpiration = 12 * time.Hour
)

func main() {
	log.Printf("Kathisto v%s - Server-Side rendering with Go/PhantomJS\n", version)
	pubDir := os.Getenv("PUBLIC_DIR")
	if pubDir == "" {
		pubDir = "/dist"
	}

	// Create a PhantomJS renderer and attach the prerender func to /
	r := NewPJSRenderer(cacheExpiration, fmt.Sprintf("Kathisto/%s", version))
	rs := NewService(r, os.Getenv("STRICT_HOST"), pubDir)
	http.HandleFunc("/", rs.Prerender)

	// Spin up a TLS goroutine if a cert and key are found
	certFile, keyFile := os.Getenv("CERT_FILE"), os.Getenv("KEY_FILE")
	if certFile != "" && keyFile != "" {
		log.Println("Listening on port :443")
		go http.ListenAndServeTLS(":443", certFile, keyFile, nil)
	}

	// Always run a basic http server
	log.Println("Listening on port :80")
	http.ListenAndServe(":80", nil)
}
<commit_msg>Getting customer port from environment<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"
)

const (
	version         = "0.2"
	cacheExpiration = 12 * time.Hour
)

func main() {
	log.Printf("Kathisto v%s - Server-Side rendering with Go/PhantomJS\n", version)
	pubDir := os.Getenv("PUBLIC_DIR")
	if pubDir == "" {
		pubDir = "/dist"
	}

	// Create a PhantomJS renderer and attach the prerender func to /
	r := NewPJSRenderer(cacheExpiration, fmt.Sprintf("Kathisto/%s", version))
	rs := NewService(r, os.Getenv("STRICT_HOST"), pubDir)
	http.HandleFunc("/", rs.Prerender)

	// Spin up a TLS goroutine if a cert and key are found
	certFile, keyFile := os.Getenv("CERT_FILE"), os.Getenv("KEY_FILE")
	if certFile != "" && keyFile != "" {
		log.Println("Listening on port :443")
		go http.ListenAndServeTLS(":443", certFile, keyFile, nil)
	}

	port := os.Getenv("PORT")
	if port == "" {
		port = "80"
	}

	// Always run a basic http server
	log.Println("Listening on port :", port)
	http.ListenAndServe(":"+port, nil)
}
<|endoftext|>


<commit_before>package main

// cors-proxy is a reverse proxy to allow cross origin
// requests (eg. from a javascript XHTTPRequest) to
// another service that doesn't respond to OPTIONS requests.
//
// Author: Patrice FERLET <metal3d@gmail.com>
// Licence: BSD

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
)

var (
	// host:port to connect to.
	portTo = "127.0.0.1:8000"
	// host:port to listen.
	listen = "0.0.0.0:3000"
	// verbose message.
	verbose = false
)

// handleReverseRequest writes back the server response to client.
// If an "OPTIONS" request is called, we
// only return Access-Control-Allow-* to let XHttpRequest working.
func handleReverseRequest(w http.ResponseWriter, r *http.Request) {

	// check scheme
	scheme := "http"
	if r.TLS != nil {
		scheme = "https"
	}

	// build url
	toCall := fmt.Sprintf("%s://%s%s", scheme, portTo, r.URL.String())
	debug("Create request for ", toCall)

	// always allow access origin
	w.Header().Add("Access-Control-Allow-Origin", "*")

	if r.Method == "OPTIONS" {
		debug("CORS asked for ", toCall)
		for n, h := range r.Header {
			if strings.Contains(n, "Access-Control-Request") {
				for _, h := range h {
					k := strings.Replace(n, "Request", "Allow", 1)
					w.Header().Add(k, h)
				}
			}
		}
		// end
		return
	}

	// create request to server
	req, err := http.NewRequest(r.Method, toCall, r.Body)

	// add ALL header to the connection
	for n, h := range r.Header {
		for _, h := range h {
			req.Header.Add(n, h)
		}
	}

	// create a basic client to send request
	client := http.Client{}
	if r.TLS != nil {
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(err.Error()))
		return
	}

	// copy the reponse from server to the connected client request
	w.WriteHeader(resp.StatusCode)
	for h, v := range resp.Header {
		for _, v := range v {
			w.Header().Add(h, v)
		}
	}

	wr, err := io.Copy(w, resp.Body)
	if err != nil {
		log.Println(wr, err)
	} else {
		debug("Writen", wr, "bytes")
	}

}

// validateFlags checks if host:port format is ok.
func validateFlags() {
	for _, f := range []string{portTo, listen} {
		if !strings.Contains(f, ":") {
			log.Fatalf("%s is not right, you must use a coma mark to separate host and port", f)
		}

	}

	parts := strings.Split(portTo, ":")
	if parts[0] == "" {
		log.Println("You didn't set host to connect, using 127.0.0.1:" + parts[1])
		portTo = "127.0.0.1:" + parts[1]
	}

}

// debug writes message when verbose flag is true.
func debug(v ...interface{}) {
	if verbose {
		log.Println(v...)
	}
}

func main() {
	flag.StringVar(&portTo, "p", portTo, "service port")
	flag.StringVar(&listen, "l", listen, "listen interface")
	flag.BoolVar(&verbose, "v", verbose, "verbose")
	flag.Parse()

	validateFlags()
	http.HandleFunc("/", handleReverseRequest)
	log.Println(listen, "-->", portTo)
	http.ListenAndServe(listen, nil)
}
<commit_msg>Fixup a problem on certain CORS request<commit_after>package main

// cors-proxy is a reverse proxy to allow cross origin
// requests (eg. from a javascript XHTTPRequest) to
// another service that doesn't respond to OPTIONS requests.
//
// Author: Patrice FERLET <metal3d@gmail.com>
// Licence: BSD

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
)

var (
	// host:port to connect to.
	portTo = "127.0.0.1:8000"
	// host:port to listen.
	listen = "0.0.0.0:3000"
	// verbose message.
	verbose = false
)

// handleReverseRequest writes back the server response to client.
// If an "OPTIONS" request is called, we
// only return Access-Control-Allow-* to let XHttpRequest working.
func handleReverseRequest(w http.ResponseWriter, r *http.Request) {

	// check scheme
	scheme := "http"
	if r.TLS != nil {
		scheme = "https"
	}

	// build url
	toCall := fmt.Sprintf("%s://%s%s", scheme, portTo, r.URL.String())
	debug("Create request for ", toCall)

	// always allow access origin
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "GET, PUT, POST, HEAD, TRACE, DELETE, PATCH, COPY, HEAD, LINK, OPTIONS")

	if r.Method == "OPTIONS" {
		debug("CORS asked for ", toCall)
		for n, h := range r.Header {
			if strings.Contains(n, "Access-Control-Request") {
				for _, h := range h {
					k := strings.Replace(n, "Request", "Allow", 1)
					w.Header().Add(k, h)
				}
			}
		}
		// end
		return
	}

	// create request to server
	req, err := http.NewRequest(r.Method, toCall, r.Body)

	// add ALL header to the connection
	for n, h := range r.Header {
		for _, h := range h {
			req.Header.Add(n, h)
		}
	}

	// create a basic client to send request
	client := http.Client{}
	if r.TLS != nil {
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(err.Error()))
		return
	}

	// copy the reponse from server to the connected client request
	w.WriteHeader(resp.StatusCode)
	for h, v := range resp.Header {
		for _, v := range v {
			w.Header().Add(h, v)
		}
	}

	wr, err := io.Copy(w, resp.Body)
	if err != nil {
		log.Println(wr, err)
	} else {
		debug("Writen", wr, "bytes")
	}

}

// validateFlags checks if host:port format is ok.
func validateFlags() {
	for _, f := range []string{portTo, listen} {
		if !strings.Contains(f, ":") {
			log.Fatalf("%s is not right, you must use a coma mark to separate host and port", f)
		}

	}

	parts := strings.Split(portTo, ":")
	if parts[0] == "" {
		log.Println("You didn't set host to connect, using 127.0.0.1:" + parts[1])
		portTo = "127.0.0.1:" + parts[1]
	}

}

// debug writes message when verbose flag is true.
func debug(v ...interface{}) {
	if verbose {
		log.Println(v...)
	}
}

func main() {
	flag.StringVar(&portTo, "p", portTo, "service port")
	flag.StringVar(&listen, "l", listen, "listen interface")
	flag.BoolVar(&verbose, "v", verbose, "verbose")
	flag.Parse()

	validateFlags()
	http.HandleFunc("/", handleReverseRequest)
	log.Println(listen, "-->", portTo)
	http.ListenAndServe(listen, nil)
}
<|endoftext|>