<commit_before>package jwtauthd

import (
    "github.com/spf13/cobra"
    "github.com/spf13/pflag"
    "github.com/spf13/viper"
    "fmt"
)

type Args struct {
    Port int
    Conf string
}

type JwtAuthCommand struct {
    Args  Args
    Cmd   *cobra.Command
    Viper *viper.Viper
}

var JwtAuth *JwtAuthCommand = &JwtAuthCommand{}

func UsageTemplate() string {
    return `Usage:{{if .Runnable}}
  {{if .HasAvailableFlags}}{{appendIfNotPresent .UseLine "[OPTIONS] COMMAND [arg...]"}}{{else}}{{.UseLine}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
  {{ .CommandPath}} [command]{{end}}{{if gt .Aliases 0}}
Aliases:
  {{.NameAndAliases}}
{{end}}{{if .HasExample}}
Examples:
{{ .Example }}{{end}}{{ if .HasAvailableSubCommands}}
Commands:{{range .Commands}}{{if .IsAvailableCommand}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableLocalFlags}}
Options:
{{.LocalFlags.FlagUsages | trimRightSpace}}{{end}}{{ if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsHelpCommand}}
  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableSubCommands }}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`
}

func init() {
    JwtAuth.Viper = viper.GetViper()
    JwtAuth.Cmd = &cobra.Command{
        Use: "jwt-authd",
        Short: "Jwt auth server",
        Long: "User login information verification service",
        SilenceErrors: true,
        RunE: func(cmd *cobra.Command, args []string) error {
            // todo
            fmt.Println(JwtAuth.Args)
            return nil
        },
        PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
            // todo

            return nil
        },
    }
    JwtAuth.Cmd.SetUsageTemplate(UsageTemplate())

    var PFlags *pflag.FlagSet = JwtAuth.Cmd.Flags()

    PFlags.IntVarP(&JwtAuth.Args.Port, "port", "p", 6010, "set the server listening port")
    PFlags.StringVarP(&JwtAuth.Args.Conf, "config", "c", "/etc/jwt_authd.json", "configuration file specifying")

    JwtAuth.Viper.BindPFlag("port", PFlags.Lookup("port"));

    // todo
    JwtAuth.Viper.SetConfigFile(JwtAuth.Args.Conf)
    if err := JwtAuth.Viper.ReadConfig(); err != nil {
        fmt.Println("can not read config: ", err)
    }
}<commit_msg>debug<commit_after>package jwtauthd

import (
    "github.com/spf13/cobra"
    "github.com/spf13/pflag"
    "github.com/spf13/viper"
    "fmt"
)

type Args struct {
    Port int
    Conf string
}

type JwtAuthCommand struct {
    Args  Args
    Cmd   *cobra.Command
    Viper *viper.Viper
}

var JwtAuth *JwtAuthCommand = &JwtAuthCommand{}

func UsageTemplate() string {
    return `Usage:{{if .Runnable}}
  {{if .HasAvailableFlags}}{{appendIfNotPresent .UseLine "[OPTIONS] COMMAND [arg...]"}}{{else}}{{.UseLine}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
  {{ .CommandPath}} [command]{{end}}{{if gt .Aliases 0}}
Aliases:
  {{.NameAndAliases}}
{{end}}{{if .HasExample}}
Examples:
{{ .Example }}{{end}}{{ if .HasAvailableSubCommands}}
Commands:{{range .Commands}}{{if .IsAvailableCommand}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableLocalFlags}}
Options:
{{.LocalFlags.FlagUsages | trimRightSpace}}{{end}}{{ if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsHelpCommand}}
  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableSubCommands }}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`
}

func init() {
    JwtAuth.Viper = viper.GetViper()
    JwtAuth.Cmd = &cobra.Command{
        Use: "jwt-authd",
        Short: "Jwt auth server",
        Long: "User login information verification service",
        SilenceErrors: true,
        RunE: func(cmd *cobra.Command, args []string) error {
            // todo
            fmt.Println(JwtAuth.Viper.Get("port"))
            return nil
        },
        PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
            // todo

            return nil
        },
    }
    JwtAuth.Cmd.SetUsageTemplate(UsageTemplate())

    var PFlags *pflag.FlagSet = JwtAuth.Cmd.Flags()

    PFlags.IntVarP(&JwtAuth.Args.Port, "port", "p", 6010, "set the server listening port")
    PFlags.StringVarP(&JwtAuth.Args.Conf, "config", "c", "/etc/jwt_authd.json", "configuration file specifying")

    JwtAuth.Viper.BindPFlag("port", PFlags.Lookup("port"));

    // todo
    JwtAuth.Viper.SetConfigFile(JwtAuth.Args.Conf)
    if err := JwtAuth.Viper.ReadInConfig(); err != nil {
        fmt.Println("can not read config: ", err)
    }
}<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"flag"
	"fmt"
	"github.com/google/git-appraise/commands/output"
	"github.com/google/git-appraise/repository"
	"github.com/google/git-appraise/review"
)

var listFlagSet = flag.NewFlagSet("list", flag.ExitOnError)

var (
	listAll = listFlagSet.Bool("a", false, "List all reviews (not just the open ones).")
)

// listReviews lists all extant reviews.
// TODO(ojarjur): Add more flags for filtering the output (e.g. filtering by reviewer or status).
func listReviews(repo repository.Repo, args []string) {
	listFlagSet.Parse(args)
	var reviews []review.Summary
	if *listAll {
		reviews = review.ListAll(repo)
		fmt.Printf("Loaded %d reviews:\n", len(reviews))
	} else {
		reviews = review.ListOpen(repo)
		fmt.Printf("Loaded %d open reviews:\n", len(reviews))
	}
	for _, r := range reviews {
		output.PrintSummary(&r)
	}
}

// listCmd defines the "list" subcommand.
var listCmd = &Command{
	Usage: func(arg0 string) {
		fmt.Printf("Usage: %s list [<option>...]\n\nOptions:\n", arg0)
		listFlagSet.PrintDefaults()
	},
	RunMethod: func(repo repository.Repo, args []string) error {
		listReviews(repo, args)
		return nil
	},
}
<commit_msg>Add -json flag for list command<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/google/git-appraise/commands/output"
	"github.com/google/git-appraise/repository"
	"github.com/google/git-appraise/review"
)

var listFlagSet = flag.NewFlagSet("list", flag.ExitOnError)

var (
	listAll        = listFlagSet.Bool("a", false, "List all reviews (not just the open ones).")
	listJSONOutput = listFlagSet.Bool("json", false, "Format the output as JSON")
)

// listReviews lists all extant reviews.
// TODO(ojarjur): Add more flags for filtering the output (e.g. filtering by reviewer or status).
func listReviews(repo repository.Repo, args []string) error {
	listFlagSet.Parse(args)
	var reviews []review.Summary
	if *listAll {
		reviews = review.ListAll(repo)
		if !*listJSONOutput {
			fmt.Printf("Loaded %d reviews:\n", len(reviews))
		}
	} else {
		reviews = review.ListOpen(repo)
		if !*listJSONOutput {
			fmt.Printf("Loaded %d open reviews:\n", len(reviews))
		}
	}
	if *listJSONOutput {
		b, err := json.MarshalIndent(reviews, "", "  ")
		if err != nil {
			return err
		}
		fmt.Println(string(b))
		return nil
	}
	for _, r := range reviews {
		output.PrintSummary(&r)
	}
	return nil
}

// listCmd defines the "list" subcommand.
var listCmd = &Command{
	Usage: func(arg0 string) {
		fmt.Printf("Usage: %s list [<option>...]\n\nOptions:\n", arg0)
		listFlagSet.PrintDefaults()
	},
	RunMethod: func(repo repository.Repo, args []string) error {
		return listReviews(repo, args)
	},
}
<|endoftext|>


<commit_before>// Command kepubify is a fast, standalone, EPUB to KEPUB converter.
package main

import (
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pgaskin/kepubify/v4/internal/zip"
	"github.com/pgaskin/kepubify/v4/kepub"
	"github.com/spf13/pflag"
)

var version = "v3-dev"

func main() {
	pflag.CommandLine.SortFlags = false

	verbose := pflag.BoolP("verbose", "v", false, "Show extra information in output")
	sversion := pflag.Bool("version", false, "Show the version")
	help := pflag.BoolP("help", "h", false, "Show this help text")

	for _, flag := range []string{"verbose", "version", "help"} {
		pflag.CommandLine.SetAnnotation(flag, "category", []string{"1.General Options"})
	}

	update := pflag.BoolP("update", "u", false, "Don't reconvert files which have already been converted (i.e. don't overwrite output files)")
	inplace := pflag.BoolP("inplace", "i", false, "Don't add the _converted suffix to converted files and directories")
	nopreservedirs := pflag.Bool("no-preserve-dirs", false, "Flatten the directory structure of the input (an error will be shown if there are conflicts)")
	output := pflag.StringP("output", "o", "", "[>1 inputs || 1 file input with existing dir output]: Directory to place converted files/dirs under; [1 file input with nonexistent output]: Output filename; [1 dir input]: Output directory for contents of input (default: current directory)")
	calibre := pflag.Bool("calibre", false, "Use .kepub instead of .kepub.epub as the output extension (for Calibre compatibility, only use if you know what you are doing)")
	copy := pflag.StringSliceP("copy", "x", nil, "Copy files with the specified extension (with a leading period) to the output unchanged (no effect if the filename ends up the same)")

	for _, flag := range []string{"update", "inplace", "no-preserve-dirs", "output", "calibre", "copy"} {
		pflag.CommandLine.SetAnnotation(flag, "category", []string{"2.Output Options"})
	}

	smartenpunct := pflag.Bool("smarten-punctuation", false, "Smarten punctuation (smart quotes, dashes, etc) (excluding pre and code tags)")
	css := pflag.StringArrayP("css", "c", nil, "Custom CSS to add to ebook")
	hyphenate := pflag.Bool("hyphenate", false, "Force enable hyphenation")
	nohyphenate := pflag.Bool("no-hyphenate", false, "Force disable hyphenation")
	fullscreenfixes := pflag.Bool("fullscreen-reading-fixes", false, "Enable fullscreen reading bugfixes based on https://www.mobileread.com/forums/showpost.php?p=3113460&postcount=16")
	replace := pflag.StringArrayP("replace", "r", nil, "Find and replace on all html files (repeat any number of times) (format: find|replace)")

	for _, flag := range []string{"smarten-punctuation", "css", "hyphenate", "no-hyphenate", "fullscreen-reading-fixes", "replace"} {
		pflag.CommandLine.SetAnnotation(flag, "category", []string{"3.Conversion Options"})
	}

	// --- Parse options --- //

	pflag.Parse()

	if *sversion {
		fmt.Printf("kepubify %s\n", version)
		exit(0)
		return
	}

	if *help || pflag.NArg() == 0 {
		helpExit()
		return
	}

	if *hyphenate && *nohyphenate {
		fmt.Printf("Error: --hyphenate and --no-hyphenate are mutally exclusive. See --help for more details.\n")
		exit(2)
		return
	}

	for _, c := range *copy {
		if len(c) == 0 || c[0] != '.' {
			fmt.Printf("Error: --copy argument %#v doesn't have a leading period. See --help for more details.\n", c)
			exit(2)
			return
		}
	}

	// --- Make converter --- //

	var opts []kepub.ConverterOption
	for _, v := range *css {
		opts = append(opts, kepub.ConverterOptionAddCSS(v))
	}
	if *hyphenate {
		opts = append(opts, kepub.ConverterOptionHyphenate(true))
	} else if *nohyphenate {
		opts = append(opts, kepub.ConverterOptionHyphenate(false))
	}
	if *smartenpunct {
		opts = append(opts, kepub.ConverterOptionSmartypants())
	}
	if *fullscreenfixes {
		opts = append(opts, kepub.ConverterOptionFullScreenFixes())
	}
	for _, r := range *replace {
		spl := strings.SplitN(r, "|", 2)
		if len(spl) != 2 {
			fmt.Fprintf(os.Stderr, "Error: Parse replacement %#v: must be in format `find|replace`\n", r)
			exit(1)
			return
		}
		opts = append(opts, kepub.ConverterOptionFindReplace(spl[0], spl[1]))
	}
	converter := kepub.NewConverterWithOptions(opts...)

	// --- Transform paths --- //

	ext := ".kepub.epub"
	if *calibre {
		ext = ".kepub"
	}

	pathMap, skipList, err := transformer{
		NoPreserveDirs:   *nopreservedirs,
		Update:           *update,
		Inplace:          *inplace,
		Suffixes:         []string{".epub"},
		ExcludeSuffixes:  []string{".kepub.epub"},
		PreserveSuffixes: *copy,
		TargetSuffix:     ext,
	}.TransformPaths(*output, pflag.Args()...)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		exit(1)
		return
	}

	// --- Convert --- //

	fmt.Printf("Kepubify %s\n", version)
	if *calibre {
		fmt.Printf("Using extension %s for Calibre compatibility (this is meant for use with a Calibre library and will not work directly on a Kobo reader)\n", ext)
	}
	fmt.Printf("\n")

	var inputs []string
	for key := range pathMap {
		inputs = append(inputs, key)
	}
	inputs = append(inputs, skipList...)
	sort.Strings(inputs)

	// --- Conversion Pipeline --- //
	var cur int64                                 // progress
	var converted, copied, skipped, errored int64 // counters (sum == total)
	var errs sync.Map                             // input -> error
	total := int64(len(pathMap) + len(skipList))  // immutable

	var allWg sync.WaitGroup

	// logging

	type loge struct {
		stderr bool
		msg    string
	}
	logCh := make(chan loge)
	log := func(stderr bool, format string, a ...interface{}) {
		logCh <- loge{stderr, fmt.Sprintf(format, a...)}
	}

	allWg.Add(1)
	go func() {
		defer allWg.Done()
		for l := range logCh {
			if l.stderr {
				fmt.Fprint(os.Stderr, l.msg)
			} else {
				fmt.Print(l.msg)
			}
		}
	}()

	// tasks

	inCh := make(chan [2]string)

	allWg.Add(1)
	go func() {
		defer allWg.Done()
		defer close(inCh)
		for _, input := range inputs {
			output, ok := pathMap[input]
			if !ok {
				atomic.AddInt64(&cur, 1)
				atomic.AddInt64(&skipped, 1)
				log(false, "[%3d/%3d] Skipping %s\n", atomic.LoadInt64(&cur), total, input)
				continue
			}
			input := input // copy
			inCh <- [2]string{input, output}
		}
	}()

	// processing

	allWg.Add(1)
	go func() {
		defer allWg.Done()
		defer close(logCh)

		var workerWg sync.WaitGroup
		defer workerWg.Wait()

		for i := 0; i < runtime.NumCPU(); i++ {
			workerWg.Add(1)
			go func() {
				defer workerWg.Done()
				for t := range inCh {
					var i int64
					for {
						if tmp := atomic.LoadInt64(&cur); atomic.CompareAndSwapInt64(&cur, tmp, tmp+1) {
							i = tmp + 1
							break
						}
					}
					input, output := t[0], t[1]
					switch {
					case !strings.HasSuffix(output, ext):
						if !*verbose {
							log(false, "[%3d/%3d] Copying %s\n", i, total, input)
						} else {
							log(false, "[%3d/%3d] Copying %s\n          => %s\n", atomic.LoadInt64(&cur), total, input, output)
						}
						if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						if err := copyFile(input, output); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						atomic.AddInt64(&copied, 1)
					default:
						if !*verbose {
							log(false, "[%3d/%3d] Converting %s\n", i, total, input)
						} else {
							log(false, "[%3d/%3d] Converting %s\n          => %s\n", atomic.LoadInt64(&cur), total, input, output)
						}
						if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						if err := func() error {
							fi, err := zip.OpenReader(input)
							if err != nil {
								return err
							}
							defer fi.Close()

							fo, err := os.CreateTemp(filepath.Dir(output), ".kepubify."+filepath.Base(output)+".*")
							if err != nil {
								return err
							}
							defer os.Remove(fo.Name())

							if err := converter.Convert(context.Background(), fo, fi); err != nil {
								return err
							}

							if err := fo.Sync(); err != nil {
								return err
							}

							if err := fo.Close(); err != nil {
								return err
							}

							if err := os.Rename(fo.Name(), output); err != nil {
								return err
							}

							return nil
						}(); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						atomic.AddInt64(&converted, 1)
					}
				}
			}()
		}
	}()

	allWg.Wait()

	fmt.Printf("\n%d total: %d converted, %d copied, %d skipped, %d errored\n", total, converted, copied, skipped, errored)

	var tmp bool
	errs.Range(func(input, err interface{}) bool {
		if !tmp {
			fmt.Fprintf(os.Stderr, "\nErrors:\n")
			tmp = true
		}
		fmt.Fprintf(os.Stderr, "  %#v\n  => %#v\n  Error: %v\n\n", input, pathMap[input.(string)], err)
		return true
	})
	if tmp {
		exit(1)
		return
	}
	exit(0)
}

func helpExit() {
	fmt.Fprintf(os.Stderr, "Usage: kepubify [options] input_path [input_path]...\n")
	fmt.Fprintf(os.Stderr, "\nVersion:\n  kepubify %s\n", version)

	categories := map[string]*pflag.FlagSet{}
	pflag.VisitAll(func(flag *pflag.Flag) {
		category := flag.Annotations["category"][0] // this will panic if the category is not set, which is intended
		if _, ok := categories[category]; !ok {
			categories[category] = pflag.NewFlagSet("tmp", pflag.ExitOnError)
			categories[category].SortFlags = false
		}
		categories[category].AddFlag(flag)
	})

	var categoriesSort []string
	for category := range categories {
		categoriesSort = append(categoriesSort, category)
	}
	sort.Strings(categoriesSort)

	for _, category := range categoriesSort {
		fmt.Fprintf(os.Stderr, "\n%s:\n%s", strings.Split(category, ".")[1], categories[category].FlagUsagesWrapped(160))
	}

	fmt.Fprintf(os.Stderr, "\nLinks:\n")
	for _, v := range [][]string{
		{"Website", "https://pgaskin.net/kepubify"},
		{"Source Code", "https://github.com/pgaskin/kepubify"},
		{"Bugs/Support", "https://github.com/pgaskin/kepubify/issues"},
		{"MobileRead", "http://mr.gd/forums/showthread.php?t=295287"},
	} {
		fmt.Fprintf(os.Stderr, "  %-12s - %s\n", v[0], v[1])
	}

	exit(0)
}

func exit(status int) {
	if runtime.GOOS == "windows" {
		time.Sleep(time.Second * 2)
	}
	os.Exit(status)
}

func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer out.Close()

	if _, err = io.Copy(out, in); err != nil {
		return err
	}
	return out.Close() // note: this runs before the deferred Close, so errors will work correctly
}
<commit_msg>cmd/kepubify: Add flag for ConverterOptionDummyTitlepage (#33)<commit_after>// Command kepubify is a fast, standalone, EPUB to KEPUB converter.
package main

import (
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pgaskin/kepubify/v4/internal/zip"
	"github.com/pgaskin/kepubify/v4/kepub"
	"github.com/spf13/pflag"
)

var version = "v3-dev"

func main() {
	pflag.CommandLine.SortFlags = false

	verbose := pflag.BoolP("verbose", "v", false, "Show extra information in output")
	sversion := pflag.Bool("version", false, "Show the version")
	help := pflag.BoolP("help", "h", false, "Show this help text")

	for _, flag := range []string{"verbose", "version", "help"} {
		pflag.CommandLine.SetAnnotation(flag, "category", []string{"1.General Options"})
	}

	update := pflag.BoolP("update", "u", false, "Don't reconvert files which have already been converted (i.e. don't overwrite output files)")
	inplace := pflag.BoolP("inplace", "i", false, "Don't add the _converted suffix to converted files and directories")
	nopreservedirs := pflag.Bool("no-preserve-dirs", false, "Flatten the directory structure of the input (an error will be shown if there are conflicts)")
	output := pflag.StringP("output", "o", "", "[>1 inputs || 1 file input with existing dir output]: Directory to place converted files/dirs under; [1 file input with nonexistent output]: Output filename; [1 dir input]: Output directory for contents of input (default: current directory)")
	calibre := pflag.Bool("calibre", false, "Use .kepub instead of .kepub.epub as the output extension (for Calibre compatibility, only use if you know what you are doing)")
	copy := pflag.StringSliceP("copy", "x", nil, "Copy files with the specified extension (with a leading period) to the output unchanged (no effect if the filename ends up the same)")

	for _, flag := range []string{"update", "inplace", "no-preserve-dirs", "output", "calibre", "copy"} {
		pflag.CommandLine.SetAnnotation(flag, "category", []string{"2.Output Options"})
	}

	smartenpunct := pflag.Bool("smarten-punctuation", false, "Smarten punctuation (smart quotes, dashes, etc) (excluding pre and code tags)")
	css := pflag.StringArrayP("css", "c", nil, "Custom CSS to add to ebook")
	hyphenate := pflag.Bool("hyphenate", false, "Force enable hyphenation")
	nohyphenate := pflag.Bool("no-hyphenate", false, "Force disable hyphenation")
	fullscreenfixes := pflag.Bool("fullscreen-reading-fixes", false, "Enable fullscreen reading bugfixes based on https://www.mobileread.com/forums/showpost.php?p=3113460&postcount=16")
	adddummytitlepage := pflag.Bool("add-dummy-titlepage", false, "Force-enables the dummy titlepage to fix layout issues with the first content file on certain books (this is enabled when needed using a heuristic if not specified)")
	noadddummytitlepage := pflag.Bool("no-add-dummy-titlepage", false, "Force-disables the dummy titlepage")
	replace := pflag.StringArrayP("replace", "r", nil, "Find and replace on all html files (repeat any number of times) (format: find|replace)")

	for _, flag := range []string{"smarten-punctuation", "css", "hyphenate", "no-hyphenate", "fullscreen-reading-fixes", "add-dummy-titlepage", "no-add-dummy-titlepage", "replace"} {
		pflag.CommandLine.SetAnnotation(flag, "category", []string{"3.Conversion Options"})
	}

	// --- Parse options --- //

	pflag.Parse()

	if *sversion {
		fmt.Printf("kepubify %s\n", version)
		exit(0)
		return
	}

	if *help || pflag.NArg() == 0 {
		helpExit()
		return
	}

	if *hyphenate && *nohyphenate {
		fmt.Printf("Error: --hyphenate and --no-hyphenate are mutally exclusive. See --help for more details.\n")
		exit(2)
		return
	}

	for _, c := range *copy {
		if len(c) == 0 || c[0] != '.' {
			fmt.Printf("Error: --copy argument %#v doesn't have a leading period. See --help for more details.\n", c)
			exit(2)
			return
		}
	}

	// --- Make converter --- //

	var opts []kepub.ConverterOption
	for _, v := range *css {
		opts = append(opts, kepub.ConverterOptionAddCSS(v))
	}
	if *hyphenate {
		opts = append(opts, kepub.ConverterOptionHyphenate(true))
	} else if *nohyphenate {
		opts = append(opts, kepub.ConverterOptionHyphenate(false))
	}
	if *smartenpunct {
		opts = append(opts, kepub.ConverterOptionSmartypants())
	}
	if *fullscreenfixes {
		opts = append(opts, kepub.ConverterOptionFullScreenFixes())
	}
	if *adddummytitlepage {
		opts = append(opts, kepub.ConverterOptionDummyTitlepage(true))
	} else if *noadddummytitlepage {
		opts = append(opts, kepub.ConverterOptionDummyTitlepage(false))
	}
	for _, r := range *replace {
		spl := strings.SplitN(r, "|", 2)
		if len(spl) != 2 {
			fmt.Fprintf(os.Stderr, "Error: Parse replacement %#v: must be in format `find|replace`\n", r)
			exit(1)
			return
		}
		opts = append(opts, kepub.ConverterOptionFindReplace(spl[0], spl[1]))
	}
	converter := kepub.NewConverterWithOptions(opts...)

	// --- Transform paths --- //

	ext := ".kepub.epub"
	if *calibre {
		ext = ".kepub"
	}

	pathMap, skipList, err := transformer{
		NoPreserveDirs:   *nopreservedirs,
		Update:           *update,
		Inplace:          *inplace,
		Suffixes:         []string{".epub"},
		ExcludeSuffixes:  []string{".kepub.epub"},
		PreserveSuffixes: *copy,
		TargetSuffix:     ext,
	}.TransformPaths(*output, pflag.Args()...)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		exit(1)
		return
	}

	// --- Convert --- //

	fmt.Printf("Kepubify %s\n", version)
	if *calibre {
		fmt.Printf("Using extension %s for Calibre compatibility (this is meant for use with a Calibre library and will not work directly on a Kobo reader)\n", ext)
	}
	fmt.Printf("\n")

	var inputs []string
	for key := range pathMap {
		inputs = append(inputs, key)
	}
	inputs = append(inputs, skipList...)
	sort.Strings(inputs)

	// --- Conversion Pipeline --- //
	var cur int64                                 // progress
	var converted, copied, skipped, errored int64 // counters (sum == total)
	var errs sync.Map                             // input -> error
	total := int64(len(pathMap) + len(skipList))  // immutable

	var allWg sync.WaitGroup

	// logging

	type loge struct {
		stderr bool
		msg    string
	}
	logCh := make(chan loge)
	log := func(stderr bool, format string, a ...interface{}) {
		logCh <- loge{stderr, fmt.Sprintf(format, a...)}
	}

	allWg.Add(1)
	go func() {
		defer allWg.Done()
		for l := range logCh {
			if l.stderr {
				fmt.Fprint(os.Stderr, l.msg)
			} else {
				fmt.Print(l.msg)
			}
		}
	}()

	// tasks

	inCh := make(chan [2]string)

	allWg.Add(1)
	go func() {
		defer allWg.Done()
		defer close(inCh)
		for _, input := range inputs {
			output, ok := pathMap[input]
			if !ok {
				atomic.AddInt64(&cur, 1)
				atomic.AddInt64(&skipped, 1)
				log(false, "[%3d/%3d] Skipping %s\n", atomic.LoadInt64(&cur), total, input)
				continue
			}
			input := input // copy
			inCh <- [2]string{input, output}
		}
	}()

	// processing

	allWg.Add(1)
	go func() {
		defer allWg.Done()
		defer close(logCh)

		var workerWg sync.WaitGroup
		defer workerWg.Wait()

		for i := 0; i < runtime.NumCPU(); i++ {
			workerWg.Add(1)
			go func() {
				defer workerWg.Done()
				for t := range inCh {
					var i int64
					for {
						if tmp := atomic.LoadInt64(&cur); atomic.CompareAndSwapInt64(&cur, tmp, tmp+1) {
							i = tmp + 1
							break
						}
					}
					input, output := t[0], t[1]
					switch {
					case !strings.HasSuffix(output, ext):
						if !*verbose {
							log(false, "[%3d/%3d] Copying %s\n", i, total, input)
						} else {
							log(false, "[%3d/%3d] Copying %s\n          => %s\n", atomic.LoadInt64(&cur), total, input, output)
						}
						if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						if err := copyFile(input, output); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						atomic.AddInt64(&copied, 1)
					default:
						if !*verbose {
							log(false, "[%3d/%3d] Converting %s\n", i, total, input)
						} else {
							log(false, "[%3d/%3d] Converting %s\n          => %s\n", atomic.LoadInt64(&cur), total, input, output)
						}
						if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						if err := func() error {
							fi, err := zip.OpenReader(input)
							if err != nil {
								return err
							}
							defer fi.Close()

							fo, err := os.CreateTemp(filepath.Dir(output), ".kepubify."+filepath.Base(output)+".*")
							if err != nil {
								return err
							}
							defer os.Remove(fo.Name())

							if err := converter.Convert(context.Background(), fo, fi); err != nil {
								return err
							}

							if err := fo.Sync(); err != nil {
								return err
							}

							if err := fo.Close(); err != nil {
								return err
							}

							if err := os.Rename(fo.Name(), output); err != nil {
								return err
							}

							return nil
						}(); err != nil {
							errs.Store(input, err)
							atomic.AddInt64(&errored, 1)
							log(true, "          Error (%d): %v\n", i, err)
							continue
						}
						atomic.AddInt64(&converted, 1)
					}
				}
			}()
		}
	}()

	allWg.Wait()

	fmt.Printf("\n%d total: %d converted, %d copied, %d skipped, %d errored\n", total, converted, copied, skipped, errored)

	var tmp bool
	errs.Range(func(input, err interface{}) bool {
		if !tmp {
			fmt.Fprintf(os.Stderr, "\nErrors:\n")
			tmp = true
		}
		fmt.Fprintf(os.Stderr, "  %#v\n  => %#v\n  Error: %v\n\n", input, pathMap[input.(string)], err)
		return true
	})
	if tmp {
		exit(1)
		return
	}
	exit(0)
}

func helpExit() {
	fmt.Fprintf(os.Stderr, "Usage: kepubify [options] input_path [input_path]...\n")
	fmt.Fprintf(os.Stderr, "\nVersion:\n  kepubify %s\n", version)

	categories := map[string]*pflag.FlagSet{}
	pflag.VisitAll(func(flag *pflag.Flag) {
		category := flag.Annotations["category"][0] // this will panic if the category is not set, which is intended
		if _, ok := categories[category]; !ok {
			categories[category] = pflag.NewFlagSet("tmp", pflag.ExitOnError)
			categories[category].SortFlags = false
		}
		categories[category].AddFlag(flag)
	})

	var categoriesSort []string
	for category := range categories {
		categoriesSort = append(categoriesSort, category)
	}
	sort.Strings(categoriesSort)

	for _, category := range categoriesSort {
		fmt.Fprintf(os.Stderr, "\n%s:\n%s", strings.Split(category, ".")[1], categories[category].FlagUsagesWrapped(160))
	}

	fmt.Fprintf(os.Stderr, "\nLinks:\n")
	for _, v := range [][]string{
		{"Website", "https://pgaskin.net/kepubify"},
		{"Source Code", "https://github.com/pgaskin/kepubify"},
		{"Bugs/Support", "https://github.com/pgaskin/kepubify/issues"},
		{"MobileRead", "http://mr.gd/forums/showthread.php?t=295287"},
	} {
		fmt.Fprintf(os.Stderr, "  %-12s - %s\n", v[0], v[1])
	}

	exit(0)
}

func exit(status int) {
	if runtime.GOOS == "windows" {
		time.Sleep(time.Second * 2)
	}
	os.Exit(status)
}

func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer out.Close()

	if _, err = io.Copy(out, in); err != nil {
		return err
	}
	return out.Close() // note: this runs before the deferred Close, so errors will work correctly
}
<|endoftext|>


<commit_before>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package commands

import (
	"fmt"
	"regexp"
	"strings"
	"sync"

	"github.com/morikuni/aec"
	"github.com/openfaas/faas-cli/builder"
	"github.com/openfaas/faas-cli/stack"
	"github.com/spf13/cobra"
)

func init() {
	faasCmd.AddCommand(pushCmd)

	pushCmd.Flags().IntVar(&parallel, "parallel", 1, "Push images in parallel to depth specified.")
}

// pushCmd handles pushing function container images to a remote repo
var pushCmd = &cobra.Command{
	Use:   `push -f YAML_FILE [--regex "REGEX"] [--filter "WILDCARD"] [--parallel]`,
	Short: "Push OpenFaaS functions to remote registry (Docker Hub)",
	Long: `Pushes the OpenFaaS function container image(s) defined in the supplied YAML
config to a remote repository.

These container images must already be present in your local image cache.`,

	Example: `  faas-cli push -f https://domain/path/myfunctions.yml
  faas-cli push -f ./stack.yml
  faas-cli push -f ./stack.yml --parallel 4
  faas-cli push -f ./stack.yml --filter "*gif*"
  faas-cli push -f ./stack.yml --regex "fn[0-9]_.*"`,
	RunE: runPush,
}

func runPush(cmd *cobra.Command, args []string) error {

	var services stack.Services
	if len(yamlFile) > 0 {
		parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter)
		if err != nil {
			return err
		}

		if parsedServices != nil {
			services = *parsedServices
		}
	}

	if len(services.Functions) > 0 {
		invalidImages := validateImages(&services)
		if len(invalidImages) > 0 {
			return fmt.Errorf("Unable to push one or more of your functions to Docker Hub\n" + invalidImages + "\n\nYou must provide a username or registry prefix to the Function's image such as user1/function1.\nIf you need a Docker Hub account, you can sign up here: https://hub.docker.com")
		}

		pushStack(&services, parallel)
	} else {
		return fmt.Errorf("you must supply a valid YAML file")
	}
	return nil
}

func pushImage(image string) {
	builder.ExecCommand("./", []string{"docker", "push", image})
}

func pushStack(services *stack.Services, queueDepth int) {
	wg := sync.WaitGroup{}

	workChannel := make(chan stack.Function)

	for i := 0; i < queueDepth; i++ {
		go func(index int) {
			wg.Add(1)
			for function := range workChannel {
				fmt.Printf(aec.YellowF.Apply("[%d] > Pushing %s.\n"), index, function.Name)
				if len(function.Image) == 0 {
					fmt.Println("Please provide a valid Image value in the YAML file.")
				} else if function.SkipBuild {
					fmt.Printf("Skipping %s\n", function.Name)
				} else {
					pushImage(function.Image)
					fmt.Printf(aec.YellowF.Apply("[%d] < Pushing %s done.\n"), index, function.Name)
				}
			}

			fmt.Printf(aec.YellowF.Apply("[%d] worker done.\n"), index)
			wg.Done()
		}(i)
	}

	for k, function := range services.Functions {
		function.Name = k
		workChannel <- function
	}

	close(workChannel)

	wg.Wait()

}

func validateImages(services *stack.Services) string {
	errMsg := make([]string, 0)
	for name, function := range services.Functions {
		if function.SkipBuild {
			continue
		}
		ip := "(\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\b)(:\\d+)?"
		re := regexp.MustCompile("([a-z].+)\\/|" + ip + "\\/")
		ma := re.FindAllString(function.Image, 1)

		if len(ma) < 1 {
			errMsg = append(errMsg, fmt.Sprintf(" - %s", name))
		}
	}
	return strings.Join(errMsg, "\n")
}
<commit_msg>updated push error to use string literal. updated error message to be more succinct<commit_after>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package commands

import (
	"fmt"
	"strings"
	"sync"

	"github.com/morikuni/aec"
	"github.com/openfaas/faas-cli/builder"
	"github.com/openfaas/faas-cli/stack"
	"github.com/spf13/cobra"
)

func init() {
	faasCmd.AddCommand(pushCmd)

	pushCmd.Flags().IntVar(&parallel, "parallel", 1, "Push images in parallel to depth specified.")
}

// pushCmd handles pushing function container images to a remote repo
var pushCmd = &cobra.Command{
	Use:   `push -f YAML_FILE [--regex "REGEX"] [--filter "WILDCARD"] [--parallel]`,
	Short: "Push OpenFaaS functions to remote registry (Docker Hub)",
	Long: `Pushes the OpenFaaS function container image(s) defined in the supplied YAML
config to a remote repository.

These container images must already be present in your local image cache.`,

	Example: `  faas-cli push -f https://domain/path/myfunctions.yml
  faas-cli push -f ./stack.yml
  faas-cli push -f ./stack.yml --parallel 4
  faas-cli push -f ./stack.yml --filter "*gif*"
  faas-cli push -f ./stack.yml --regex "fn[0-9]_.*"`,
	RunE: runPush,
}

func runPush(cmd *cobra.Command, args []string) error {

	var services stack.Services
	if len(yamlFile) > 0 {
		parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter)
		if err != nil {
			return err
		}

		if parsedServices != nil {
			services = *parsedServices
		}
	}

	if len(services.Functions) > 0 {
		invalidImages := validateImages(&services)
		if len(invalidImages) > 0 {
			return fmt.Errorf(`
Unable to push one or more of your functions to Docker Hub
` + invalidImages + `

You must provide a username or registry prefix to the Function's image such as user1/function1`)
		}

		pushStack(&services, parallel)
	} else {
		return fmt.Errorf("you must supply a valid YAML file")
	}
	return nil
}

func pushImage(image string) {
	builder.ExecCommand("./", []string{"docker", "push", image})
}

func pushStack(services *stack.Services, queueDepth int) {
	wg := sync.WaitGroup{}

	workChannel := make(chan stack.Function)

	for i := 0; i < queueDepth; i++ {
		go func(index int) {
			wg.Add(1)
			for function := range workChannel {
				fmt.Printf(aec.YellowF.Apply("[%d] > Pushing %s.\n"), index, function.Name)
				if len(function.Image) == 0 {
					fmt.Println("Please provide a valid Image value in the YAML file.")
				} else if function.SkipBuild {
					fmt.Printf("Skipping %s\n", function.Name)
				} else {
					pushImage(function.Image)
					fmt.Printf(aec.YellowF.Apply("[%d] < Pushing %s done.\n"), index, function.Name)
				}
			}

			fmt.Printf(aec.YellowF.Apply("[%d] worker done.\n"), index)
			wg.Done()
		}(i)
	}

	for k, function := range services.Functions {
		function.Name = k
		workChannel <- function
	}

	close(workChannel)

	wg.Wait()

}

func validateImages(services *stack.Services) string {
	errMsg := make([]string, 0)
	for name, function := range services.Functions {
		if function.SkipBuild {
			continue
		}

		if !strings.Contains(function.Image, `/`) {
			errMsg = append(errMsg, fmt.Sprintf(" - %s", name))
		}
	}
	return strings.Join(errMsg, "\n")
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Huawei Technologies Co., Ltd. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
This module implements a entry into the OpenSDS REST service.

*/

package main

import (
	"fmt"

	"github.com/opensds/opensds/pkg/db"
	"github.com/opensds/opensds/pkg/dock"
	"github.com/opensds/opensds/pkg/dock/server"
	. "github.com/opensds/opensds/pkg/utils/config"
	"github.com/opensds/opensds/pkg/utils/daemon"
	"github.com/opensds/opensds/pkg/utils/logs"
)

func init() {
	def := GetDefaultConfig()
	flag := CONF.Flag
	flag.StringVar(&CONF.OsdsDock.ApiEndpoint, "api-endpoint", def.OsdsDock.ApiEndpoint, "Listen endpoint of controller service")
	flag.StringVar(&CONF.Database.Endpoint, "db-endpoint", def.Database.Endpoint, "Connection endpoint of database service")
	flag.StringVar(&CONF.Database.Driver, "db-driver", def.Database.Driver, "Driver name of database service")
	// flag.StringVar(&CONF.Database.Credential, "db-credential", def.Database.Credential, "Connection credential of database service")
	daemon.SetDaemonFlag(&CONF.OsdsDock.Daemon, def.OsdsDock.Daemon)
	CONF.Load("/etc/opensds/opensds.conf")
	daemon.CheckAndRunDaemon(CONF.OsdsDock.Daemon)
}

func main() {
	// Open OpenSDS dock service log file.
	logs.InitLogs()
	defer logs.FlushLogs()

	// Set up database session.
	db.Init(&CONF.Database)

	// Automatically discover dock and pool resources from backends.
	dock.Brain = dock.NewDockHub()
	switch CONF.OsdsDock.DockType {
	case "provisioner":
		if err := dock.Brain.TriggerDiscovery(); err != nil {
			panic(err)
		}
		break
	case "attacher":
		if err := dock.Brain.RegisterDock(); err != nil {
			panic(err)
		}
		break
	default:
		panic(fmt.Errorf("Dock type (%s) is not supportes!", CONF.OsdsDock.DockType))
	}

	// Construct dock module grpc server struct and start the listen mechanism
	// of dock module.
	ds := server.NewDockServer(CONF.OsdsDock.ApiEndpoint)
	if err := ds.Run(); err != nil {
		panic(err)
	}
}
<commit_msg>Fix a spelling error<commit_after>// Copyright (c) 2017 Huawei Technologies Co., Ltd. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
This module implements a entry into the OpenSDS REST service.

*/

package main

import (
	"fmt"

	"github.com/opensds/opensds/pkg/db"
	"github.com/opensds/opensds/pkg/dock"
	"github.com/opensds/opensds/pkg/dock/server"
	. "github.com/opensds/opensds/pkg/utils/config"
	"github.com/opensds/opensds/pkg/utils/daemon"
	"github.com/opensds/opensds/pkg/utils/logs"
)

func init() {
	def := GetDefaultConfig()
	flag := CONF.Flag
	flag.StringVar(&CONF.OsdsDock.ApiEndpoint, "api-endpoint", def.OsdsDock.ApiEndpoint, "Listen endpoint of controller service")
	flag.StringVar(&CONF.Database.Endpoint, "db-endpoint", def.Database.Endpoint, "Connection endpoint of database service")
	flag.StringVar(&CONF.Database.Driver, "db-driver", def.Database.Driver, "Driver name of database service")
	// flag.StringVar(&CONF.Database.Credential, "db-credential", def.Database.Credential, "Connection credential of database service")
	daemon.SetDaemonFlag(&CONF.OsdsDock.Daemon, def.OsdsDock.Daemon)
	CONF.Load("/etc/opensds/opensds.conf")
	daemon.CheckAndRunDaemon(CONF.OsdsDock.Daemon)
}

func main() {
	// Open OpenSDS dock service log file.
	logs.InitLogs()
	defer logs.FlushLogs()

	// Set up database session.
	db.Init(&CONF.Database)

	// Automatically discover dock and pool resources from backends.
	dock.Brain = dock.NewDockHub()
	switch CONF.OsdsDock.DockType {
	case "provisioner":
		if err := dock.Brain.TriggerDiscovery(); err != nil {
			panic(err)
		}
		break
	case "attacher":
		if err := dock.Brain.RegisterDock(); err != nil {
			panic(err)
		}
		break
	default:
		panic(fmt.Errorf("Dock type (%s) is not supported!", CONF.OsdsDock.DockType))
	}

	// Construct dock module grpc server struct and start the listen mechanism
	// of dock module.
	ds := server.NewDockServer(CONF.OsdsDock.ApiEndpoint)
	if err := ds.Run(); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command pushback is a service that monitors a set of GitHub repositories
// for incoming Pull Requests, replies with contribution instructions, and
// closes the request. This is for projects that don't use Pull Requests.
package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"

	"cloud.google.com/go/compute/metadata"
)

const (
	authMetadataKey   = "pushback-credentials"
	secretMetadataKey = "pushback-webhook-secret"
	pollInterval      = 30 * time.Minute
)

var repos = []string{
	"golang/arch",
	"golang/benchmarks",
	"golang/blog",
	"golang/build",
	"golang/crypto",
	"golang/debug",
	"golang/example",
	"golang/exp",
	"golang/gddo",
	"golang/go",
	"golang/gofrontend",
	"golang/image",
	"golang/mobile",
	"golang/net",
	"golang/oauth2",
	"golang/playground",
	"golang/perf",
	"golang/proposal",
	"golang/review",
	"golang/sublime-build",
	"golang/sublime-config",
	"golang/sync",
	"golang/talks",
	"golang/term",
	"golang/text",
	"golang/time",
	"golang/tools",
	"golang/tour",
}

func main() {
	go poll()
	http.HandleFunc("/webhook", webhook)
	http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "OK")
	})
	log.Fatal(http.ListenAndServe(":8080", nil))
}

const debug = false

func logErr(w io.Writer, desc string, err error) {
	log.Println(desc, err)
	if debug {
		fmt.Fprintln(w, desc, err)
	}
}

func poll() {
	ticker := time.NewTicker(pollInterval)
	for {
		for _, repo := range repos {
			if err := pollRepo(repo); err != nil {
				log.Printf("polling repo %v: %v", repo, err)
			}
		}
		<-ticker.C
	}
}

func pollRepo(repo string) error {
	v := url.Values{"q": {fmt.Sprintf("type:pr state:open repo:%v", repo)}}
	u := fmt.Sprintf("https://api.github.com/search/issues?%v", v.Encode())
	body, err := doRequest("GET", u, nil)
	if err != nil {
		return err
	}
	var results struct {
		Items []struct {
			Number int
		}
	}
	if err := json.Unmarshal(body, &results); err != nil {
		return err
	}
	for _, r := range results.Items {
		if repo == "golang/go" && r.Number == 9220 {
			// This is a placeholder issue to remind people
			// that we don't use pull requests; don't close it.
			continue
		}
		if err := closePR(repo, r.Number); err != nil {
			log.Printf("closing pr %v#%v: %v", repo, r.Number, err)
		}
	}
	return nil
}

func webhook(w http.ResponseWriter, r *http.Request) {
	if r.Header.Get("X-Github-Event") != "pull_request" {
		// Only handle pull request notifications.
		return
	}
	body, err := validate(r)
	if err != nil {
		http.Error(w, "Forbidden", http.StatusForbidden)
		logErr(w, "Error validating request:", err)
		return
	}
	var req struct {
		Action     string
		Number     int
		Repository struct {
			Full_Name string
		}
	}
	if err := json.Unmarshal(body, &req); err != nil {
		http.Error(w, "Bad request", http.StatusBadRequest)
		logErr(w, "Error decoding request:", err)
		return
	}
	if req.Action != "opened" {
		// Only handle "opened" actions.
		return
	}
	if err := closePR(req.Repository.Full_Name, req.Number); err != nil {
		http.Error(w, "Server error", http.StatusInternalServerError)
		logErr(w, "Error closing PR:", err)
		return
	}
	fmt.Fprintln(w, "OK")
}

// validate compares the signature in the request header with the body.
func validate(r *http.Request) (body []byte, err error) {
	// Decode signature header.
	sigHeader := r.Header.Get("X-Hub-Signature")
	sigParts := strings.SplitN(sigHeader, "=", 2)
	if len(sigParts) != 2 {
		return nil, fmt.Errorf("Bad signature header: %q", sigHeader)
	}
	var h func() hash.Hash
	switch alg := sigParts[0]; alg {
	case "sha1":
		h = sha1.New
	case "sha256":
		h = sha256.New
	default:
		return nil, fmt.Errorf("Unsupported hash algorithm: %q", alg)
	}
	gotSig, err := hex.DecodeString(sigParts[1])
	if err != nil {
		return nil, err
	}

	// Compute expected signature.
	key, err := metadata.ProjectAttributeValue(secretMetadataKey)
	if err != nil {
		return nil, err
	}
	body, err = ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	mac := hmac.New(h, []byte(key))
	mac.Write(body)
	expectSig := mac.Sum(nil)

	if !hmac.Equal(gotSig, expectSig) {
		return nil, fmt.Errorf("Invalid signature %X, want %x", gotSig, expectSig)
	}
	return body, nil
}

// closePR posts a helpful message before closing the specified pull request.
func closePR(repo string, id int) error {
	// Post the comment.
	url := fmt.Sprintf("https://api.github.com/repos/%v/issues/%v/comments", repo, id)
	if _, err := doRequest("POST", url, bytes.NewReader(messageJSON)); err != nil {
		return fmt.Errorf("POST to %v: %v", url, err)
	}

	// Close the issue.
	url = fmt.Sprintf("https://api.github.com/repos/%v/pulls/%v", repo, id)
	if _, err := doRequest("PATCH", url, strings.NewReader(`{"state":"closed"}`)); err != nil {
		return fmt.Errorf("PATCH to %v: %v", url, err)
	}

	return nil
}

// doRequest makes an authenticated request to the GitHub API.
func doRequest(method, url string, body io.Reader) ([]byte, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	// These values are cached, so we can fetch them every time.
	userpass, err := metadata.ProjectAttributeValue(authMetadataKey)
	if err != nil {
		return nil, err
	}
	p := strings.SplitN(userpass, ":", 2)
	if len(p) != 2 {
		return nil, errors.New("bad authentication data")
	}
	req.SetBasicAuth(p[0], p[1])

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	respBody, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode/100 != 2 {
		return respBody, fmt.Errorf("Bad response: %v\nBody:\n%s", resp.Status, respBody)
	}
	return respBody, nil

}

const message = `
Hi! Thanks for the PR!

Unfortunately, the Go project doesn't use GitHub's Pull Requests,
so we can't accept your contribution this way.
We instead use a code review system called Gerrit.

The good news is, I'm here to help.

From here, you have two options:

1. Read our [Contribution Guidelines](https://golang.org/doc/contribute.html) to learn how to send a change with Gerrit.
2. Or, [create an issue](https://golang.org/issue/new) about the issue this PR addresses, so that someone else can fix it.

I'm going to close this Pull Request now.
Please don't be offended! :-)

Thanks again,

GopherBot (on behalf of the Go Team)
`

var messageJSON []byte

func init() {
	var err error
	messageJSON, err = json.Marshal(struct {
		Body string `json:"body"`
	}{message})
	if err != nil {
		panic(err)
	}
}
<commit_msg>cmd/pushback: keep oauth2 placeholder pull request open<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command pushback is a service that monitors a set of GitHub repositories
// for incoming Pull Requests, replies with contribution instructions, and
// closes the request. This is for projects that don't use Pull Requests.
package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"

	"cloud.google.com/go/compute/metadata"
)

const (
	authMetadataKey   = "pushback-credentials"
	secretMetadataKey = "pushback-webhook-secret"
	pollInterval      = 30 * time.Minute
)

var repos = []string{
	"golang/arch",
	"golang/benchmarks",
	"golang/blog",
	"golang/build",
	"golang/crypto",
	"golang/debug",
	"golang/example",
	"golang/exp",
	"golang/gddo",
	"golang/go",
	"golang/gofrontend",
	"golang/image",
	"golang/mobile",
	"golang/net",
	"golang/oauth2",
	"golang/playground",
	"golang/perf",
	"golang/proposal",
	"golang/review",
	"golang/sublime-build",
	"golang/sublime-config",
	"golang/sync",
	"golang/talks",
	"golang/term",
	"golang/text",
	"golang/time",
	"golang/tools",
	"golang/tour",
}

// keepOpen represents PRs that shouldn't be automatically closed
// by pushback. These PRs are dummy PRs telling users that Only
// Gerrit CLs are accepted to increase the awareness that the Go
// projects cannot accept PRs.
var keepOpen = []struct {
	repo string
	id   int
}{
	{"golang/go", 9920},
	{"golang/oauth2", 217},
}

func main() {
	go poll()
	http.HandleFunc("/webhook", webhook)
	http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "OK")
	})
	log.Fatal(http.ListenAndServe(":8080", nil))
}

const debug = false

func logErr(w io.Writer, desc string, err error) {
	log.Println(desc, err)
	if debug {
		fmt.Fprintln(w, desc, err)
	}
}

func poll() {
	ticker := time.NewTicker(pollInterval)
	for {
		for _, repo := range repos {
			if err := pollRepo(repo); err != nil {
				log.Printf("polling repo %v: %v", repo, err)
			}
		}
		<-ticker.C
	}
}

func pollRepo(repo string) error {
	v := url.Values{"q": {fmt.Sprintf("type:pr state:open repo:%v", repo)}}
	u := fmt.Sprintf("https://api.github.com/search/issues?%v", v.Encode())
	body, err := doRequest("GET", u, nil)
	if err != nil {
		return err
	}
	var results struct {
		Items []struct {
			Number int
		}
	}
	if err := json.Unmarshal(body, &results); err != nil {
		return err
	}

	for _, r := range results.Items {
		if err := closePR(repo, r.Number); err != nil {
			log.Printf("closing pr %v#%v: %v", repo, r.Number, err)
		}
	}
	return nil
}

func webhook(w http.ResponseWriter, r *http.Request) {
	if r.Header.Get("X-Github-Event") != "pull_request" {
		// Only handle pull request notifications.
		return
	}
	body, err := validate(r)
	if err != nil {
		http.Error(w, "Forbidden", http.StatusForbidden)
		logErr(w, "Error validating request:", err)
		return
	}
	var req struct {
		Action     string
		Number     int
		Repository struct {
			Full_Name string
		}
	}
	if err := json.Unmarshal(body, &req); err != nil {
		http.Error(w, "Bad request", http.StatusBadRequest)
		logErr(w, "Error decoding request:", err)
		return
	}
	if req.Action != "opened" {
		// Only handle "opened" actions.
		return
	}
	if err := closePR(req.Repository.Full_Name, req.Number); err != nil {
		http.Error(w, "Server error", http.StatusInternalServerError)
		logErr(w, "Error closing PR:", err)
		return
	}
	fmt.Fprintln(w, "OK")
}

// validate compares the signature in the request header with the body.
func validate(r *http.Request) (body []byte, err error) {
	// Decode signature header.
	sigHeader := r.Header.Get("X-Hub-Signature")
	sigParts := strings.SplitN(sigHeader, "=", 2)
	if len(sigParts) != 2 {
		return nil, fmt.Errorf("Bad signature header: %q", sigHeader)
	}
	var h func() hash.Hash
	switch alg := sigParts[0]; alg {
	case "sha1":
		h = sha1.New
	case "sha256":
		h = sha256.New
	default:
		return nil, fmt.Errorf("Unsupported hash algorithm: %q", alg)
	}
	gotSig, err := hex.DecodeString(sigParts[1])
	if err != nil {
		return nil, err
	}

	// Compute expected signature.
	key, err := metadata.ProjectAttributeValue(secretMetadataKey)
	if err != nil {
		return nil, err
	}
	body, err = ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	mac := hmac.New(h, []byte(key))
	mac.Write(body)
	expectSig := mac.Sum(nil)

	if !hmac.Equal(gotSig, expectSig) {
		return nil, fmt.Errorf("Invalid signature %X, want %x", gotSig, expectSig)
	}
	return body, nil
}

// closePR posts a helpful message before closing the specified pull request.
// Placeholder PRs are skipped.
func closePR(repo string, id int) error {
	for _, ko := range keepOpen {
		if repo == ko.repo && id == ko.id {
			// This is a placeholder issue to remind people
			// that we don't use pull requests; don't close it.
			return nil
		}
	}

	// Post the comment.
	url := fmt.Sprintf("https://api.github.com/repos/%v/issues/%v/comments", repo, id)
	if _, err := doRequest("POST", url, bytes.NewReader(messageJSON)); err != nil {
		return fmt.Errorf("POST to %v: %v", url, err)
	}

	// Close the issue.
	url = fmt.Sprintf("https://api.github.com/repos/%v/pulls/%v", repo, id)
	if _, err := doRequest("PATCH", url, strings.NewReader(`{"state":"closed"}`)); err != nil {
		return fmt.Errorf("PATCH to %v: %v", url, err)
	}

	return nil
}

// doRequest makes an authenticated request to the GitHub API.
func doRequest(method, url string, body io.Reader) ([]byte, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	// These values are cached, so we can fetch them every time.
	userpass, err := metadata.ProjectAttributeValue(authMetadataKey)
	if err != nil {
		return nil, err
	}
	p := strings.SplitN(userpass, ":", 2)
	if len(p) != 2 {
		return nil, errors.New("bad authentication data")
	}
	req.SetBasicAuth(p[0], p[1])

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	respBody, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode/100 != 2 {
		return respBody, fmt.Errorf("Bad response: %v\nBody:\n%s", resp.Status, respBody)
	}
	return respBody, nil

}

const message = `
Hi! Thanks for the PR!

Unfortunately, the Go project doesn't use GitHub's Pull Requests,
so we can't accept your contribution this way.
We instead use a code review system called Gerrit.

The good news is, I'm here to help.

From here, you have two options:

1. Read our [Contribution Guidelines](https://golang.org/doc/contribute.html) to learn how to send a change with Gerrit.
2. Or, [create an issue](https://golang.org/issue/new) about the issue this PR addresses, so that someone else can fix it.

I'm going to close this Pull Request now.
Please don't be offended! :-)

Thanks again,

GopherBot (on behalf of the Go Team)
`

var messageJSON []byte

func init() {
	var err error
	messageJSON, err = json.Marshal(struct {
		Body string `json:"body"`
	}{message})
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"github.com/AlexanderThaller/logger"
)

func Test_RunNoDataPath(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "NoDataPath")

	command := new(Command)
	err := command.Run()

	got := err.Error()
	expected := "the datapath can not be empty"

	testerr_output(t, l, err, got, expected)
}

func Test_RunDates(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Dates")

	action := ActionDates

	expected := `2014-10-31
2014-11-02
`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunList(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "List")

	action := ActionList
	expected := `TestNotes
TestTodos
TestTracks
`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunNotes(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Notes")
	l.SetLevel(logger.Info)

	action := ActionNotes
	expected := "= Lablog -- Notes\n"
	expected += AsciiDocSettings + "\n\n"
	expected += `== TestNotes

=== 2014-10-31T21:36:31.49146148+01:00
Test1

=== 2014-10-31T21:36:33.49871531+01:00
Test2

=== 2014-10-31T21:36:35.138412374+01:00
Test3

=== 2014-10-31T21:36:36.810478305+01:00
Test4

=== 2014-10-31T21:36:38.450479686+01:00
Test5

`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunTodos(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Todos")
	l.SetLevel(logger.Info)

	action := ActionTodos
	expected := "= Lablog -- Todos\n"
	expected += AsciiDocSettings + "\n\n"
	expected += `== TestTodos

* Test1
* Test2
* Test3
* Test4
* Test5
* Test7

`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunTracks(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Tracks")
	l.SetLevel(logger.Info)

	action := ActionTracks
	expected := "= Lablog -- Tracks\n"
	expected += AsciiDocSettings + "\n\n"
	expected += `== TestTracks

* 2014-11-02T00:46:27.250010094+01:00 - Test1
* 2014-11-02T00:46:31.186052306+01:00 - Test2
* 2014-11-02T00:46:32.794131714+01:00 - Test3
* 2014-11-02T00:46:34.322047221+01:00 - Test4
* 2014-11-02T00:46:35.658221386+01:00 - Test5
* 2014-11-02T00:46:38.385921461+01:00
* 2014-11-02T00:46:51.833861322+01:00 - Test6
* 2014-11-02T00:46:53.713763133+01:00 - Test6
* 2014-11-02T00:46:55.609688812+01:00 - Test7
* 2014-11-02T00:46:56.713668447+01:00 - Test7
* 2014-11-02T00:46:57.953493565+01:00 - Test7

`

	testCommandRunOutput(t, l, action, expected)
}
<commit_msg>added test for project listing.<commit_after>package main

import (
	"testing"

	"github.com/AlexanderThaller/logger"
)

func Test_RunNoDataPath(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "NoDataPath")

	command := new(Command)
	err := command.Run()

	got := err.Error()
	expected := "the datapath can not be empty"

	testerr_output(t, l, err, got, expected)
}

func Test_RunDates(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Dates")

	action := ActionDates

	expected := `2014-10-31
2014-11-02
`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunList(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "List")

	action := ActionList
	expected := `TestNotes
TestTodos
TestTracks
`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunNotes(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Notes")
	l.SetLevel(logger.Info)

	action := ActionNotes
	expected := "= Lablog -- Notes\n"
	expected += AsciiDocSettings + "\n\n"
	expected += `== TestNotes

=== 2014-10-31T21:36:31.49146148+01:00
Test1

=== 2014-10-31T21:36:33.49871531+01:00
Test2

=== 2014-10-31T21:36:35.138412374+01:00
Test3

=== 2014-10-31T21:36:36.810478305+01:00
Test4

=== 2014-10-31T21:36:38.450479686+01:00
Test5

`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunProjects(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Projects")

	action := ActionProjects
	expected := `TestNotes
TestTodos
TestTracks
`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunTodos(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Todos")
	l.SetLevel(logger.Info)

	action := ActionTodos
	expected := "= Lablog -- Todos\n"
	expected += AsciiDocSettings + "\n\n"
	expected += `== TestTodos

* Test1
* Test2
* Test3
* Test4
* Test5
* Test7

`

	testCommandRunOutput(t, l, action, expected)
}

func Test_RunTracks(t *testing.T) {
	l := logger.New(Name, "Test", "Command", "Run", "Tracks")
	l.SetLevel(logger.Info)

	action := ActionTracks
	expected := "= Lablog -- Tracks\n"
	expected += AsciiDocSettings + "\n\n"
	expected += `== TestTracks

* 2014-11-02T00:46:27.250010094+01:00 - Test1
* 2014-11-02T00:46:31.186052306+01:00 - Test2
* 2014-11-02T00:46:32.794131714+01:00 - Test3
* 2014-11-02T00:46:34.322047221+01:00 - Test4
* 2014-11-02T00:46:35.658221386+01:00 - Test5
* 2014-11-02T00:46:38.385921461+01:00
* 2014-11-02T00:46:51.833861322+01:00 - Test6
* 2014-11-02T00:46:53.713763133+01:00 - Test6
* 2014-11-02T00:46:55.609688812+01:00 - Test7
* 2014-11-02T00:46:56.713668447+01:00 - Test7
* 2014-11-02T00:46:57.953493565+01:00 - Test7

`

	testCommandRunOutput(t, l, action, expected)
}
<|endoftext|>


<commit_before>package docker

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"
)

func closeWrap(args ...io.Closer) error {
	e := false
	ret := fmt.Errorf("Error closing elements")
	for _, c := range args {
		if err := c.Close(); err != nil {
			e = true
			ret = fmt.Errorf("%s\n%s", ret, err)
		}
	}
	if e {
		return ret
	}
	return nil
}

func setTimeout(t *testing.T, msg string, d time.Duration, f func()) {
	c := make(chan bool)

	// Make sure we are not too long
	go func() {
		time.Sleep(d)
		c <- true
	}()
	go func() {
		f()
		c <- false
	}()
	if <-c {
		t.Fatal(msg)
	}
}

func assertPipe(input, output string, r io.Reader, w io.Writer, count int) error {
	for i := 0; i < count; i++ {
		if _, err := w.Write([]byte(input)); err != nil {
			return err
		}
		o, err := bufio.NewReader(r).ReadString('\n')
		if err != nil {
			return err
		}
		if strings.Trim(o, " \r\n") != output {
			return fmt.Errorf("Unexpected output. Expected [%s], received [%s]", output, o)
		}
	}
	return nil
}

// TestRunHostname checks that 'docker run -h' correctly sets a custom hostname
func TestRunHostname(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	srv := &Server{runtime: runtime}

	var stdin, stdout bytes.Buffer
	setTimeout(t, "CmdRun timed out", 2*time.Second, func() {
		if err := srv.CmdRun(ioutil.NopCloser(&stdin), &nopWriteCloser{&stdout}, "-h", "foobar", GetTestImage(runtime).Id, "hostname"); err != nil {
			t.Fatal(err)
		}
	})
	if output := string(stdout.Bytes()); output != "foobar\n" {
		t.Fatalf("'hostname' should display '%s', not '%s'", "foobar\n", output)
	}
}

// Expected behaviour: the process dies when the client disconnects
func TestRunDisconnect(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	srv := &Server{runtime: runtime}

	stdin, stdinPipe := io.Pipe()
	stdout, stdoutPipe := io.Pipe()
	c1 := make(chan struct{})
	go func() {
		// We're simulating a disconnect so the return value doesn't matter. What matters is the
		// fact that CmdRun returns.
		srv.CmdRun(stdin, stdoutPipe, "-i", GetTestImage(runtime).Id, "/bin/cat")
		close(c1)
	}()

	setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() {
		if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil {
			t.Fatal(err)
		}
	})

	// Close pipes (simulate disconnect)
	if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil {
		t.Fatal(err)
	}

	// as the pipes are close, we expect the process to die,
	// therefore CmdRun to unblock. Wait for CmdRun
	setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() {
		<-c1
	})

	// Client disconnect after run -i should cause stdin to be closed, which should
	// cause /bin/cat to exit.
	setTimeout(t, "Waiting for /bin/cat to exit timed out", 2*time.Second, func() {
		container := runtime.List()[0]
		container.Wait()
		if container.State.Running {
			t.Fatalf("/bin/cat is still running after closing stdin")
		}
	})
}

// TestAttachStdin checks attaching to stdin without stdout and stderr.
// 'docker run -i -a stdin' should sends the client's stdin to the command,
// then detach from it and print the container id.
func TestAttachStdin(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)
	srv := &Server{runtime: runtime}

	stdinR, stdinW := io.Pipe()
	var stdout bytes.Buffer

	ch := make(chan struct{})
	go func() {
		srv.CmdRun(stdinR, &stdout, "-i", "-a", "stdin", GetTestImage(runtime).Id, "sh", "-c", "echo hello; cat")
		close(ch)
	}()

	// Send input to the command, close stdin, wait for CmdRun to return
	setTimeout(t, "Read/Write timed out", 2*time.Second, func() {
		if _, err := stdinW.Write([]byte("hi there\n")); err != nil {
			t.Fatal(err)
		}
		stdinW.Close()
		<-ch
	})

	// Check output
	cmdOutput := string(stdout.Bytes())
	container := runtime.List()[0]
	if cmdOutput != container.ShortId()+"\n" {
		t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortId()+"\n", cmdOutput)
	}

	setTimeout(t, "Waiting for command to exit timed out", 2*time.Second, func() {
		container.Wait()
	})

	// Check logs
	if cmdLogs, err := container.ReadLog("stdout"); err != nil {
		t.Fatal(err)
	} else {
		if output, err := ioutil.ReadAll(cmdLogs); err != nil {
			t.Fatal(err)
		} else {
			expectedLog := "hello\nhi there\n"
			if string(output) != expectedLog {
				t.Fatalf("Unexpected logs: should be '%s', not '%s'\n", expectedLog, output)
			}
		}
	}
}

// Expected behaviour, the process stays alive when the client disconnects
func TestAttachDisconnect(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	srv := &Server{runtime: runtime}

	container, err := runtime.Create(
		&Config{
			Image:     GetTestImage(runtime).Id,
			Memory:    33554432,
			Cmd:       []string{"/bin/cat"},
			OpenStdin: true,
		},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container)

	// Start the process
	if err := container.Start(); err != nil {
		t.Fatal(err)
	}

	stdin, stdinPipe := io.Pipe()
	stdout, stdoutPipe := io.Pipe()

	// Attach to it
	c1 := make(chan struct{})
	go func() {
		// We're simulating a disconnect so the return value doesn't matter. What matters is the
		// fact that CmdAttach returns.
		srv.CmdAttach(stdin, stdoutPipe, container.Id)
		close(c1)
	}()

	setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() {
		if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil {
			t.Fatal(err)
		}
	})
	// Close pipes (client disconnects)
	if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil {
		t.Fatal(err)
	}

	// Wait for attach to finish, the client disconnected, therefore, Attach finished his job
	setTimeout(t, "Waiting for CmdAttach timed out", 2*time.Second, func() {
		<-c1
	})
	// We closed stdin, expect /bin/cat to still be running
	// Wait a little bit to make sure container.monitor() did his thing
	err = container.WaitTimeout(500 * time.Millisecond)
	if err == nil || !container.State.Running {
		t.Fatalf("/bin/cat is not running after closing stdin")
	}

	// Try to avoid the timeoout in destroy. Best effort, don't check error
	cStdin, _ := container.StdinPipe()
	cStdin.Close()
}
<commit_msg>Add a TestRunExit, make sure cmdRun returns after process dies<commit_after>package docker

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"
)

func closeWrap(args ...io.Closer) error {
	e := false
	ret := fmt.Errorf("Error closing elements")
	for _, c := range args {
		if err := c.Close(); err != nil {
			e = true
			ret = fmt.Errorf("%s\n%s", ret, err)
		}
	}
	if e {
		return ret
	}
	return nil
}

func setTimeout(t *testing.T, msg string, d time.Duration, f func()) {
	c := make(chan bool)

	// Make sure we are not too long
	go func() {
		time.Sleep(d)
		c <- true
	}()
	go func() {
		f()
		c <- false
	}()
	if <-c {
		t.Fatal(msg)
	}
}

func assertPipe(input, output string, r io.Reader, w io.Writer, count int) error {
	for i := 0; i < count; i++ {
		if _, err := w.Write([]byte(input)); err != nil {
			return err
		}
		o, err := bufio.NewReader(r).ReadString('\n')
		if err != nil {
			return err
		}
		if strings.Trim(o, " \r\n") != output {
			return fmt.Errorf("Unexpected output. Expected [%s], received [%s]", output, o)
		}
	}
	return nil
}

// TestRunHostname checks that 'docker run -h' correctly sets a custom hostname
func TestRunHostname(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	srv := &Server{runtime: runtime}

	var stdin, stdout bytes.Buffer
	setTimeout(t, "CmdRun timed out", 2*time.Second, func() {
		if err := srv.CmdRun(ioutil.NopCloser(&stdin), &nopWriteCloser{&stdout}, "-h", "foobar", GetTestImage(runtime).Id, "hostname"); err != nil {
			t.Fatal(err)
		}
	})
	if output := string(stdout.Bytes()); output != "foobar\n" {
		t.Fatalf("'hostname' should display '%s', not '%s'", "foobar\n", output)
	}
}

func TestRunExit(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	srv := &Server{runtime: runtime}

	stdin, stdinPipe := io.Pipe()
	stdout, stdoutPipe := io.Pipe()
	c1 := make(chan struct{})
	go func() {
		if err := srv.CmdRun(stdin, stdoutPipe, "-i", GetTestImage(runtime).Id, "/bin/cat"); err != nil {
			t.Error(err)
		}
		close(c1)
	}()

	setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() {
		if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil {
			t.Fatal(err)
		}
	})

	container := runtime.List()[0]

	// Closing /bin/cat stdin, expect it to exit
	p, err := container.StdinPipe()
	if err != nil {
		t.Fatal(err)
	}
	if err := p.Close(); err != nil {
		t.Fatal(err)
	}

	// as the process exited, CmdRun must finish and unblock. Wait for it
	setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() {
		<-c1
	})

	// Make sure that the client has been disconnected
	setTimeout(t, "The client should have been disconnected once the remote process exited.", 2*time.Second, func() {
		if _, err := stdin.Read([]byte{}); err != nil {
			if err != io.EOF {
				t.Fatal(err)
			}
		}
	})

	// Cleanup pipes
	if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil {
		t.Fatal(err)
	}
}

// Expected behaviour: the process dies when the client disconnects
func TestRunDisconnect(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	srv := &Server{runtime: runtime}

	stdin, stdinPipe := io.Pipe()
	stdout, stdoutPipe := io.Pipe()
	c1 := make(chan struct{})
	go func() {
		// We're simulating a disconnect so the return value doesn't matter. What matters is the
		// fact that CmdRun returns.
		srv.CmdRun(stdin, stdoutPipe, "-i", GetTestImage(runtime).Id, "/bin/cat")
		close(c1)
	}()

	setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() {
		if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil {
			t.Fatal(err)
		}
	})

	// Close pipes (simulate disconnect)
	if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil {
		t.Fatal(err)
	}

	// as the pipes are close, we expect the process to die,
	// therefore CmdRun to unblock. Wait for CmdRun
	setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() {
		<-c1
	})

	// Client disconnect after run -i should cause stdin to be closed, which should
	// cause /bin/cat to exit.
	setTimeout(t, "Waiting for /bin/cat to exit timed out", 2*time.Second, func() {
		container := runtime.List()[0]
		container.Wait()
		if container.State.Running {
			t.Fatalf("/bin/cat is still running after closing stdin")
		}
	})
}

// TestAttachStdin checks attaching to stdin without stdout and stderr.
// 'docker run -i -a stdin' should sends the client's stdin to the command,
// then detach from it and print the container id.
func TestAttachStdin(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)
	srv := &Server{runtime: runtime}

	stdinR, stdinW := io.Pipe()
	var stdout bytes.Buffer

	ch := make(chan struct{})
	go func() {
		srv.CmdRun(stdinR, &stdout, "-i", "-a", "stdin", GetTestImage(runtime).Id, "sh", "-c", "echo hello; cat")
		close(ch)
	}()

	// Send input to the command, close stdin, wait for CmdRun to return
	setTimeout(t, "Read/Write timed out", 2*time.Second, func() {
		if _, err := stdinW.Write([]byte("hi there\n")); err != nil {
			t.Fatal(err)
		}
		stdinW.Close()
		<-ch
	})

	// Check output
	cmdOutput := string(stdout.Bytes())
	container := runtime.List()[0]
	if cmdOutput != container.ShortId()+"\n" {
		t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortId()+"\n", cmdOutput)
	}

	setTimeout(t, "Waiting for command to exit timed out", 2*time.Second, func() {
		container.Wait()
	})

	// Check logs
	if cmdLogs, err := container.ReadLog("stdout"); err != nil {
		t.Fatal(err)
	} else {
		if output, err := ioutil.ReadAll(cmdLogs); err != nil {
			t.Fatal(err)
		} else {
			expectedLog := "hello\nhi there\n"
			if string(output) != expectedLog {
				t.Fatalf("Unexpected logs: should be '%s', not '%s'\n", expectedLog, output)
			}
		}
	}
}

// Expected behaviour, the process stays alive when the client disconnects
func TestAttachDisconnect(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	srv := &Server{runtime: runtime}

	container, err := runtime.Create(
		&Config{
			Image:     GetTestImage(runtime).Id,
			Memory:    33554432,
			Cmd:       []string{"/bin/cat"},
			OpenStdin: true,
		},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container)

	// Start the process
	if err := container.Start(); err != nil {
		t.Fatal(err)
	}

	stdin, stdinPipe := io.Pipe()
	stdout, stdoutPipe := io.Pipe()

	// Attach to it
	c1 := make(chan struct{})
	go func() {
		// We're simulating a disconnect so the return value doesn't matter. What matters is the
		// fact that CmdAttach returns.
		srv.CmdAttach(stdin, stdoutPipe, container.Id)
		close(c1)
	}()

	setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() {
		if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil {
			t.Fatal(err)
		}
	})
	// Close pipes (client disconnects)
	if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil {
		t.Fatal(err)
	}

	// Wait for attach to finish, the client disconnected, therefore, Attach finished his job
	setTimeout(t, "Waiting for CmdAttach timed out", 2*time.Second, func() {
		<-c1
	})
	// We closed stdin, expect /bin/cat to still be running
	// Wait a little bit to make sure container.monitor() did his thing
	err = container.WaitTimeout(500 * time.Millisecond)
	if err == nil || !container.State.Running {
		t.Fatalf("/bin/cat is not running after closing stdin")
	}

	// Try to avoid the timeoout in destroy. Best effort, don't check error
	cStdin, _ := container.StdinPipe()
	cStdin.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/cisco/senml"
	"io/ioutil"
	"net/http"
	"os"
	"runtime/pprof"
	"errors"
)

var doIndentPtr = flag.Bool("i", false, "indent output")
var doPrintPtr = flag.Bool("print", false, "print output to stdout")
var doResolvePtr = flag.Bool("resolve", false, "resolve SenML records")
var postUrl = flag.String("post", "", "URL to HTTP POST output to")
var topic = flag.String("topic", "senml", "Apache Kafka topic or InfluxDB series name ")

var doJsonPtr = flag.Bool("json", false, "output JSON formatted SenML ")
var doCborPtr = flag.Bool("cbor", false, "output CBOR formatted SenML ")
var doXmlPtr = flag.Bool("xml", false, "output XML formatted SenML ")
var doCsvPtr = flag.Bool("csv", false, "output CSV formatted SenML ")
var doMpackPtr = flag.Bool("mpack", false, "output MessagePack formatted SenML ")
var doLinpPtr = flag.Bool("linp", false, "output InfluxDB LineProtcol formatted SenML ")
var doJsonLinePtr = flag.Bool("jsonl", false, "outpute JSON formatted SenML Record lines")

var doIJsonStreamPtr = flag.Bool("ijson", false, "input JSON formatted SenML")
var doIJsonLinePtr = flag.Bool("ijsonl", false, "input JSON formatted SenML Record lines")
var doIXmlPtr = flag.Bool("ixml", false, "input XML formatted SenML ")
var doICborPtr = flag.Bool("icbor", false, "input CBOR formatted SenML ")
var doIMpackPtr = flag.Bool("impack", false, "input MessagePack formatted SenML ")

func decodeTimed(msg []byte) (senml.SenML, error) {
	var s senml.SenML
	var err error

	var format senml.Format = senml.JSON
	switch {
	case *doIJsonStreamPtr:
		format = senml.JSON
	case *doIJsonLinePtr:
		format = senml.JSONLINE
	case *doICborPtr:
		format = senml.CBOR
	case *doIXmlPtr:
		format = senml.XML
	case *doIMpackPtr:
		format = senml.MPACK
	}

	s, err = senml.Decode(msg, format)

	return s, err
}

func outputData(data []byte) error {
	if *doPrintPtr {
		fmt.Print(string(data))
	}

	if len(*postUrl) != 0 {
		fmt.Println("PostURL=<" + string(*postUrl) + ">")
		buffer := bytes.NewBuffer(data)
		resp, err := http.Post(string(*postUrl), "application/senml+json", buffer)
		if err != nil {
			fmt.Println("Post to", string(*postUrl), " got error", err.Error())
			return err
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if  resp.StatusCode  != 204 {
			fmt.Println("Post got status " , resp.Status )
			fmt.Println("Post got",  string(body) )
			return errors.New("WebServer returned: " + string(body) )
		}
	}

	return nil
}

func processData(dataIn []byte) error {
	var s senml.SenML
	var err error

	s, err = decodeTimed(dataIn)
	if err != nil {
		fmt.Println("Decode of SenML failed")
		return err
	}

	//fmt.Println( "Senml:", senml.Records )
	if *doResolvePtr {
		s = senml.Normalize(s)
	}

	var dataOut []byte
	options := senml.OutputOptions{}
	if *doIndentPtr {
		options.PrettyPrint = *doIndentPtr
	}
	options.Topic = string(*topic)
	var format senml.Format = senml.JSON
	switch {
	case *doJsonPtr:
		format = senml.JSON
	case *doJsonLinePtr:
		format = senml.JSONLINE
	case *doCborPtr:
		format = senml.CBOR
	case *doXmlPtr:
		format = senml.XML
	case *doCsvPtr:
		format = senml.CSV
	case *doMpackPtr:
		format = senml.MPACK
	case *doLinpPtr:
		format = senml.LINEP
	}
	dataOut, err = senml.Encode(s, format, options)
	if err != nil {
		fmt.Println("Encode of SenML failed")
		return err
	}

	if false {
		reader := bytes.NewReader(dataOut)
		lines := bufio.NewScanner(reader)
		for lines.Scan() {
			err = outputData( []byte( lines.Text() ) )
			if err != nil {
				fmt.Println("Output of SenML failed:", err)
				return err
			}
		}

		err = lines.Err();
		if err != nil {
			fmt.Println("Encode scanning lines in output")
		}
	} else {
		err = outputData( dataOut )
		if err != nil {
			fmt.Println("Output of SenML failed:", err)
			return err
		}
	}

	return nil
}

func main() {
	var err error

	if false {
        f, err := os.Create( "senmlCat.prof" )
        if err != nil {
            fmt.Println("error opening profile file", err)
			os.Exit(1)
        }
        pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()
    }

	flag.Parse()

	//fmt.Print("Reading file ...")
	// load the input
	msg, err := ioutil.ReadFile(flag.Arg(0))
	if err != nil {
		fmt.Println("error reading SenML file", err)
		os.Exit(1)
	}
	//fmt.Println("Done")

	err = processData(msg)
	if err != nil {
		fmt.Println("error processing SenML file", err)
		os.Exit(1)
	}
}
<commit_msg>update<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/cisco/senml"
	"io/ioutil"
	"net/http"
	"os"
	"runtime/pprof"
	"errors"
)

var doIndentPtr = flag.Bool("i", false, "indent output")
var doPrintPtr = flag.Bool("print", false, "print output to stdout")
var doResolvePtr = flag.Bool("resolve", false, "resolve SenML records")
var postUrl = flag.String("post", "", "URL to HTTP POST output to")
var topic = flag.String("topic", "senml", "Apache Kafka topic or InfluxDB series name ")

var doJsonPtr = flag.Bool("json", false, "output JSON formatted SenML ")
var doCborPtr = flag.Bool("cbor", false, "output CBOR formatted SenML ")
var doXmlPtr = flag.Bool("xml", false, "output XML formatted SenML ")
var doCsvPtr = flag.Bool("csv", false, "output CSV formatted SenML ")
var doMpackPtr = flag.Bool("mpack", false, "output MessagePack formatted SenML ")
var doLinpPtr = flag.Bool("linp", false, "output InfluxDB LineProtcol formatted SenML ")
var doJsonLinePtr = flag.Bool("jsonl", false, "outpute JSON formatted SenML Record lines")

var doIJsonStreamPtr = flag.Bool("ijson", false, "input JSON formatted SenML")
var doIJsonLinePtr = flag.Bool("ijsonl", false, "input JSON formatted SenML Record lines")
var doIXmlPtr = flag.Bool("ixml", false, "input XML formatted SenML ")
var doICborPtr = flag.Bool("icbor", false, "input CBOR formatted SenML ")
var doIMpackPtr = flag.Bool("impack", false, "input MessagePack formatted SenML ")

func decodeTimed(msg []byte) (senml.SenML, error) {
	var s senml.SenML
	var err error

	var format senml.Format = senml.JSON
	switch {
	case *doIJsonStreamPtr:
		format = senml.JSON
	case *doIJsonLinePtr:
		format = senml.JSONLINE
	case *doICborPtr:
		format = senml.CBOR
	case *doIXmlPtr:
		format = senml.XML
	case *doIMpackPtr:
		format = senml.MPACK
	}

	s, err = senml.Decode(msg, format)

	return s, err
}

func outputData(data []byte) error {
	if *doPrintPtr {
		fmt.Print(string(data))
	}

	if len(*postUrl) != 0 {
		fmt.Println("PostURL=<" + string(*postUrl) + ">")
		buffer := bytes.NewBuffer(data)
		resp, err := http.Post(string(*postUrl), "application/senml+json", buffer)
		if err != nil {
			fmt.Println("Post to", string(*postUrl), " got error", err.Error())
			return err
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if  resp.StatusCode  != 204 {
			fmt.Println("Post got status " , resp.Status )
			fmt.Println("Post got",  string(body) )
			return errors.New("WebServer returned: " + string(body) )
		}
	}

	return nil
}

func processData(dataIn []byte) error {
	var s senml.SenML
	var err error

	s, err = decodeTimed(dataIn)
	if err != nil {
		fmt.Println("Decode of SenML failed")
		return err
	}

     	//fmt.Println( "Senml:", senml.Records )
	if *doResolvePtr {
		s = senml.Normalize(s)
	}

	var dataOut []byte
	options := senml.OutputOptions{}
	if *doIndentPtr {
		options.PrettyPrint = *doIndentPtr
	}
	options.Topic = string(*topic)
	var format senml.Format = senml.JSON
	switch {
	case *doJsonPtr:
		format = senml.JSON
	case *doJsonLinePtr:
		format = senml.JSONLINE
	case *doCborPtr:
		format = senml.CBOR
	case *doXmlPtr:
		format = senml.XML
	case *doCsvPtr:
		format = senml.CSV
	case *doMpackPtr:
		format = senml.MPACK
	case *doLinpPtr:
		format = senml.LINEP
	}
	dataOut, err = senml.Encode(s, format, options)
	if err != nil {
		fmt.Println("Encode of SenML failed")
		return err
	}

	if false {
		reader := bytes.NewReader(dataOut)
		lines := bufio.NewScanner(reader)
		for lines.Scan() {
			err = outputData( []byte( lines.Text() ) )
			if err != nil {
				fmt.Println("Output of SenML failed:", err)
				return err
			}
		}

		err = lines.Err();
		if err != nil {
			fmt.Println("Encode scanning lines in output")
		}
	} else {
		err = outputData( dataOut )
		if err != nil {
			fmt.Println("Output of SenML failed:", err)
			return err
		}
	}

	return nil
}

func main() {
	var err error

	if false {
        f, err := os.Create( "senmlCat.prof" )
        if err != nil {
            fmt.Println("error opening profile file", err)
			os.Exit(1)
        }
        pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()
    }

	flag.Parse()

	//fmt.Print("Reading file ...")
	// load the input
	msg, err := ioutil.ReadFile(flag.Arg(0))
	if err != nil {
		fmt.Println("error reading SenML file", err)
		os.Exit(1)
	}
	//fmt.Println("Done")

	err = processData(msg)
	if err != nil {
		fmt.Println("error processing SenML file", err)
		os.Exit(1)
	}
}
<|endoftext|>