<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"time"
	"unsafe"

	"github.com/Sirupsen/logrus"
	"github.com/spf13/pflag"
	"golang.org/x/sys/windows"
	"golang.org/x/sys/windows/svc"
	"golang.org/x/sys/windows/svc/debug"
	"golang.org/x/sys/windows/svc/eventlog"
	"golang.org/x/sys/windows/svc/mgr"
)

var (
	flServiceName       *string
	flRegisterService   *bool
	flUnregisterService *bool
	flRunService        *bool

	setStdHandle = windows.NewLazySystemDLL("kernel32.dll").NewProc("SetStdHandle")
	oldStderr    syscall.Handle
	panicFile    *os.File

	service *handler
)

const (
	// These should match the values in event_messages.mc.
	eventInfo  = 1
	eventWarn  = 1
	eventError = 1
	eventDebug = 2
	eventPanic = 3
	eventFatal = 4

	eventExtraOffset = 10 // Add this to any event to get a string that supports extended data
)

func installServiceFlags(flags *pflag.FlagSet) {
	flServiceName = flags.String("service-name", "docker", "Set the Windows service name")
	flRegisterService = flags.Bool("register-service", false, "Register the service and exit")
	flUnregisterService = flags.Bool("unregister-service", false, "Unregister the service and exit")
	flRunService = flags.Bool("run-service", false, "")
	flags.MarkHidden("run-service")
}

type handler struct {
	tosvc     chan bool
	fromsvc   chan error
	daemonCli *DaemonCli
}

type etwHook struct {
	log *eventlog.Log
}

func (h *etwHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}

func (h *etwHook) Fire(e *logrus.Entry) error {
	var (
		etype uint16
		eid   uint32
	)

	switch e.Level {
	case logrus.PanicLevel:
		etype = windows.EVENTLOG_ERROR_TYPE
		eid = eventPanic
	case logrus.FatalLevel:
		etype = windows.EVENTLOG_ERROR_TYPE
		eid = eventFatal
	case logrus.ErrorLevel:
		etype = windows.EVENTLOG_ERROR_TYPE
		eid = eventError
	case logrus.WarnLevel:
		etype = windows.EVENTLOG_WARNING_TYPE
		eid = eventWarn
	case logrus.InfoLevel:
		etype = windows.EVENTLOG_INFORMATION_TYPE
		eid = eventInfo
	case logrus.DebugLevel:
		etype = windows.EVENTLOG_INFORMATION_TYPE
		eid = eventDebug
	default:
		return errors.New("unknown level")
	}

	// If there is additional data, include it as a second string.
	exts := ""
	if len(e.Data) > 0 {
		fs := bytes.Buffer{}
		for k, v := range e.Data {
			fs.WriteString(k)
			fs.WriteByte('=')
			fmt.Fprint(&fs, v)
			fs.WriteByte(' ')
		}

		exts = fs.String()[:fs.Len()-1]
		eid += eventExtraOffset
	}

	if h.log == nil {
		fmt.Fprintf(os.Stderr, "%s [%s]\n", e.Message, exts)
		return nil
	}

	var (
		ss  [2]*uint16
		err error
	)

	ss[0], err = syscall.UTF16PtrFromString(e.Message)
	if err != nil {
		return err
	}

	count := uint16(1)
	if exts != "" {
		ss[1], err = syscall.UTF16PtrFromString(exts)
		if err != nil {
			return err
		}

		count++
	}

	return windows.ReportEvent(h.log.Handle, etype, 0, eid, 0, count, 0, &ss[0], nil)
}

func getServicePath() (string, error) {
	p, err := exec.LookPath(os.Args[0])
	if err != nil {
		return "", err
	}
	return filepath.Abs(p)
}

func registerService() error {
	p, err := getServicePath()
	if err != nil {
		return err
	}
	m, err := mgr.Connect()
	if err != nil {
		return err
	}
	defer m.Disconnect()
	c := mgr.Config{
		ServiceType:  windows.SERVICE_WIN32_OWN_PROCESS,
		StartType:    mgr.StartAutomatic,
		ErrorControl: mgr.ErrorNormal,
		DisplayName:  "Docker Engine",
	}

	// Configure the service to launch with the arguments that were just passed.
	args := []string{"--run-service"}
	for _, a := range os.Args[1:] {
		if a != "--register-service" && a != "--unregister-service" {
			args = append(args, a)
		}
	}

	s, err := m.CreateService(*flServiceName, p, c, args...)
	if err != nil {
		return err
	}
	defer s.Close()

	// See http://stackoverflow.com/questions/35151052/how-do-i-configure-failure-actions-of-a-windows-service-written-in-go
	const (
		scActionNone       = 0
		scActionRestart    = 1
		scActionReboot     = 2
		scActionRunCommand = 3

		serviceConfigFailureActions = 2
	)

	type serviceFailureActions struct {
		ResetPeriod  uint32
		RebootMsg    *uint16
		Command      *uint16
		ActionsCount uint32
		Actions      uintptr
	}

	type scAction struct {
		Type  uint32
		Delay uint32
	}
	t := []scAction{
		{Type: scActionRestart, Delay: uint32(60 * time.Second / time.Millisecond)},
		{Type: scActionRestart, Delay: uint32(60 * time.Second / time.Millisecond)},
		{Type: scActionNone},
	}
	lpInfo := serviceFailureActions{ResetPeriod: uint32(24 * time.Hour / time.Second), ActionsCount: uint32(3), Actions: uintptr(unsafe.Pointer(&t[0]))}
	err = windows.ChangeServiceConfig2(s.Handle, serviceConfigFailureActions, (*byte)(unsafe.Pointer(&lpInfo)))
	if err != nil {
		return err
	}

	return eventlog.Install(*flServiceName, p, false, eventlog.Info|eventlog.Warning|eventlog.Error)
}

func unregisterService() error {
	m, err := mgr.Connect()
	if err != nil {
		return err
	}
	defer m.Disconnect()

	s, err := m.OpenService(*flServiceName)
	if err != nil {
		return err
	}
	defer s.Close()

	eventlog.Remove(*flServiceName)
	err = s.Delete()
	if err != nil {
		return err
	}
	return nil
}

func initService(daemonCli *DaemonCli) (bool, error) {
	if *flUnregisterService {
		if *flRegisterService {
			return true, errors.New("--register-service and --unregister-service cannot be used together")
		}
		return true, unregisterService()
	}

	if *flRegisterService {
		return true, registerService()
	}

	if !*flRunService {
		return false, nil
	}

	interactive, err := svc.IsAnInteractiveSession()
	if err != nil {
		return false, err
	}

	h := &handler{
		tosvc:     make(chan bool),
		fromsvc:   make(chan error),
		daemonCli: daemonCli,
	}

	var log *eventlog.Log
	if !interactive {
		log, err = eventlog.Open(*flServiceName)
		if err != nil {
			return false, err
		}
	}

	logrus.AddHook(&etwHook{log})
	logrus.SetOutput(ioutil.Discard)

	service = h
	go func() {
		if interactive {
			err = debug.Run(*flServiceName, h)
		} else {
			err = svc.Run(*flServiceName, h)
		}

		h.fromsvc <- err
	}()

	// Wait for the first signal from the service handler.
	err = <-h.fromsvc
	if err != nil {
		return false, err
	}
	return false, nil
}

func (h *handler) started() error {
	// This must be delayed until daemonCli initializes Config.Root
	err := initPanicFile(filepath.Join(h.daemonCli.Config.Root, "panic.log"))
	if err != nil {
		return err
	}

	h.tosvc <- false
	return nil
}

func (h *handler) stopped(err error) {
	logrus.Debugf("Stopping service: %v", err)
	h.tosvc <- err != nil
	<-h.fromsvc
}

func (h *handler) Execute(_ []string, r <-chan svc.ChangeRequest, s chan<- svc.Status) (bool, uint32) {
	s <- svc.Status{State: svc.StartPending, Accepts: 0}
	// Unblock initService()
	h.fromsvc <- nil

	// Wait for initialization to complete.
	failed := <-h.tosvc
	if failed {
		logrus.Debug("Aborting service start due to failure during initialization")
		return true, 1
	}

	s <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE)}
	logrus.Debug("Service running")
Loop:
	for {
		select {
		case failed = <-h.tosvc:
			break Loop
		case c := <-r:
			switch c.Cmd {
			case svc.Cmd(windows.SERVICE_CONTROL_PARAMCHANGE):
				h.daemonCli.reloadConfig()
			case svc.Interrogate:
				s <- c.CurrentStatus
			case svc.Stop, svc.Shutdown:
				s <- svc.Status{State: svc.StopPending, Accepts: 0}
				h.daemonCli.stop()
			}
		}
	}

	removePanicFile()
	if failed {
		return true, 1
	}
	return false, 0
}

func initPanicFile(path string) error {
	var err error
	panicFile, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		return err
	}

	st, err := panicFile.Stat()
	if err != nil {
		return err
	}

	// If there are contents in the file already, move the file out of the way
	// and replace it.
	if st.Size() > 0 {
		panicFile.Close()
		os.Rename(path, path+".old")
		panicFile, err = os.Create(path)
		if err != nil {
			return err
		}
	}

	// Update STD_ERROR_HANDLE to point to the panic file so that Go writes to
	// it when it panics. Remember the old stderr to restore it before removing
	// the panic file.
	sh := syscall.STD_ERROR_HANDLE
	h, err := syscall.GetStdHandle(sh)
	if err != nil {
		return err
	}

	oldStderr = h

	r, _, err := setStdHandle.Call(uintptr(sh), uintptr(panicFile.Fd()))
	if r == 0 && err != nil {
		return err
	}

	return nil
}

func removePanicFile() {
	if st, err := panicFile.Stat(); err == nil {
		if st.Size() == 0 {
			sh := syscall.STD_ERROR_HANDLE
			setStdHandle.Call(uintptr(sh), uintptr(oldStderr))
			panicFile.Close()
			os.Remove(panicFile.Name())
		}
	}
}
<commit_msg>Windows: Add service dependency ConDrv<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"time"
	"unsafe"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/system"
	"github.com/spf13/pflag"
	"golang.org/x/sys/windows"
	"golang.org/x/sys/windows/svc"
	"golang.org/x/sys/windows/svc/debug"
	"golang.org/x/sys/windows/svc/eventlog"
	"golang.org/x/sys/windows/svc/mgr"
)

var (
	flServiceName       *string
	flRegisterService   *bool
	flUnregisterService *bool
	flRunService        *bool

	setStdHandle = windows.NewLazySystemDLL("kernel32.dll").NewProc("SetStdHandle")
	oldStderr    syscall.Handle
	panicFile    *os.File

	service *handler
)

const (
	// These should match the values in event_messages.mc.
	eventInfo  = 1
	eventWarn  = 1
	eventError = 1
	eventDebug = 2
	eventPanic = 3
	eventFatal = 4

	eventExtraOffset = 10 // Add this to any event to get a string that supports extended data
)

func installServiceFlags(flags *pflag.FlagSet) {
	flServiceName = flags.String("service-name", "docker", "Set the Windows service name")
	flRegisterService = flags.Bool("register-service", false, "Register the service and exit")
	flUnregisterService = flags.Bool("unregister-service", false, "Unregister the service and exit")
	flRunService = flags.Bool("run-service", false, "")
	flags.MarkHidden("run-service")
}

type handler struct {
	tosvc     chan bool
	fromsvc   chan error
	daemonCli *DaemonCli
}

type etwHook struct {
	log *eventlog.Log
}

func (h *etwHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}

func (h *etwHook) Fire(e *logrus.Entry) error {
	var (
		etype uint16
		eid   uint32
	)

	switch e.Level {
	case logrus.PanicLevel:
		etype = windows.EVENTLOG_ERROR_TYPE
		eid = eventPanic
	case logrus.FatalLevel:
		etype = windows.EVENTLOG_ERROR_TYPE
		eid = eventFatal
	case logrus.ErrorLevel:
		etype = windows.EVENTLOG_ERROR_TYPE
		eid = eventError
	case logrus.WarnLevel:
		etype = windows.EVENTLOG_WARNING_TYPE
		eid = eventWarn
	case logrus.InfoLevel:
		etype = windows.EVENTLOG_INFORMATION_TYPE
		eid = eventInfo
	case logrus.DebugLevel:
		etype = windows.EVENTLOG_INFORMATION_TYPE
		eid = eventDebug
	default:
		return errors.New("unknown level")
	}

	// If there is additional data, include it as a second string.
	exts := ""
	if len(e.Data) > 0 {
		fs := bytes.Buffer{}
		for k, v := range e.Data {
			fs.WriteString(k)
			fs.WriteByte('=')
			fmt.Fprint(&fs, v)
			fs.WriteByte(' ')
		}

		exts = fs.String()[:fs.Len()-1]
		eid += eventExtraOffset
	}

	if h.log == nil {
		fmt.Fprintf(os.Stderr, "%s [%s]\n", e.Message, exts)
		return nil
	}

	var (
		ss  [2]*uint16
		err error
	)

	ss[0], err = syscall.UTF16PtrFromString(e.Message)
	if err != nil {
		return err
	}

	count := uint16(1)
	if exts != "" {
		ss[1], err = syscall.UTF16PtrFromString(exts)
		if err != nil {
			return err
		}

		count++
	}

	return windows.ReportEvent(h.log.Handle, etype, 0, eid, 0, count, 0, &ss[0], nil)
}

func getServicePath() (string, error) {
	p, err := exec.LookPath(os.Args[0])
	if err != nil {
		return "", err
	}
	return filepath.Abs(p)
}

func registerService() error {
	p, err := getServicePath()
	if err != nil {
		return err
	}
	m, err := mgr.Connect()
	if err != nil {
		return err
	}
	defer m.Disconnect()

	depends := []string{}

	// This dependency is required on build 14393 (RS1)
	// it is added to the platform in newer builds
	if system.GetOSVersion().Build == 14393 {
		depends = append(depends, "ConDrv")
	}

	c := mgr.Config{
		ServiceType:  windows.SERVICE_WIN32_OWN_PROCESS,
		StartType:    mgr.StartAutomatic,
		ErrorControl: mgr.ErrorNormal,
		Dependencies: depends,
		DisplayName:  "Docker Engine",
	}

	// Configure the service to launch with the arguments that were just passed.
	args := []string{"--run-service"}
	for _, a := range os.Args[1:] {
		if a != "--register-service" && a != "--unregister-service" {
			args = append(args, a)
		}
	}

	s, err := m.CreateService(*flServiceName, p, c, args...)
	if err != nil {
		return err
	}
	defer s.Close()

	// See http://stackoverflow.com/questions/35151052/how-do-i-configure-failure-actions-of-a-windows-service-written-in-go
	const (
		scActionNone       = 0
		scActionRestart    = 1
		scActionReboot     = 2
		scActionRunCommand = 3

		serviceConfigFailureActions = 2
	)

	type serviceFailureActions struct {
		ResetPeriod  uint32
		RebootMsg    *uint16
		Command      *uint16
		ActionsCount uint32
		Actions      uintptr
	}

	type scAction struct {
		Type  uint32
		Delay uint32
	}
	t := []scAction{
		{Type: scActionRestart, Delay: uint32(60 * time.Second / time.Millisecond)},
		{Type: scActionRestart, Delay: uint32(60 * time.Second / time.Millisecond)},
		{Type: scActionNone},
	}
	lpInfo := serviceFailureActions{ResetPeriod: uint32(24 * time.Hour / time.Second), ActionsCount: uint32(3), Actions: uintptr(unsafe.Pointer(&t[0]))}
	err = windows.ChangeServiceConfig2(s.Handle, serviceConfigFailureActions, (*byte)(unsafe.Pointer(&lpInfo)))
	if err != nil {
		return err
	}

	return eventlog.Install(*flServiceName, p, false, eventlog.Info|eventlog.Warning|eventlog.Error)
}

func unregisterService() error {
	m, err := mgr.Connect()
	if err != nil {
		return err
	}
	defer m.Disconnect()

	s, err := m.OpenService(*flServiceName)
	if err != nil {
		return err
	}
	defer s.Close()

	eventlog.Remove(*flServiceName)
	err = s.Delete()
	if err != nil {
		return err
	}
	return nil
}

func initService(daemonCli *DaemonCli) (bool, error) {
	if *flUnregisterService {
		if *flRegisterService {
			return true, errors.New("--register-service and --unregister-service cannot be used together")
		}
		return true, unregisterService()
	}

	if *flRegisterService {
		return true, registerService()
	}

	if !*flRunService {
		return false, nil
	}

	interactive, err := svc.IsAnInteractiveSession()
	if err != nil {
		return false, err
	}

	h := &handler{
		tosvc:     make(chan bool),
		fromsvc:   make(chan error),
		daemonCli: daemonCli,
	}

	var log *eventlog.Log
	if !interactive {
		log, err = eventlog.Open(*flServiceName)
		if err != nil {
			return false, err
		}
	}

	logrus.AddHook(&etwHook{log})
	logrus.SetOutput(ioutil.Discard)

	service = h
	go func() {
		if interactive {
			err = debug.Run(*flServiceName, h)
		} else {
			err = svc.Run(*flServiceName, h)
		}

		h.fromsvc <- err
	}()

	// Wait for the first signal from the service handler.
	err = <-h.fromsvc
	if err != nil {
		return false, err
	}
	return false, nil
}

func (h *handler) started() error {
	// This must be delayed until daemonCli initializes Config.Root
	err := initPanicFile(filepath.Join(h.daemonCli.Config.Root, "panic.log"))
	if err != nil {
		return err
	}

	h.tosvc <- false
	return nil
}

func (h *handler) stopped(err error) {
	logrus.Debugf("Stopping service: %v", err)
	h.tosvc <- err != nil
	<-h.fromsvc
}

func (h *handler) Execute(_ []string, r <-chan svc.ChangeRequest, s chan<- svc.Status) (bool, uint32) {
	s <- svc.Status{State: svc.StartPending, Accepts: 0}
	// Unblock initService()
	h.fromsvc <- nil

	// Wait for initialization to complete.
	failed := <-h.tosvc
	if failed {
		logrus.Debug("Aborting service start due to failure during initialization")
		return true, 1
	}

	s <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE)}
	logrus.Debug("Service running")
Loop:
	for {
		select {
		case failed = <-h.tosvc:
			break Loop
		case c := <-r:
			switch c.Cmd {
			case svc.Cmd(windows.SERVICE_CONTROL_PARAMCHANGE):
				h.daemonCli.reloadConfig()
			case svc.Interrogate:
				s <- c.CurrentStatus
			case svc.Stop, svc.Shutdown:
				s <- svc.Status{State: svc.StopPending, Accepts: 0}
				h.daemonCli.stop()
			}
		}
	}

	removePanicFile()
	if failed {
		return true, 1
	}
	return false, 0
}

func initPanicFile(path string) error {
	var err error
	panicFile, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		return err
	}

	st, err := panicFile.Stat()
	if err != nil {
		return err
	}

	// If there are contents in the file already, move the file out of the way
	// and replace it.
	if st.Size() > 0 {
		panicFile.Close()
		os.Rename(path, path+".old")
		panicFile, err = os.Create(path)
		if err != nil {
			return err
		}
	}

	// Update STD_ERROR_HANDLE to point to the panic file so that Go writes to
	// it when it panics. Remember the old stderr to restore it before removing
	// the panic file.
	sh := syscall.STD_ERROR_HANDLE
	h, err := syscall.GetStdHandle(sh)
	if err != nil {
		return err
	}

	oldStderr = h

	r, _, err := setStdHandle.Call(uintptr(sh), uintptr(panicFile.Fd()))
	if r == 0 && err != nil {
		return err
	}

	return nil
}

func removePanicFile() {
	if st, err := panicFile.Stat(); err == nil {
		if st.Size() == 0 {
			sh := syscall.STD_ERROR_HANDLE
			setStdHandle.Call(uintptr(sh), uintptr(oldStderr))
			panicFile.Close()
			os.Remove(panicFile.Name())
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"k8s.io/release/pkg/release"
	reg "sigs.k8s.io/promo-tools/v3/legacy/dockerregistry"
	"sigs.k8s.io/release-sdk/git"
	"sigs.k8s.io/release-sdk/github"
	"sigs.k8s.io/release-utils/util"
)

const (
	k8sioRepo             = "k8s.io"
	k8sioDefaultBranch    = "main"
	promotionBranchSuffix = "-image-promotion"
)

// promoteCommand is the krel subcommand to promote conainer images
var imagePromoteCommand = &cobra.Command{
	Use:   "promote-images",
	Short: "Starts an image promotion for a tag of kubernetes or kubernetes-sigs images",
	Long: `krel promote

The 'promote' subcommand of krel updates the image promoter manifests
and creates a PR in kubernetes/k8s.io`,
	SilenceUsage:  true,
	SilenceErrors: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		// Run the PR creation function
		return runPromote(promoteOpts)
	},
}

type promoteOptions struct {
	project         string
	userFork        string
	tags            []string
	reviewers       string
	interactiveMode bool
}

func (o *promoteOptions) Validate() error {
	if len(o.tags) == 0 {
		return errors.New("cannot start promotion --tag is required")
	}
	if o.userFork == "" {
		return errors.New("cannot start promotion --fork is required")
	}

	// Check the fork slug
	if _, _, err := git.ParseRepoSlug(o.userFork); err != nil {
		return errors.Wrap(err, "checking user's fork")
	}

	// Verify we got a valid tag
	for _, tag := range o.tags {
		if _, err := util.TagStringToSemver(tag); err != nil {
			return errors.Wrapf(err, "verifying tag: %s", tag)
		}
	}

	// Check that the GitHub token is set
	token, isSet := os.LookupEnv(github.TokenEnvKey)
	if !isSet || token == "" {
		return errors.New("cannot promote images if GitHub token is not set")
	}
	return nil
}

var promoteOpts = &promoteOptions{}

func init() {
	imagePromoteCommand.PersistentFlags().StringVar(
		&promoteOpts.project,
		"project",
		release.Kubernetes,
		"the name of the project to promote images for",
	)

	imagePromoteCommand.PersistentFlags().StringSliceVarP(
		&promoteOpts.tags,
		"tag",
		"t",
		[]string{},
		"version tag of the images we will promote",
	)

	imagePromoteCommand.PersistentFlags().StringVar(
		&promoteOpts.userFork,
		"fork",
		"",
		"the user's fork of kubernetes/k8s.io",
	)

	imagePromoteCommand.PersistentFlags().StringVar(
		&promoteOpts.reviewers,
		"reviewers",
		"@kubernetes/release-engineering",
		"the list of GitHub users or teams to assign to the PR",
	)

	imagePromoteCommand.PersistentFlags().BoolVarP(
		&promoteOpts.interactiveMode,
		"interactive",
		"i",
		false,
		"interactive mode, asks before every step",
	)

	for _, flagName := range []string{"tag", "fork"} {
		if err := imagePromoteCommand.MarkPersistentFlagRequired(flagName); err != nil {
			logrus.Error(errors.Wrapf(err, "marking tag %s as required", flagName))
		}
	}

	rootCmd.AddCommand(imagePromoteCommand)
}

func runPromote(opts *promoteOptions) error {
	// Check the cmd line opts
	if err := opts.Validate(); err != nil {
		return errors.Wrap(err, "checking command line options")
	}

	ctx := context.Background()

	// Validate options
	branchname := opts.project + "-" + opts.tags[0] + promotionBranchSuffix

	// Get the github org and repo from the fork slug
	userForkOrg, userForkRepo, err := git.ParseRepoSlug(opts.userFork)
	if err != nil {
		return errors.Wrap(err, "parsing user's fork")
	}
	if userForkRepo == "" {
		userForkRepo = k8sioRepo
	}

	// Check Environment
	gh := github.New()

	// Verify the repository is a fork of k8s.io
	if err = verifyFork(
		branchname, userForkOrg, userForkRepo, git.DefaultGithubOrg, k8sioRepo,
	); err != nil {
		return errors.Wrapf(err, "while checking fork of %s/%s ", git.DefaultGithubOrg, k8sioRepo)
	}

	// Clone k8s.io
	repo, err := prepareFork(branchname, git.DefaultGithubOrg, k8sioRepo, userForkOrg, userForkRepo)
	if err != nil {
		return errors.Wrap(err, "while preparing k/k8s.io fork")
	}

	defer func() {
		if mustRun(opts, "Clean fork directory?") {
			err = repo.Cleanup()
		} else {
			logrus.Infof("All modified files will be left untouched in %s", repo.Dir())
		}
	}()

	// Path to the promoter image list
	imagesListPath := filepath.Join(
		release.GCRIOPathProd,
		"images",
		filepath.Base(release.GCRIOPathStagingPrefix)+opts.project,
		"images.yaml",
	)

	// Read the current manifest to check later if new images come up
	oldlist := make([]byte, 0)

	// Run the promoter manifest grower
	if mustRun(opts, "Grow the manifests to add the new tags?") {
		if util.Exists(filepath.Join(repo.Dir(), imagesListPath)) {
			logrus.Debug("Reading the current image promoter manifest (image list)")
			oldlist, err = os.ReadFile(filepath.Join(repo.Dir(), imagesListPath))
			if err != nil {
				return errors.Wrap(err, "while reading the current promoter image list")
			}
		}

		for _, tag := range opts.tags {
			opt := reg.GrowManifestOptions{}
			if err := opt.Populate(
				filepath.Join(repo.Dir(), release.GCRIOPathProd),
				release.GCRIOPathStagingPrefix+opts.project, "", "", tag); err != nil {
				return errors.Wrapf(err, "populating image promoter options for tag %s", tag)
			}

			if err := opt.Validate(); err != nil {
				return errors.Wrapf(err, "validate promoter options for tag %s", tag)
			}

			logrus.Infof("Growing manifests with images matching tag %s", tag)
			if err := reg.GrowManifest(ctx, &opt); err != nil {
				return errors.Wrapf(err, "Growing manifest with tag %s", tag)
			}
		}
	}

	// Re-write the image list without the mock images
	rawImageList, err := release.NewPromoterImageListFromFile(filepath.Join(repo.Dir(), imagesListPath))
	if err != nil {
		return errors.Wrap(err, "parsing the current manifest")
	}

	// Create a new imagelist to copy the non-mock images
	newImageList := &release.ImagePromoterImages{}

	// Copy all non mock-images:
	for _, imageData := range *rawImageList {
		if !strings.Contains(imageData.Name, "mock/") {
			*newImageList = append(*newImageList, imageData)
		}
	}

	// Write the modified manifest
	if err := newImageList.Write(filepath.Join(repo.Dir(), imagesListPath)); err != nil {
		return errors.Wrap(err, "while writing the promoter image list")
	}

	// Check if the image list was modified
	if len(oldlist) > 0 {
		logrus.Debug("Checking if the image list was modified")
		// read the newly modified manifest
		newlist, err := os.ReadFile(filepath.Join(repo.Dir(), imagesListPath))
		if err != nil {
			return errors.Wrap(err, "while reading the modified manifest images list")
		}

		// If the manifest was not modified, exit now
		if string(newlist) == string(oldlist) {
			logrus.Info("No changes detected in the promoter images list, exiting without changes")
			return nil
		}
	}

	// add the modified manifest to staging
	logrus.Debugf("Adding %s to staging area", imagesListPath)
	if err := repo.Add(imagesListPath); err != nil {
		return errors.Wrap(err, "adding image manifest to staging area")
	}

	commitMessage := "releng: Image promotion for " + opts.project + " " + strings.Join(opts.tags, " / ")

	// Commit files
	logrus.Debug("Creating commit")
	if err := repo.UserCommit(commitMessage); err != nil {
		return errors.Wrapf(err, "Error creating commit in %s/%s", git.DefaultGithubOrg, k8sioRepo)
	}

	// Push to fork
	if mustRun(opts, fmt.Sprintf("Push changes to user's fork at %s/%s?", userForkOrg, userForkRepo)) {
		logrus.Infof("Pushing manifest changes to %s/%s", userForkOrg, userForkRepo)
		if err := repo.PushToRemote(userForkName, branchname); err != nil {
			return errors.Wrapf(err, "pushing %s to %s/%s", userForkName, userForkOrg, userForkRepo)
		}
	} else {
		// Exit if no push was made

		logrus.Infof("Exiting without creating a PR since changes were not pushed to %s/%s", userForkOrg, userForkRepo)
		return nil
	}

	prBody := fmt.Sprintf("Image promotion for %s %s\n", opts.project, strings.Join(opts.tags, " / "))
	prBody += "This is an automated PR generated from `krel The Kubernetes Release Toolbox`\n\n"
	prBody += fmt.Sprintf("/hold\ncc: %s\n", opts.reviewers)

	// Create the Pull Request
	if mustRun(opts, "Create pull request?") {
		pr, err := gh.CreatePullRequest(
			git.DefaultGithubOrg, k8sioRepo, k8sioDefaultBranch,
			fmt.Sprintf("%s:%s", userForkOrg, branchname),
			commitMessage, prBody,
		)
		if err != nil {
			return errors.Wrap(err, "creating the pull request in k/k8s.io")
		}
		logrus.Infof(
			"Successfully created PR: %s%s/%s/pull/%d",
			github.GitHubURL, git.DefaultGithubOrg, k8sioRepo, pr.GetNumber(),
		)
	}

	// Success!
	return nil
}

// mustRun avoids running when a users chooses n in interactive mode
func mustRun(opts *promoteOptions, question string) bool {
	if !opts.interactiveMode {
		return true
	}
	_, success, err := util.Ask(fmt.Sprintf("%s (Y/n)", question), "y:Y:yes|n:N:no|y", 10)
	if err != nil {
		logrus.Error(err)
		if err.(util.UserInputError).IsCtrlC() {
			os.Exit(1)
		}
		return false
	}
	if success {
		return true
	}
	return false
}
<commit_msg>krel/promote-images: make an error message more verbose<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"k8s.io/release/pkg/release"
	reg "sigs.k8s.io/promo-tools/v3/legacy/dockerregistry"
	"sigs.k8s.io/release-sdk/git"
	"sigs.k8s.io/release-sdk/github"
	"sigs.k8s.io/release-utils/util"
)

const (
	k8sioRepo             = "k8s.io"
	k8sioDefaultBranch    = "main"
	promotionBranchSuffix = "-image-promotion"
)

// promoteCommand is the krel subcommand to promote conainer images
var imagePromoteCommand = &cobra.Command{
	Use:   "promote-images",
	Short: "Starts an image promotion for a tag of kubernetes or kubernetes-sigs images",
	Long: `krel promote

The 'promote' subcommand of krel updates the image promoter manifests
and creates a PR in kubernetes/k8s.io`,
	SilenceUsage:  true,
	SilenceErrors: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		// Run the PR creation function
		return runPromote(promoteOpts)
	},
}

type promoteOptions struct {
	project         string
	userFork        string
	tags            []string
	reviewers       string
	interactiveMode bool
}

func (o *promoteOptions) Validate() error {
	if len(o.tags) == 0 {
		return errors.New("cannot start promotion --tag is required")
	}
	if o.userFork == "" {
		return errors.New("cannot start promotion --fork is required")
	}

	// Check the fork slug
	if _, _, err := git.ParseRepoSlug(o.userFork); err != nil {
		return errors.Wrap(err, "checking user's fork")
	}

	// Verify we got a valid tag
	for _, tag := range o.tags {
		if _, err := util.TagStringToSemver(tag); err != nil {
			return errors.Wrapf(err, "verifying tag: %s", tag)
		}
	}

	// Check that the GitHub token is set
	token, isSet := os.LookupEnv(github.TokenEnvKey)
	if !isSet || token == "" {
		return fmt.Errorf("cannot promote images if GitHub token env var %s is not set", github.TokenEnvKey)
	}
	return nil
}

var promoteOpts = &promoteOptions{}

func init() {
	imagePromoteCommand.PersistentFlags().StringVar(
		&promoteOpts.project,
		"project",
		release.Kubernetes,
		"the name of the project to promote images for",
	)

	imagePromoteCommand.PersistentFlags().StringSliceVarP(
		&promoteOpts.tags,
		"tag",
		"t",
		[]string{},
		"version tag of the images we will promote",
	)

	imagePromoteCommand.PersistentFlags().StringVar(
		&promoteOpts.userFork,
		"fork",
		"",
		"the user's fork of kubernetes/k8s.io",
	)

	imagePromoteCommand.PersistentFlags().StringVar(
		&promoteOpts.reviewers,
		"reviewers",
		"@kubernetes/release-engineering",
		"the list of GitHub users or teams to assign to the PR",
	)

	imagePromoteCommand.PersistentFlags().BoolVarP(
		&promoteOpts.interactiveMode,
		"interactive",
		"i",
		false,
		"interactive mode, asks before every step",
	)

	for _, flagName := range []string{"tag", "fork"} {
		if err := imagePromoteCommand.MarkPersistentFlagRequired(flagName); err != nil {
			logrus.Error(errors.Wrapf(err, "marking tag %s as required", flagName))
		}
	}

	rootCmd.AddCommand(imagePromoteCommand)
}

func runPromote(opts *promoteOptions) error {
	// Check the cmd line opts
	if err := opts.Validate(); err != nil {
		return errors.Wrap(err, "checking command line options")
	}

	ctx := context.Background()

	// Validate options
	branchname := opts.project + "-" + opts.tags[0] + promotionBranchSuffix

	// Get the github org and repo from the fork slug
	userForkOrg, userForkRepo, err := git.ParseRepoSlug(opts.userFork)
	if err != nil {
		return errors.Wrap(err, "parsing user's fork")
	}
	if userForkRepo == "" {
		userForkRepo = k8sioRepo
	}

	// Check Environment
	gh := github.New()

	// Verify the repository is a fork of k8s.io
	if err = verifyFork(
		branchname, userForkOrg, userForkRepo, git.DefaultGithubOrg, k8sioRepo,
	); err != nil {
		return errors.Wrapf(err, "while checking fork of %s/%s ", git.DefaultGithubOrg, k8sioRepo)
	}

	// Clone k8s.io
	repo, err := prepareFork(branchname, git.DefaultGithubOrg, k8sioRepo, userForkOrg, userForkRepo)
	if err != nil {
		return errors.Wrap(err, "while preparing k/k8s.io fork")
	}

	defer func() {
		if mustRun(opts, "Clean fork directory?") {
			err = repo.Cleanup()
		} else {
			logrus.Infof("All modified files will be left untouched in %s", repo.Dir())
		}
	}()

	// Path to the promoter image list
	imagesListPath := filepath.Join(
		release.GCRIOPathProd,
		"images",
		filepath.Base(release.GCRIOPathStagingPrefix)+opts.project,
		"images.yaml",
	)

	// Read the current manifest to check later if new images come up
	oldlist := make([]byte, 0)

	// Run the promoter manifest grower
	if mustRun(opts, "Grow the manifests to add the new tags?") {
		if util.Exists(filepath.Join(repo.Dir(), imagesListPath)) {
			logrus.Debug("Reading the current image promoter manifest (image list)")
			oldlist, err = os.ReadFile(filepath.Join(repo.Dir(), imagesListPath))
			if err != nil {
				return errors.Wrap(err, "while reading the current promoter image list")
			}
		}

		for _, tag := range opts.tags {
			opt := reg.GrowManifestOptions{}
			if err := opt.Populate(
				filepath.Join(repo.Dir(), release.GCRIOPathProd),
				release.GCRIOPathStagingPrefix+opts.project, "", "", tag); err != nil {
				return errors.Wrapf(err, "populating image promoter options for tag %s", tag)
			}

			if err := opt.Validate(); err != nil {
				return errors.Wrapf(err, "validate promoter options for tag %s", tag)
			}

			logrus.Infof("Growing manifests with images matching tag %s", tag)
			if err := reg.GrowManifest(ctx, &opt); err != nil {
				return errors.Wrapf(err, "Growing manifest with tag %s", tag)
			}
		}
	}

	// Re-write the image list without the mock images
	rawImageList, err := release.NewPromoterImageListFromFile(filepath.Join(repo.Dir(), imagesListPath))
	if err != nil {
		return errors.Wrap(err, "parsing the current manifest")
	}

	// Create a new imagelist to copy the non-mock images
	newImageList := &release.ImagePromoterImages{}

	// Copy all non mock-images:
	for _, imageData := range *rawImageList {
		if !strings.Contains(imageData.Name, "mock/") {
			*newImageList = append(*newImageList, imageData)
		}
	}

	// Write the modified manifest
	if err := newImageList.Write(filepath.Join(repo.Dir(), imagesListPath)); err != nil {
		return errors.Wrap(err, "while writing the promoter image list")
	}

	// Check if the image list was modified
	if len(oldlist) > 0 {
		logrus.Debug("Checking if the image list was modified")
		// read the newly modified manifest
		newlist, err := os.ReadFile(filepath.Join(repo.Dir(), imagesListPath))
		if err != nil {
			return errors.Wrap(err, "while reading the modified manifest images list")
		}

		// If the manifest was not modified, exit now
		if string(newlist) == string(oldlist) {
			logrus.Info("No changes detected in the promoter images list, exiting without changes")
			return nil
		}
	}

	// add the modified manifest to staging
	logrus.Debugf("Adding %s to staging area", imagesListPath)
	if err := repo.Add(imagesListPath); err != nil {
		return errors.Wrap(err, "adding image manifest to staging area")
	}

	commitMessage := "releng: Image promotion for " + opts.project + " " + strings.Join(opts.tags, " / ")

	// Commit files
	logrus.Debug("Creating commit")
	if err := repo.UserCommit(commitMessage); err != nil {
		return errors.Wrapf(err, "Error creating commit in %s/%s", git.DefaultGithubOrg, k8sioRepo)
	}

	// Push to fork
	if mustRun(opts, fmt.Sprintf("Push changes to user's fork at %s/%s?", userForkOrg, userForkRepo)) {
		logrus.Infof("Pushing manifest changes to %s/%s", userForkOrg, userForkRepo)
		if err := repo.PushToRemote(userForkName, branchname); err != nil {
			return errors.Wrapf(err, "pushing %s to %s/%s", userForkName, userForkOrg, userForkRepo)
		}
	} else {
		// Exit if no push was made

		logrus.Infof("Exiting without creating a PR since changes were not pushed to %s/%s", userForkOrg, userForkRepo)
		return nil
	}

	prBody := fmt.Sprintf("Image promotion for %s %s\n", opts.project, strings.Join(opts.tags, " / "))
	prBody += "This is an automated PR generated from `krel The Kubernetes Release Toolbox`\n\n"
	prBody += fmt.Sprintf("/hold\ncc: %s\n", opts.reviewers)

	// Create the Pull Request
	if mustRun(opts, "Create pull request?") {
		pr, err := gh.CreatePullRequest(
			git.DefaultGithubOrg, k8sioRepo, k8sioDefaultBranch,
			fmt.Sprintf("%s:%s", userForkOrg, branchname),
			commitMessage, prBody,
		)
		if err != nil {
			return errors.Wrap(err, "creating the pull request in k/k8s.io")
		}
		logrus.Infof(
			"Successfully created PR: %s%s/%s/pull/%d",
			github.GitHubURL, git.DefaultGithubOrg, k8sioRepo, pr.GetNumber(),
		)
	}

	// Success!
	return nil
}

// mustRun avoids running when a users chooses n in interactive mode
func mustRun(opts *promoteOptions, question string) bool {
	if !opts.interactiveMode {
		return true
	}
	_, success, err := util.Ask(fmt.Sprintf("%s (Y/n)", question), "y:Y:yes|n:N:no|y", 10)
	if err != nil {
		logrus.Error(err)
		if err.(util.UserInputError).IsCtrlC() {
			os.Exit(1)
		}
		return false
	}
	if success {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright (c) 2016- Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package main

import (
	"fmt"
	"log"

	"github.com/docopt/docopt-go"
	"github.com/oragono/oragono/mkcerts"
	"github.com/oragono/oragono/web"
)

func main() {
	version := irc.SemVer
	usage := `oragono-web.
Usage:
	oragono-web mkcerts [--conf <filename>] [--quiet]
	oragono-web run [--conf <filename>] [--quiet]
	oragono-web -h | --help
	oragono-web --version
Options:
	--conf <filename>  Configuration file to use [default: web.yaml].
	--quiet            Don't show startup/shutdown lines.
	-h --help          Show this screen.
	--version          Show version.`

	arguments, _ := docopt.Parse(usage, nil, true, version, false)

	configfile := arguments["--conf"].(string)
	config, err := web.LoadConfig(configfile)
	if err != nil {
		log.Fatal("Config file did not load successfully:", err.Error())
	}

	if arguments["mkcerts"].(bool) {
		if !arguments["--quiet"].(bool) {
			log.Println("making self-signed certificates")
		}

		for name, conf := range config.TLSListenersConf {
			log.Printf(" making cert for %s listener\n", name)
			host := config.Host
			err := mkcerts.CreateCert("Oragono web interface", host, conf.Cert, conf.Key)
			if err == nil {
				if !arguments["--quiet"].(bool) {
					log.Printf("  Certificate created at %s : %s\n", conf.Cert, conf.Key)
				}
			} else {
				log.Fatal("  Could not create certificate:", err.Error())
			}
		}
	} else if arguments["run"].(bool) {
		irc.Log.SetLevel(config.Log)
		server := web.NewServer(config)
		if server == nil {
			log.Println("Could not load server")
			return
		}
		if !arguments["--quiet"].(bool) {
			log.Println(fmt.Sprintf("Oragono web interface v%s running", irc.SemVer))
			defer log.Println(irc.SemVer, "exiting")
		}
		server.Run()
	}
}
<commit_msg>Rip out the rest of the web interface<commit_after><|endoftext|>


<commit_before>// bridge bridges between IRC clients (RFC1459) and RobustIRC servers.
//
// Bridge instances are supposed to be long-running, and ideally as close to the
// IRC client as possible, e.g. on the same machine. When running on the same
// machine, there should not be any network problems between the IRC client and
// the bridge. Network problems between the bridge and a RobustIRC network are
// handled transparently.
package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net"
	"time"

	"github.com/robustirc/robustirc/cmd/robustirc-bridge/robustsession"

	"github.com/sorcix/irc"
)

var (
	network = flag.String("network",
		"",
		`DNS name to connect to (e.g. "robustirc.net"). The _robustirc._tcp SRV record must be present.`)

	listen = flag.String("listen",
		"localhost:6667",
		"host:port to listen on for IRC connections")

	socks = flag.String("socks", "", "host:port to listen on for SOCKS5 connections")
)

// TODO(secure): persistent state:
// - the last known server(s) in the network. added to *servers
// - for resuming sessions (later): the last seen message id, perhaps setup messages (JOINs, MODEs, …)
// for hosted mode, this state is stored per-nickname, ideally encrypted with password

type bridge struct {
	network string
}

func newBridge(network string) *bridge {
	return &bridge{
		network: network,
	}
}

type ircsession struct {
	Messages chan irc.Message
	Errors   chan error

	conn *irc.Conn
}

func newIrcsession(conn net.Conn) *ircsession {
	s := &ircsession{
		Messages: make(chan irc.Message),
		Errors:   make(chan error),
		conn:     irc.NewConn(conn),
	}
	go s.getMessages()
	return s
}

func (s *ircsession) Send(msg []byte) error {
	if _, err := s.conn.Write(msg); err != nil {
		return err
	}
	return nil
}

func (s *ircsession) Delete(killmsg string) error {
	defer s.conn.Close()

	if killmsg != "" {
		return s.conn.Encode(&irc.Message{
			Command:  "ERROR",
			Trailing: killmsg,
		})
	}

	return nil
}

func (s *ircsession) getMessages() {
	for {
		ircmsg, err := s.conn.Decode()
		if err != nil {
			s.Errors <- err
			return
		}
		log.Printf("<-irc: %q\n", ircmsg.Bytes())
		s.Messages <- *ircmsg
	}
}

func (p *bridge) handleIRC(conn net.Conn) {
	var quitmsg, killmsg string
	var waitingForPingReply bool

	ircSession := newIrcsession(conn)

	defer func() {
		if err := ircSession.Delete(killmsg); err != nil {
			log.Printf("Could not properly delete IRC session: %v\n", err)
		}
	}()

	robustSession, err := robustsession.Create(p.network)
	if err != nil {
		killmsg = fmt.Sprintf("Could not create RobustIRC session: %v", err)
		return
	}

	defer func() {
		log.Printf("deleting robustsession…\n")
		if err := robustSession.Delete(quitmsg); err != nil {
			log.Printf("Could not properly delete RobustIRC session: %v\n", err)
		}
	}()

	var sendIRC, sendRobust []byte

	keepaliveToNetwork := time.After(1 * time.Minute)
	keepaliveToClient := time.After(1 * time.Minute)
	for {
		// These two variables contain the messages to be sent to IRC/RobustIRC
		// from the previous iteration of the state machine. That way, there is
		// only one place where the error handling happens.
		if sendIRC != nil {
			if err := ircSession.Send(sendIRC); err != nil {
				quitmsg = fmt.Sprintf("Bridge: Send to IRC client: %v", err)
				return
			}
			sendIRC = nil
		}
		if sendRobust != nil {
			if err := robustSession.PostMessage(string(sendRobust)); err != nil {
				killmsg = fmt.Sprintf("Could not post message to RobustIRC: %v", err)
				return
			}
			keepaliveToNetwork = time.After(1 * time.Minute)
			keepaliveToClient = time.After(1 * time.Minute)
			sendRobust = nil
		}

		select {
		case msg := <-robustSession.Messages:
			ircmsg := irc.ParseMessage(msg)
			if ircmsg.Command == irc.PONG && len(ircmsg.Params) > 0 && ircmsg.Params[0] == "keepalive" {
				log.Printf("Swallowing keepalive PONG from server to avoid confusing the IRC client.\n")
				break
			}
			sendIRC = []byte(msg)

		case err := <-robustSession.Errors:
			killmsg = fmt.Sprintf("RobustIRC session error: %v", err)
			return

		case ircmsg := <-ircSession.Messages:
			switch ircmsg.Command {
			case irc.PONG:
				waitingForPingReply = false

			case irc.PING:
				sendIRC = (&irc.Message{
					Prefix:  robustSession.IrcPrefix,
					Command: irc.PONG,
					Params:  []string{ircmsg.Params[0]},
				}).Bytes()

			case irc.QUIT:
				quitmsg = ircmsg.Trailing
				return

			default:
				sendRobust = ircmsg.Bytes()
			}

		case err := <-ircSession.Errors:
			quitmsg = fmt.Sprintf("Bridge: Read from IRC client: %v", err)
			return

		case <-keepaliveToClient:
			// After no traffic in either direction for 1 minute, we send a PING
			// message. If a PING message was already sent, this means that we did
			// not receive a PONG message, so we close the connection with a
			// timeout.
			if waitingForPingReply {
				quitmsg = "Bridge: ping timeout"
				return
			}
			sendIRC = (&irc.Message{
				Prefix:  robustSession.IrcPrefix,
				Command: irc.PING,
				Params:  []string{"robustirc.bridge"},
			}).Bytes()
			waitingForPingReply = true

		case <-keepaliveToNetwork:
			sendRobust = []byte("PING keepalive")
			keepaliveToNetwork = time.After(1 * time.Minute)
		}
	}
}

func main() {
	flag.Parse()

	rand.Seed(time.Now().Unix())

	if *network == "" && *socks == "" {
		log.Fatal("You must specify either -network or -socks.")
	}

	// SOCKS and IRC
	if *socks != "" && *network != "" {
		go func() {
			if err := listenAndServeSocks(*socks); err != nil {
				log.Fatal(err)
			}
		}()
	}

	// SOCKS only
	if *socks != "" && *network == "" {
		log.Fatal(listenAndServeSocks(*socks))
	}

	// IRC
	if *network != "" {
		p := newBridge(*network)

		ln, err := net.Listen("tcp", *listen)
		if err != nil {
			log.Fatal(err)
		}

		log.Printf("RobustIRC IRC bridge listening on %q\n", *listen)

		for {
			conn, err := ln.Accept()
			if err != nil {
				log.Printf("Could not accept IRC client connection: %v\n", err)
				continue
			}
			go p.handleIRC(conn)
		}
	}
}
<commit_msg>bridge: skip empty lines<commit_after>// bridge bridges between IRC clients (RFC1459) and RobustIRC servers.
//
// Bridge instances are supposed to be long-running, and ideally as close to the
// IRC client as possible, e.g. on the same machine. When running on the same
// machine, there should not be any network problems between the IRC client and
// the bridge. Network problems between the bridge and a RobustIRC network are
// handled transparently.
package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net"
	"time"

	"github.com/robustirc/robustirc/cmd/robustirc-bridge/robustsession"

	"github.com/sorcix/irc"
)

var (
	network = flag.String("network",
		"",
		`DNS name to connect to (e.g. "robustirc.net"). The _robustirc._tcp SRV record must be present.`)

	listen = flag.String("listen",
		"localhost:6667",
		"host:port to listen on for IRC connections")

	socks = flag.String("socks", "", "host:port to listen on for SOCKS5 connections")
)

// TODO(secure): persistent state:
// - the last known server(s) in the network. added to *servers
// - for resuming sessions (later): the last seen message id, perhaps setup messages (JOINs, MODEs, …)
// for hosted mode, this state is stored per-nickname, ideally encrypted with password

type bridge struct {
	network string
}

func newBridge(network string) *bridge {
	return &bridge{
		network: network,
	}
}

type ircsession struct {
	Messages chan irc.Message
	Errors   chan error

	conn *irc.Conn
}

func newIrcsession(conn net.Conn) *ircsession {
	s := &ircsession{
		Messages: make(chan irc.Message),
		Errors:   make(chan error),
		conn:     irc.NewConn(conn),
	}
	go s.getMessages()
	return s
}

func (s *ircsession) Send(msg []byte) error {
	if _, err := s.conn.Write(msg); err != nil {
		return err
	}
	return nil
}

func (s *ircsession) Delete(killmsg string) error {
	defer s.conn.Close()

	if killmsg != "" {
		return s.conn.Encode(&irc.Message{
			Command:  "ERROR",
			Trailing: killmsg,
		})
	}

	return nil
}

func (s *ircsession) getMessages() {
	for {
		ircmsg, err := s.conn.Decode()
		if err != nil {
			s.Errors <- err
			return
		}
		// Skip empty lines (to prevent nil pointer dereferences).
		if ircmsg == nil {
			continue
		}
		log.Printf("<-irc: %q\n", ircmsg.Bytes())
		s.Messages <- *ircmsg
	}
}

func (p *bridge) handleIRC(conn net.Conn) {
	var quitmsg, killmsg string
	var waitingForPingReply bool

	ircSession := newIrcsession(conn)

	defer func() {
		if err := ircSession.Delete(killmsg); err != nil {
			log.Printf("Could not properly delete IRC session: %v\n", err)
		}
	}()

	robustSession, err := robustsession.Create(p.network)
	if err != nil {
		killmsg = fmt.Sprintf("Could not create RobustIRC session: %v", err)
		return
	}

	defer func() {
		log.Printf("deleting robustsession…\n")
		if err := robustSession.Delete(quitmsg); err != nil {
			log.Printf("Could not properly delete RobustIRC session: %v\n", err)
		}
	}()

	var sendIRC, sendRobust []byte

	keepaliveToNetwork := time.After(1 * time.Minute)
	keepaliveToClient := time.After(1 * time.Minute)
	for {
		// These two variables contain the messages to be sent to IRC/RobustIRC
		// from the previous iteration of the state machine. That way, there is
		// only one place where the error handling happens.
		if sendIRC != nil {
			if err := ircSession.Send(sendIRC); err != nil {
				quitmsg = fmt.Sprintf("Bridge: Send to IRC client: %v", err)
				return
			}
			sendIRC = nil
		}
		if sendRobust != nil {
			if err := robustSession.PostMessage(string(sendRobust)); err != nil {
				killmsg = fmt.Sprintf("Could not post message to RobustIRC: %v", err)
				return
			}
			keepaliveToNetwork = time.After(1 * time.Minute)
			keepaliveToClient = time.After(1 * time.Minute)
			sendRobust = nil
		}

		select {
		case msg := <-robustSession.Messages:
			ircmsg := irc.ParseMessage(msg)
			if ircmsg.Command == irc.PONG && len(ircmsg.Params) > 0 && ircmsg.Params[0] == "keepalive" {
				log.Printf("Swallowing keepalive PONG from server to avoid confusing the IRC client.\n")
				break
			}
			sendIRC = []byte(msg)

		case err := <-robustSession.Errors:
			killmsg = fmt.Sprintf("RobustIRC session error: %v", err)
			return

		case ircmsg := <-ircSession.Messages:
			switch ircmsg.Command {
			case irc.PONG:
				waitingForPingReply = false

			case irc.PING:
				sendIRC = (&irc.Message{
					Prefix:  robustSession.IrcPrefix,
					Command: irc.PONG,
					Params:  []string{ircmsg.Params[0]},
				}).Bytes()

			case irc.QUIT:
				quitmsg = ircmsg.Trailing
				return

			default:
				sendRobust = ircmsg.Bytes()
			}

		case err := <-ircSession.Errors:
			quitmsg = fmt.Sprintf("Bridge: Read from IRC client: %v", err)
			return

		case <-keepaliveToClient:
			// After no traffic in either direction for 1 minute, we send a PING
			// message. If a PING message was already sent, this means that we did
			// not receive a PONG message, so we close the connection with a
			// timeout.
			if waitingForPingReply {
				quitmsg = "Bridge: ping timeout"
				return
			}
			sendIRC = (&irc.Message{
				Prefix:  robustSession.IrcPrefix,
				Command: irc.PING,
				Params:  []string{"robustirc.bridge"},
			}).Bytes()
			waitingForPingReply = true

		case <-keepaliveToNetwork:
			sendRobust = []byte("PING keepalive")
			keepaliveToNetwork = time.After(1 * time.Minute)
		}
	}
}

func main() {
	flag.Parse()

	rand.Seed(time.Now().Unix())

	if *network == "" && *socks == "" {
		log.Fatal("You must specify either -network or -socks.")
	}

	// SOCKS and IRC
	if *socks != "" && *network != "" {
		go func() {
			if err := listenAndServeSocks(*socks); err != nil {
				log.Fatal(err)
			}
		}()
	}

	// SOCKS only
	if *socks != "" && *network == "" {
		log.Fatal(listenAndServeSocks(*socks))
	}

	// IRC
	if *network != "" {
		p := newBridge(*network)

		ln, err := net.Listen("tcp", *listen)
		if err != nil {
			log.Fatal(err)
		}

		log.Printf("RobustIRC IRC bridge listening on %q\n", *listen)

		for {
			conn, err := ln.Accept()
			if err != nil {
				log.Printf("Could not accept IRC client connection: %v\n", err)
				continue
			}
			go p.handleIRC(conn)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path"
	"reflect"
	"sync"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	size    = flag.Int("size", 5, "board size")
	zero    = flag.Bool("zero", false, "start with zero weights, not defaults")
	w1      = flag.String("w1", "", "first set of weights")
	w2      = flag.String("w2", "", "second set of weights")
	c1      = flag.String("c1", "", "custom config 1")
	c2      = flag.String("c2", "", "custom config 2")
	perturb = flag.Float64("perturb", 0.0, "perturb weights")
	seed    = flag.Int64("seed", 1, "starting seed")
	games   = flag.Int("games", 10, "number of games")
	cutoff  = flag.Int("cutoff", 81, "cut games off after how many plies")

	depth = flag.Int("depth", 3, "depth to search")
	limit = flag.Duration("limit", 0, "search duration")

	verbose = flag.Bool("verbose", false, "log results per game")

	threads = flag.Int("threads", 4, "number of parallel threads")

	out = flag.String("out", "", "directory to write ptns to")
)

type gameSpec struct {
	i            int
	white, black ai.TakPlayer
	p1color      tak.Color
}

type gameResult struct {
	spec gameSpec
	p    *tak.Position
	ms   []tak.Move
}

func main() {
	flag.Parse()

	weights1 := ai.DefaultWeights
	weights2 := ai.DefaultWeights
	if *zero {
		weights1 = ai.Weights{}
		weights2 = ai.Weights{}
	}
	if *w1 != "" {
		if err := json.Unmarshal([]byte(*w1), &weights1); err != nil {
			log.Fatal("w1:", err)
		}
	}
	if *w2 != "" {
		if err := json.Unmarshal([]byte(*w2), &weights2); err != nil {
			log.Fatal("w2:", err)
		}
	}

	cfg1 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	cfg2 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	if *c1 != "" {
		if err := json.Unmarshal([]byte(*c1), &cfg1); err != nil {
			log.Fatal("c1:", err)
		}
	}
	if *c2 != "" {
		if err := json.Unmarshal([]byte(*c2), &cfg2); err != nil {
			log.Fatal("c2:", err)
		}
	}

	var stats [2]struct {
		wins     int
		flatWins int
		roadWins int
	}
	var white, black int
	var ties int

	rc := make(chan gameResult)

	go runGames(cfg1, cfg2, weights1, weights2, *seed, rc)
	for r := range rc {
		d := r.p.WinDetails()
		if *verbose {
			log.Printf("game n=%d plies=%d p1=%s winner=%s wf=%d bf=%d ws=%d bs=%d",
				r.spec.i, r.p.MoveNumber(),
				r.spec.p1color, d.Winner,
				d.WhiteFlats,
				d.BlackFlats,
				r.p.WhiteStones(),
				r.p.BlackStones(),
			)
		}
		if d.Over {
			if d.Winner == tak.White {
				white++
			} else {
				black++
			}
			st := &stats[0]
			if d.Winner == r.spec.p1color.Flip() {
				st = &stats[1]
			}
			st.wins++
			switch d.Reason {
			case tak.FlatsWin:
				st.flatWins++
			case tak.RoadWin:
				st.roadWins++
			}
		}
		if d.Winner == tak.NoColor {
			ties++
		}
		if *out != "" {
			writeGame(*out, &r)
		}
	}

	j, _ := json.Marshal(&weights1)
	log.Printf("p1=%s", j)
	j, _ = json.Marshal(&weights2)
	log.Printf("p2=%s", j)
	log.Printf("done games=%d seed=%d ties=%d p1.wins=%d (%d road/%d flat) p2.wins=%d (%d road/%d flat)",
		*games, *seed, ties,
		stats[0].wins, stats[0].roadWins, stats[0].flatWins,
		stats[1].wins, stats[1].roadWins, stats[1].flatWins)
	a, b := int64(stats[0].wins), int64(stats[1].wins)
	if a < b {
		a, b = b, a
	}
	log.Printf("white=%d black=%d (%.2f)",
		white, black, float64(white)/float64(white+black))
	log.Printf("p[one-sided]=%f", binomTest(a, b, 0.5))
}

func writeGame(d string, r *gameResult) {
	os.MkdirAll(d, 0755)
	p := &ptn.PTN{}
	p.Tags = []ptn.Tag{
		{"Size", fmt.Sprintf("%d", r.p.Size())},
		{"Player1", r.spec.p1color.String()},
	}
	for i, m := range r.ms {
		if i%2 == 0 {
			p.Ops = append(p.Ops, &ptn.MoveNumber{Number: i/2 + 1})
		}
		p.Ops = append(p.Ops, &ptn.Move{Move: m})
	}
	ptnPath := path.Join(d, fmt.Sprintf("%d.ptn", r.spec.i))
	ioutil.WriteFile(ptnPath, []byte(p.Render()), 0644)
}

func worker(games <-chan gameSpec, out chan<- gameResult) {
	for g := range games {
		var ms []tak.Move
		p := tak.New(tak.Config{Size: *size})
		for i := 0; i < *cutoff; i++ {
			var m tak.Move
			if p.ToMove() == tak.White {
				m = g.white.GetMove(p, *limit)
			} else {
				m = g.black.GetMove(p, *limit)
			}
			p, _ = p.Move(&m)
			ms = append(ms, m)
			if ok, _ := p.GameOver(); ok {
				break
			}
		}
		out <- gameResult{
			spec: g,
			p:    p,
			ms:   ms,
		}
	}
}

func perturbWeights(p float64, w ai.Weights) ai.Weights {
	r := reflect.Indirect(reflect.ValueOf(&w))
	typ := r.Type()
	for i := 0; i < typ.NumField(); i++ {
		f := typ.Field(i)
		if f.Type.Kind() != reflect.Int {
			continue
		}
		v := r.Field(i).Interface().(int)
		adj := rand.NormFloat64() * p
		v = int(float64(v) * (1 + adj))
		r.Field(i).SetInt(int64(v))
	}

	j, _ := json.Marshal(&w)
	log.Printf("perturb: %s", j)

	return w
}

func runGames(cfg1, cfg2 ai.MinimaxConfig,
	w1, w2 ai.Weights, seed int64, rc chan<- gameResult) {
	gc := make(chan gameSpec)
	var wg sync.WaitGroup
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go func() {
			worker(gc, rc)
			wg.Done()
		}()
	}
	r := rand.New(rand.NewSource(seed))
	for g := 0; g < *games; g++ {
		var white, black ai.TakPlayer
		w1 := w1
		w2 := w2
		if *perturb != 0.0 {
			w1 = perturbWeights(*perturb, w1)
			w2 = perturbWeights(*perturb, w2)
		}
		cfg1 := cfg1
		cfg1.Evaluate = ai.MakeEvaluator(&w1)
		cfg1.Seed = r.Int63()

		cfg2 := cfg2
		cfg2.Evaluate = ai.MakeEvaluator(&w2)
		cfg2.Seed = r.Int63()

		p1 := ai.NewMinimax(cfg1)
		p2 := ai.NewMinimax(cfg2)
		seed++
		var p1color tak.Color
		if g%2 == 0 {
			white, black = p1, p2
			p1color = tak.White
		} else {
			black, white = p1, p2
			p1color = tak.Black
		}

		spec := gameSpec{
			i:       g,
			white:   white,
			black:   black,
			p1color: p1color,
		}
		gc <- spec
	}
	close(gc)
	wg.Wait()
	close(rc)
}
<commit_msg>taktician-evaluate: show configs<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path"
	"reflect"
	"sync"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	size    = flag.Int("size", 5, "board size")
	zero    = flag.Bool("zero", false, "start with zero weights, not defaults")
	w1      = flag.String("w1", "", "first set of weights")
	w2      = flag.String("w2", "", "second set of weights")
	c1      = flag.String("c1", "", "custom config 1")
	c2      = flag.String("c2", "", "custom config 2")
	perturb = flag.Float64("perturb", 0.0, "perturb weights")
	seed    = flag.Int64("seed", 1, "starting seed")
	games   = flag.Int("games", 10, "number of games")
	cutoff  = flag.Int("cutoff", 81, "cut games off after how many plies")

	depth = flag.Int("depth", 3, "depth to search")
	limit = flag.Duration("limit", 0, "search duration")

	verbose = flag.Bool("verbose", false, "log results per game")

	threads = flag.Int("threads", 4, "number of parallel threads")

	out = flag.String("out", "", "directory to write ptns to")
)

type gameSpec struct {
	i            int
	white, black ai.TakPlayer
	p1color      tak.Color
}

type gameResult struct {
	spec gameSpec
	p    *tak.Position
	ms   []tak.Move
}

func main() {
	flag.Parse()

	weights1 := ai.DefaultWeights
	weights2 := ai.DefaultWeights
	if *zero {
		weights1 = ai.Weights{}
		weights2 = ai.Weights{}
	}
	if *w1 != "" {
		if err := json.Unmarshal([]byte(*w1), &weights1); err != nil {
			log.Fatal("w1:", err)
		}
	}
	if *w2 != "" {
		if err := json.Unmarshal([]byte(*w2), &weights2); err != nil {
			log.Fatal("w2:", err)
		}
	}

	cfg1 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	cfg2 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	if *c1 != "" {
		if err := json.Unmarshal([]byte(*c1), &cfg1); err != nil {
			log.Fatal("c1:", err)
		}
	}
	if *c2 != "" {
		if err := json.Unmarshal([]byte(*c2), &cfg2); err != nil {
			log.Fatal("c2:", err)
		}
	}

	var stats [2]struct {
		wins     int
		flatWins int
		roadWins int
	}
	var white, black int
	var ties int

	rc := make(chan gameResult)

	go runGames(cfg1, cfg2, weights1, weights2, *seed, rc)
	for r := range rc {
		d := r.p.WinDetails()
		if *verbose {
			log.Printf("game n=%d plies=%d p1=%s winner=%s wf=%d bf=%d ws=%d bs=%d",
				r.spec.i, r.p.MoveNumber(),
				r.spec.p1color, d.Winner,
				d.WhiteFlats,
				d.BlackFlats,
				r.p.WhiteStones(),
				r.p.BlackStones(),
			)
		}
		if d.Over {
			if d.Winner == tak.White {
				white++
			} else {
				black++
			}
			st := &stats[0]
			if d.Winner == r.spec.p1color.Flip() {
				st = &stats[1]
			}
			st.wins++
			switch d.Reason {
			case tak.FlatsWin:
				st.flatWins++
			case tak.RoadWin:
				st.roadWins++
			}
		}
		if d.Winner == tak.NoColor {
			ties++
		}
		if *out != "" {
			writeGame(*out, &r)
		}
	}

	var j []byte
	j, _ = json.Marshal(&weights1)
	log.Printf("p1w=%s", j)
	j, _ = json.Marshal(&cfg1)
	log.Printf("p1c=%s", j)
	j, _ = json.Marshal(&weights2)
	log.Printf("p2=%s", j)
	j, _ = json.Marshal(&cfg2)
	log.Printf("p2c=%s", j)
	log.Printf("done games=%d seed=%d ties=%d p1.wins=%d (%d road/%d flat) p2.wins=%d (%d road/%d flat)",
		*games, *seed, ties,
		stats[0].wins, stats[0].roadWins, stats[0].flatWins,
		stats[1].wins, stats[1].roadWins, stats[1].flatWins)
	a, b := int64(stats[0].wins), int64(stats[1].wins)
	if a < b {
		a, b = b, a
	}
	log.Printf("white=%d black=%d (%.2f)",
		white, black, float64(white)/float64(white+black))
	log.Printf("p[one-sided]=%f", binomTest(a, b, 0.5))
}

func writeGame(d string, r *gameResult) {
	os.MkdirAll(d, 0755)
	p := &ptn.PTN{}
	p.Tags = []ptn.Tag{
		{"Size", fmt.Sprintf("%d", r.p.Size())},
		{"Player1", r.spec.p1color.String()},
	}
	for i, m := range r.ms {
		if i%2 == 0 {
			p.Ops = append(p.Ops, &ptn.MoveNumber{Number: i/2 + 1})
		}
		p.Ops = append(p.Ops, &ptn.Move{Move: m})
	}
	ptnPath := path.Join(d, fmt.Sprintf("%d.ptn", r.spec.i))
	ioutil.WriteFile(ptnPath, []byte(p.Render()), 0644)
}

func worker(games <-chan gameSpec, out chan<- gameResult) {
	for g := range games {
		var ms []tak.Move
		p := tak.New(tak.Config{Size: *size})
		for i := 0; i < *cutoff; i++ {
			var m tak.Move
			if p.ToMove() == tak.White {
				m = g.white.GetMove(p, *limit)
			} else {
				m = g.black.GetMove(p, *limit)
			}
			p, _ = p.Move(&m)
			ms = append(ms, m)
			if ok, _ := p.GameOver(); ok {
				break
			}
		}
		out <- gameResult{
			spec: g,
			p:    p,
			ms:   ms,
		}
	}
}

func perturbWeights(p float64, w ai.Weights) ai.Weights {
	r := reflect.Indirect(reflect.ValueOf(&w))
	typ := r.Type()
	for i := 0; i < typ.NumField(); i++ {
		f := typ.Field(i)
		if f.Type.Kind() != reflect.Int {
			continue
		}
		v := r.Field(i).Interface().(int)
		adj := rand.NormFloat64() * p
		v = int(float64(v) * (1 + adj))
		r.Field(i).SetInt(int64(v))
	}

	j, _ := json.Marshal(&w)
	log.Printf("perturb: %s", j)

	return w
}

func runGames(cfg1, cfg2 ai.MinimaxConfig,
	w1, w2 ai.Weights, seed int64, rc chan<- gameResult) {
	gc := make(chan gameSpec)
	var wg sync.WaitGroup
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go func() {
			worker(gc, rc)
			wg.Done()
		}()
	}
	r := rand.New(rand.NewSource(seed))
	for g := 0; g < *games; g++ {
		var white, black ai.TakPlayer
		w1 := w1
		w2 := w2
		if *perturb != 0.0 {
			w1 = perturbWeights(*perturb, w1)
			w2 = perturbWeights(*perturb, w2)
		}
		cfg1 := cfg1
		cfg1.Evaluate = ai.MakeEvaluator(&w1)
		cfg1.Seed = r.Int63()

		cfg2 := cfg2
		cfg2.Evaluate = ai.MakeEvaluator(&w2)
		cfg2.Seed = r.Int63()

		p1 := ai.NewMinimax(cfg1)
		p2 := ai.NewMinimax(cfg2)
		seed++
		var p1color tak.Color
		if g%2 == 0 {
			white, black = p1, p2
			p1color = tak.White
		} else {
			black, white = p1, p2
			p1color = tak.Black
		}

		spec := gameSpec{
			i:       g,
			white:   white,
			black:   black,
			p1color: p1color,
		}
		gc <- spec
	}
	close(gc)
	wg.Wait()
	close(rc)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package partials

import (
	"fmt"
	"html/template"
	"strings"
	"sync"
	texttemplate "text/template"

	bp "github.com/gohugoio/hugo/bufferpool"
	"github.com/gohugoio/hugo/deps"
)

// TestTemplateProvider is global deps.ResourceProvider.
// NOTE: It's currently unused.
var TestTemplateProvider deps.ResourceProvider

// partialCache represents a cache of partials protected by a mutex.
type partialCache struct {
	sync.RWMutex
	p map[string]interface{}
}

// New returns a new instance of the templates-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
	return &Namespace{
		deps:           deps,
		cachedPartials: partialCache{p: make(map[string]interface{})},
	}
}

// Namespace provides template functions for the "templates" namespace.
type Namespace struct {
	deps           *deps.Deps
	cachedPartials partialCache
}

// Include executes the named partial and returns either a string,
// when the partial is a text/template, or template.HTML when html/template.
func (ns *Namespace) Include(name string, contextList ...interface{}) (interface{}, error) {
	if strings.HasPrefix("partials/", name) {
		name = name[8:]
	}
	var context interface{}

	if len(contextList) == 0 {
		context = nil
	} else {
		context = contextList[0]
	}

	for _, n := range []string{"partials/" + name, "theme/partials/" + name} {
		templ := ns.deps.Tmpl.Lookup(n)
		if templ == nil {
			// For legacy reasons.
			templ = ns.deps.Tmpl.Lookup(n + ".html")
		}
		if templ != nil {
			b := bp.GetBuffer()
			defer bp.PutBuffer(b)

			if err := templ.Execute(b, context); err != nil {
				return "", err
			}

			if _, ok := templ.Template.(*texttemplate.Template); ok {
				s := b.String()
				if ns.deps.Metrics != nil {
					ns.deps.Metrics.TrackValue(n, s)
				}
				return s, nil
			}

			s := b.String()
			if ns.deps.Metrics != nil {
				ns.deps.Metrics.TrackValue(n, s)
			}
			return template.HTML(s), nil

		}
	}

	return "", fmt.Errorf("Partial %q not found", name)
}

// IncludeCached executes and caches partial templates.  An optional variant
// string parameter (a string slice actually, but be only use a variadic
// argument to make it optional) can be passed so that a given partial can have
// multiple uses. The cache is created with name+variant as the key.
func (ns *Namespace) IncludeCached(name string, context interface{}, variant ...string) (interface{}, error) {
	key := name
	if len(variant) > 0 {
		for i := 0; i < len(variant); i++ {
			key += variant[i]
		}
	}
	return ns.getOrCreate(key, name, context)
}

func (ns *Namespace) getOrCreate(key, name string, context interface{}) (p interface{}, err error) {
	var ok bool

	ns.cachedPartials.RLock()
	p, ok = ns.cachedPartials.p[key]
	ns.cachedPartials.RUnlock()

	if ok {
		return
	}

	ns.cachedPartials.Lock()
	if p, ok = ns.cachedPartials.p[key]; !ok {
		ns.cachedPartials.Unlock()
		p, err = ns.Include(name, context)

		ns.cachedPartials.Lock()
		ns.cachedPartials.p[key] = p

	}
	ns.cachedPartials.Unlock()

	return
}
<commit_msg>tpl/partials: Fix cache locking<commit_after>// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package partials

import (
	"fmt"
	"html/template"
	"strings"
	"sync"
	texttemplate "text/template"

	bp "github.com/gohugoio/hugo/bufferpool"
	"github.com/gohugoio/hugo/deps"
)

// TestTemplateProvider is global deps.ResourceProvider.
// NOTE: It's currently unused.
var TestTemplateProvider deps.ResourceProvider

// partialCache represents a cache of partials protected by a mutex.
type partialCache struct {
	sync.RWMutex
	p map[string]interface{}
}

// New returns a new instance of the templates-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
	return &Namespace{
		deps:           deps,
		cachedPartials: partialCache{p: make(map[string]interface{})},
	}
}

// Namespace provides template functions for the "templates" namespace.
type Namespace struct {
	deps           *deps.Deps
	cachedPartials partialCache
}

// Include executes the named partial and returns either a string,
// when the partial is a text/template, or template.HTML when html/template.
func (ns *Namespace) Include(name string, contextList ...interface{}) (interface{}, error) {
	if strings.HasPrefix("partials/", name) {
		name = name[8:]
	}
	var context interface{}

	if len(contextList) == 0 {
		context = nil
	} else {
		context = contextList[0]
	}

	for _, n := range []string{"partials/" + name, "theme/partials/" + name} {
		templ := ns.deps.Tmpl.Lookup(n)
		if templ == nil {
			// For legacy reasons.
			templ = ns.deps.Tmpl.Lookup(n + ".html")
		}
		if templ != nil {
			b := bp.GetBuffer()
			defer bp.PutBuffer(b)

			if err := templ.Execute(b, context); err != nil {
				return "", err
			}

			if _, ok := templ.Template.(*texttemplate.Template); ok {
				s := b.String()
				if ns.deps.Metrics != nil {
					ns.deps.Metrics.TrackValue(n, s)
				}
				return s, nil
			}

			s := b.String()
			if ns.deps.Metrics != nil {
				ns.deps.Metrics.TrackValue(n, s)
			}
			return template.HTML(s), nil

		}
	}

	return "", fmt.Errorf("Partial %q not found", name)
}

// IncludeCached executes and caches partial templates.  An optional variant
// string parameter (a string slice actually, but be only use a variadic
// argument to make it optional) can be passed so that a given partial can have
// multiple uses. The cache is created with name+variant as the key.
func (ns *Namespace) IncludeCached(name string, context interface{}, variant ...string) (interface{}, error) {
	key := name
	if len(variant) > 0 {
		for i := 0; i < len(variant); i++ {
			key += variant[i]
		}
	}
	return ns.getOrCreate(key, name, context)
}

func (ns *Namespace) getOrCreate(key, name string, context interface{}) (interface{}, error) {

	ns.cachedPartials.RLock()
	p, ok := ns.cachedPartials.p[key]
	ns.cachedPartials.RUnlock()

	if ok {
		return p, nil
	}

	ns.cachedPartials.Lock()
	defer ns.cachedPartials.Unlock()

	// Double-check.
	if p, ok = ns.cachedPartials.p[key]; ok {
		return p, nil
	}

	p, err := ns.Include(name, context)
	if err != nil {
		return nil, err
	}

	ns.cachedPartials.p[key] = p

	return p, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/url"
	"os"
	"strconv"
	"time"

	mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount"
	"github.com/urfave/cli"
)

// Set up custom help text for gcsfuse; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} [bucket] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func newApp() (app *cli.App) {
	dirModeValue := new(OctalInt)
	*dirModeValue = 0755

	fileModeValue := new(OctalInt)
	*fileModeValue = 0644

	app = &cli.App{
		Name:    "gcsfuse",
		Version: getVersion(),
		Usage:   "Mount a specified GCS bucket or all accessible buckets locally",
		Writer:  os.Stderr,
		Flags: []cli.Flag{

			cli.StringFlag{
				Name:  "app-name",
				Value: "",
				Usage: "The application name of this mount.",
			},

			cli.BoolFlag{
				Name:  "foreground",
				Usage: "Stay in the foreground after mounting.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.GenericFlag{
				Name:  "dir-mode",
				Value: dirModeValue,
				Usage: "Permissions bits for directories, in octal.",
			},

			cli.GenericFlag{
				Name:  "file-mode",
				Value: fileModeValue,
				Usage: "Permission bits for files, in octal.",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: -1,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: -1,
				Usage: "GID owner of all inodes.",
			},

			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See " +
					"docs/semantics.md",
			},

			cli.StringFlag{
				Name:  "only-dir",
				Usage: "Mount only the given directory, relative to the bucket root.",
			},

			cli.IntFlag{
				Name:  "rename-dir-limit",
				Value: 0,
				Usage: "Allow rename a directory containing fewer descendants " +
					"than this limit.",
			},

			/////////////////////////
			// GCS
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "https://storage.googleapis.com:443",
				Usage: "The endpoint to connect to.",
			},

			cli.StringFlag{
				Name:  "billing-project",
				Value: "",
				Usage: "Project to use for billing when accessing requester pays buckets. " +
					"(default: none)",
			},

			cli.StringFlag{
				Name:  "key-file",
				Value: "",
				Usage: "Absolute path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},

			cli.StringFlag{
				Name:  "token-url",
				Value: "",
				Usage: "An url for getting an access token when key-file is absent.",
			},

			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},

			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: -1,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "max-retry-sleep",
				Value: time.Minute,
				Usage: "The maximum duration allowed to sleep in a retry loop with " +
					"exponential backoff for failed requests to GCS backend. Once the " +
					"backoff duration exceeds this limit, the retry stops. The default " +
					"is 1 minute. A value of 0 disables retries.",
			},

			cli.IntFlag{
				Name:  "stat-cache-capacity",
				Value: 4096,
				Usage: "How many entries can the stat cache hold (impacts memory consumption)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			cli.BoolFlag{
				Name:  "local-file-cache",
				Usage: "Cache GCS files on local disk for reads.",
			},

			cli.StringFlag{
				Name:  "temp-dir",
				Value: "",
				Usage: "Absolute path to temporary directory for local GCS object " +
					"copies. (default: system default, likely /tmp)",
			},

			cli.BoolFlag{
				Name: "disable-http2",
				Usage: "Once set, the protocol used for communicating with " +
					"GCS backend would be HTTP/1.1, instead of the default HTTP/2.",
			},

			cli.IntFlag{
				Name:  "max-conns-per-host",
				Value: 10,
				Usage: "The max number of TCP connections allowed per server. " +
					"This is effective when --disable-http2 is set.",
			},

			/////////////////////////
			// Monitoring & Logging
			/////////////////////////

			cli.DurationFlag{
				Name:  "stackdriver-export-interval",
				Value: 0,
				Usage: "Export metrics to stackdriver with this interval. The default value 0 indicates no exporting.",
			},

			cli.StringFlag{
				Name:  "log-file",
				Value: "",
				Usage: "The file for storing logs that can be parsed by " +
					"fluentd. When not provided, plain text logs are printed to " +
					"stdout.",
			},

			cli.StringFlag{
				Name:  "log-format",
				Value: "json",
				Usage: "The format of the log file: 'text' or 'json'.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_fs",
				Usage: "Enable file system debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_gcs",
				Usage: "Print GCS request and timing information.",
			},

			cli.BoolFlag{
				Name:  "debug_http",
				Usage: "Dump HTTP requests and responses to/from GCS.",
			},

			cli.BoolFlag{
				Name:  "debug_invariants",
				Usage: "Panic when internal invariants are violated.",
			},

			cli.BoolFlag{
				Name:  "debug_mutex",
				Usage: "Print debug messages when a mutex is held too long.",
			},
		},
	}

	return
}

type flagStorage struct {
	AppName    string
	Foreground bool

	// File system
	MountOptions   map[string]string
	DirMode        os.FileMode
	FileMode       os.FileMode
	Uid            int64
	Gid            int64
	ImplicitDirs   bool
	OnlyDir        string
	RenameDirLimit int64

	// GCS
	Endpoint                           *url.URL
	BillingProject                     string
	KeyFile                            string
	TokenUrl                           string
	EgressBandwidthLimitBytesPerSecond float64
	OpRateLimitHz                      float64

	// Tuning
	MaxRetrySleep     time.Duration
	StatCacheCapacity int
	StatCacheTTL      time.Duration
	TypeCacheTTL      time.Duration
	LocalFileCache    bool
	TempDir           string
	DisableHTTP2      bool
	MaxConnsPerHost   int

	// Monitoring & Logging
	StackdriverExportInterval time.Duration
	LogFile                   string
	LogFormat                 string

	// Debugging
	DebugFuse       bool
	DebugFS         bool
	DebugGCS        bool
	DebugHTTP       bool
	DebugInvariants bool
	DebugMutex      bool
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	endpoint, err := url.Parse(c.String("endpoint"))
	if err != nil {
		fmt.Printf("Could not parse endpoint")
		return nil
	}
	flags = &flagStorage{
		AppName:    c.String("app-name"),
		Foreground: c.Bool("foreground"),

		// File system
		MountOptions:   make(map[string]string),
		DirMode:        os.FileMode(*c.Generic("dir-mode").(*OctalInt)),
		FileMode:       os.FileMode(*c.Generic("file-mode").(*OctalInt)),
		Uid:            int64(c.Int("uid")),
		Gid:            int64(c.Int("gid")),
		ImplicitDirs:   c.Bool("implicit-dirs"),
		OnlyDir:        c.String("only-dir"),
		RenameDirLimit: int64(c.Int("rename-dir-limit")),

		// GCS,
		Endpoint:                           endpoint,
		BillingProject:                     c.String("billing-project"),
		KeyFile:                            c.String("key-file"),
		TokenUrl:                           c.String("token-url"),
		EgressBandwidthLimitBytesPerSecond: c.Float64("limit-bytes-per-sec"),
		OpRateLimitHz:                      c.Float64("limit-ops-per-sec"),

		// Tuning,
		MaxRetrySleep:     c.Duration("max-retry-sleep"),
		StatCacheCapacity: c.Int("stat-cache-capacity"),
		StatCacheTTL:      c.Duration("stat-cache-ttl"),
		TypeCacheTTL:      c.Duration("type-cache-ttl"),
		LocalFileCache:    c.Bool("local-file-cache"),
		TempDir:           c.String("temp-dir"),
		DisableHTTP2:      c.Bool("disable-http2"),
		MaxConnsPerHost:   c.Int("max-conns-per-host"),

		// Monitoring & Logging
		StackdriverExportInterval: c.Duration("stackdriver-export-interval"),
		LogFile:                   c.String("log-file"),
		LogFormat:                 c.String("log-format"),

		// Debugging,
		DebugFuse:       c.Bool("debug_fuse"),
		DebugGCS:        c.Bool("debug_gcs"),
		DebugFS:         c.Bool("debug_fs"),
		DebugHTTP:       c.Bool("debug_http"),
		DebugInvariants: c.Bool("debug_invariants"),
		DebugMutex:      c.Bool("debug_mutex"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		mountpkg.ParseOptions(flags.MountOptions, o)
	}

	return
}

// A cli.Generic that can be used with cli.GenericFlag to obtain an int flag
// that is parsed in octal.
type OctalInt int

var _ cli.Generic = (*OctalInt)(nil)

func (oi *OctalInt) Set(value string) (err error) {
	tmp, err := strconv.ParseInt(value, 8, 32)
	if err != nil {
		err = fmt.Errorf("Parsing as octal: %w", err)
		return
	}

	*oi = OctalInt(tmp)
	return
}

func (oi OctalInt) String() string {
	return fmt.Sprintf("%o", oi)
}
<commit_msg>Mark experimental features in the flag desc<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/url"
	"os"
	"strconv"
	"time"

	mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount"
	"github.com/urfave/cli"
)

// Set up custom help text for gcsfuse; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} [bucket] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func newApp() (app *cli.App) {
	dirModeValue := new(OctalInt)
	*dirModeValue = 0755

	fileModeValue := new(OctalInt)
	*fileModeValue = 0644

	app = &cli.App{
		Name:    "gcsfuse",
		Version: getVersion(),
		Usage:   "Mount a specified GCS bucket or all accessible buckets locally",
		Writer:  os.Stderr,
		Flags: []cli.Flag{

			cli.StringFlag{
				Name:  "app-name",
				Value: "",
				Usage: "The application name of this mount.",
			},

			cli.BoolFlag{
				Name:  "foreground",
				Usage: "Stay in the foreground after mounting.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.GenericFlag{
				Name:  "dir-mode",
				Value: dirModeValue,
				Usage: "Permissions bits for directories, in octal.",
			},

			cli.GenericFlag{
				Name:  "file-mode",
				Value: fileModeValue,
				Usage: "Permission bits for files, in octal.",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: -1,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: -1,
				Usage: "GID owner of all inodes.",
			},

			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See " +
					"docs/semantics.md",
			},

			cli.StringFlag{
				Name:  "only-dir",
				Usage: "Mount only the given directory, relative to the bucket root.",
			},

			cli.IntFlag{
				Name:  "rename-dir-limit",
				Value: 0,
				Usage: "Allow rename a directory containing fewer descendants " +
					"than this limit.",
			},

			/////////////////////////
			// GCS
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "https://storage.googleapis.com:443",
				Usage: "The endpoint to connect to.",
			},

			cli.StringFlag{
				Name:  "billing-project",
				Value: "",
				Usage: "Project to use for billing when accessing requester pays buckets. " +
					"(default: none)",
			},

			cli.StringFlag{
				Name:  "key-file",
				Value: "",
				Usage: "Absolute path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},

			cli.StringFlag{
				Name:  "token-url",
				Value: "",
				Usage: "An url for getting an access token when key-file is absent.",
			},

			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},

			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: -1,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "max-retry-sleep",
				Value: time.Minute,
				Usage: "The maximum duration allowed to sleep in a retry loop with " +
					"exponential backoff for failed requests to GCS backend. Once the " +
					"backoff duration exceeds this limit, the retry stops. The default " +
					"is 1 minute. A value of 0 disables retries.",
			},

			cli.IntFlag{
				Name:  "stat-cache-capacity",
				Value: 4096,
				Usage: "How many entries can the stat cache hold (impacts memory consumption)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			cli.BoolFlag{
				Name:  "local-file-cache",
				Usage: "Experimental: Cache GCS files on local disk for reads.",
			},

			cli.StringFlag{
				Name:  "temp-dir",
				Value: "",
				Usage: "Absolute path to temporary directory for local GCS object " +
					"copies. (default: system default, likely /tmp)",
			},

			cli.BoolFlag{
				Name: "disable-http2",
				Usage: "Once set, the protocol used for communicating with " +
					"GCS backend would be HTTP/1.1, instead of the default HTTP/2.",
			},

			cli.IntFlag{
				Name:  "max-conns-per-host",
				Value: 10,
				Usage: "The max number of TCP connections allowed per server. " +
					"This is effective when --disable-http2 is set.",
			},

			/////////////////////////
			// Monitoring & Logging
			/////////////////////////

			cli.DurationFlag{
				Name:  "stackdriver-export-interval",
				Value: 0,
				Usage: "Experimental: Export metrics to stackdriver with this interval. The default value 0 indicates no exporting.",
			},

			cli.StringFlag{
				Name:  "log-file",
				Value: "",
				Usage: "The file for storing logs that can be parsed by " +
					"fluentd. When not provided, plain text logs are printed to " +
					"stdout.",
			},

			cli.StringFlag{
				Name:  "log-format",
				Value: "json",
				Usage: "The format of the log file: 'text' or 'json'.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_fs",
				Usage: "Enable file system debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_gcs",
				Usage: "Print GCS request and timing information.",
			},

			cli.BoolFlag{
				Name:  "debug_http",
				Usage: "Dump HTTP requests and responses to/from GCS.",
			},

			cli.BoolFlag{
				Name:  "debug_invariants",
				Usage: "Panic when internal invariants are violated.",
			},

			cli.BoolFlag{
				Name:  "debug_mutex",
				Usage: "Print debug messages when a mutex is held too long.",
			},
		},
	}

	return
}

type flagStorage struct {
	AppName    string
	Foreground bool

	// File system
	MountOptions   map[string]string
	DirMode        os.FileMode
	FileMode       os.FileMode
	Uid            int64
	Gid            int64
	ImplicitDirs   bool
	OnlyDir        string
	RenameDirLimit int64

	// GCS
	Endpoint                           *url.URL
	BillingProject                     string
	KeyFile                            string
	TokenUrl                           string
	EgressBandwidthLimitBytesPerSecond float64
	OpRateLimitHz                      float64

	// Tuning
	MaxRetrySleep     time.Duration
	StatCacheCapacity int
	StatCacheTTL      time.Duration
	TypeCacheTTL      time.Duration
	LocalFileCache    bool
	TempDir           string
	DisableHTTP2      bool
	MaxConnsPerHost   int

	// Monitoring & Logging
	StackdriverExportInterval time.Duration
	LogFile                   string
	LogFormat                 string

	// Debugging
	DebugFuse       bool
	DebugFS         bool
	DebugGCS        bool
	DebugHTTP       bool
	DebugInvariants bool
	DebugMutex      bool
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	endpoint, err := url.Parse(c.String("endpoint"))
	if err != nil {
		fmt.Printf("Could not parse endpoint")
		return nil
	}
	flags = &flagStorage{
		AppName:    c.String("app-name"),
		Foreground: c.Bool("foreground"),

		// File system
		MountOptions:   make(map[string]string),
		DirMode:        os.FileMode(*c.Generic("dir-mode").(*OctalInt)),
		FileMode:       os.FileMode(*c.Generic("file-mode").(*OctalInt)),
		Uid:            int64(c.Int("uid")),
		Gid:            int64(c.Int("gid")),
		ImplicitDirs:   c.Bool("implicit-dirs"),
		OnlyDir:        c.String("only-dir"),
		RenameDirLimit: int64(c.Int("rename-dir-limit")),

		// GCS,
		Endpoint:                           endpoint,
		BillingProject:                     c.String("billing-project"),
		KeyFile:                            c.String("key-file"),
		TokenUrl:                           c.String("token-url"),
		EgressBandwidthLimitBytesPerSecond: c.Float64("limit-bytes-per-sec"),
		OpRateLimitHz:                      c.Float64("limit-ops-per-sec"),

		// Tuning,
		MaxRetrySleep:     c.Duration("max-retry-sleep"),
		StatCacheCapacity: c.Int("stat-cache-capacity"),
		StatCacheTTL:      c.Duration("stat-cache-ttl"),
		TypeCacheTTL:      c.Duration("type-cache-ttl"),
		LocalFileCache:    c.Bool("local-file-cache"),
		TempDir:           c.String("temp-dir"),
		DisableHTTP2:      c.Bool("disable-http2"),
		MaxConnsPerHost:   c.Int("max-conns-per-host"),

		// Monitoring & Logging
		StackdriverExportInterval: c.Duration("stackdriver-export-interval"),
		LogFile:                   c.String("log-file"),
		LogFormat:                 c.String("log-format"),

		// Debugging,
		DebugFuse:       c.Bool("debug_fuse"),
		DebugGCS:        c.Bool("debug_gcs"),
		DebugFS:         c.Bool("debug_fs"),
		DebugHTTP:       c.Bool("debug_http"),
		DebugInvariants: c.Bool("debug_invariants"),
		DebugMutex:      c.Bool("debug_mutex"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		mountpkg.ParseOptions(flags.MountOptions, o)
	}

	return
}

// A cli.Generic that can be used with cli.GenericFlag to obtain an int flag
// that is parsed in octal.
type OctalInt int

var _ cli.Generic = (*OctalInt)(nil)

func (oi *OctalInt) Set(value string) (err error) {
	tmp, err := strconv.ParseInt(value, 8, 32)
	if err != nil {
		err = fmt.Errorf("Parsing as octal: %w", err)
		return
	}

	*oi = OctalInt(tmp)
	return
}

func (oi OctalInt) String() string {
	return fmt.Sprintf("%o", oi)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"time"

	mountpkg "github.com/googlecloudplatform/gcsfuse/mount"
)

type flagStorage struct {
	MountOptions                       map[string]string
	Uid                                int64
	Gid                                int64
	FileMode                           uint
	DirMode                            uint
	TempDir                            string
	TempDirLimit                       int64
	GCSChunkSize                       uint64
	ImplicitDirs                       bool
	StatCacheTTL                       time.Duration
	TypeCacheTTL                       time.Duration
	OpRateLimitHz                      float64
	EgressBandwidthLimitBytesPerSecond float64
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlagSet(fs *flag.FlagSet) (flags *flagStorage) {
	flags = new(flagStorage)

	flags.MountOptions = make(map[string]string)
	fs.Var(
		mountpkg.OptionValue(flags.MountOptions),
		"o",
		"Additional system-specific mount options. Be careful!")

	fs.Int64Var(
		&flags.Uid,
		"uid",
		-1,
		"If non-negative, the UID that owns all inodes. The default is the UID of "+
			"the gcsfuse process.")

	fs.Int64Var(
		&flags.Gid,
		"gid",
		-1,
		"If non-negative, the GID that owns all inodes. The default is the GID of "+
			"the gcsfuse process.")

	fs.UintVar(
		&flags.FileMode,
		"file_mode",
		0644,
		"Permissions bits for files. Default is 0644.")

	fs.UintVar(
		&flags.DirMode,
		"dir_mode",
		0755,
		"Permissions bits for directories. Default is 0755.")

	fs.StringVar(
		&flags.TempDir,
		"temp_dir", "",
		"The temporary directory in which to store local copies of GCS objects. "+
			"If empty, the system default (probably /tmp) will be used.")

	fs.Int64Var(
		&flags.TempDirLimit,
		"temp_dir_bytes", 1<<31,
		"A desired limit on the number of bytes used in --temp_dir. May be "+
			"exceeded for dirty files that have not been flushed or closed.")

	fs.Uint64Var(
		&flags.GCSChunkSize,
		"gcs_chunk_size", 1<<24,
		"If set to a non-zero value N, split up GCS objects into multiple "+
			"chunks of size at most N when reading, and do not read or cache "+
			"unnecessary chunks.")

	fs.BoolVar(
		&flags.ImplicitDirs,
		"implicit_dirs",
		false,
		"Implicitly define directories based on their content. See "+
			"docs/semantics.md.")

	fs.DurationVar(
		&flags.StatCacheTTL,
		"stat_cache_ttl",
		time.Minute,
		"How long to cache StatObject results from GCS.")

	fs.DurationVar(
		&flags.TypeCacheTTL,
		"type_cache_ttl",
		time.Minute,
		"How long to cache name -> file/dir type mappings in directory inodes.")

	fs.Float64Var(
		&flags.OpRateLimitHz,
		"op_rate_limit_hz",
		-1,
		"If positive, a limit on the rate at which we send requests to GCS, "+
			"measured over a 30-second window.")

	fs.Float64Var(
		&flags.EgressBandwidthLimitBytesPerSecond,
		"egress_bandwidth_limit_bytes_per_second",
		-1,
		"If positive, a limit on the GCS -> gcsfuse bandwidth for reading "+
			"objects, measured over a 30-second window.")

	return
}
<commit_msg>Set a default GCS request rate limit of 5 Hz.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"time"

	mountpkg "github.com/googlecloudplatform/gcsfuse/mount"
)

type flagStorage struct {
	MountOptions                       map[string]string
	Uid                                int64
	Gid                                int64
	FileMode                           uint
	DirMode                            uint
	TempDir                            string
	TempDirLimit                       int64
	GCSChunkSize                       uint64
	ImplicitDirs                       bool
	StatCacheTTL                       time.Duration
	TypeCacheTTL                       time.Duration
	OpRateLimitHz                      float64
	EgressBandwidthLimitBytesPerSecond float64
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlagSet(fs *flag.FlagSet) (flags *flagStorage) {
	flags = new(flagStorage)

	flags.MountOptions = make(map[string]string)
	fs.Var(
		mountpkg.OptionValue(flags.MountOptions),
		"o",
		"Additional system-specific mount options. Be careful!")

	fs.Int64Var(
		&flags.Uid,
		"uid",
		-1,
		"If non-negative, the UID that owns all inodes. The default is the UID of "+
			"the gcsfuse process.")

	fs.Int64Var(
		&flags.Gid,
		"gid",
		-1,
		"If non-negative, the GID that owns all inodes. The default is the GID of "+
			"the gcsfuse process.")

	fs.UintVar(
		&flags.FileMode,
		"file_mode",
		0644,
		"Permissions bits for files. Default is 0644.")

	fs.UintVar(
		&flags.DirMode,
		"dir_mode",
		0755,
		"Permissions bits for directories. Default is 0755.")

	fs.StringVar(
		&flags.TempDir,
		"temp_dir", "",
		"The temporary directory in which to store local copies of GCS objects. "+
			"If empty, the system default (probably /tmp) will be used.")

	fs.Int64Var(
		&flags.TempDirLimit,
		"temp_dir_bytes", 1<<31,
		"A desired limit on the number of bytes used in --temp_dir. May be "+
			"exceeded for dirty files that have not been flushed or closed.")

	fs.Uint64Var(
		&flags.GCSChunkSize,
		"gcs_chunk_size", 1<<24,
		"If set to a non-zero value N, split up GCS objects into multiple "+
			"chunks of size at most N when reading, and do not read or cache "+
			"unnecessary chunks.")

	fs.BoolVar(
		&flags.ImplicitDirs,
		"implicit_dirs",
		false,
		"Implicitly define directories based on their content. See "+
			"docs/semantics.md.")

	fs.DurationVar(
		&flags.StatCacheTTL,
		"stat_cache_ttl",
		time.Minute,
		"How long to cache StatObject results from GCS.")

	fs.DurationVar(
		&flags.TypeCacheTTL,
		"type_cache_ttl",
		time.Minute,
		"How long to cache name -> file/dir type mappings in directory inodes.")

	fs.Float64Var(
		&flags.OpRateLimitHz,
		"op_rate_limit_hz",
		5.0,
		"If positive, a limit on the rate at which we send requests to GCS, "+
			"measured over a 30-second window.")

	fs.Float64Var(
		&flags.EgressBandwidthLimitBytesPerSecond,
		"egress_bandwidth_limit_bytes_per_second",
		-1,
		"If positive, a limit on the GCS -> gcsfuse bandwidth for reading "+
			"objects, measured over a 30-second window.")

	return
}
<|endoftext|>