<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package controller

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/names"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/api/controller"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/juju/block"
	"github.com/juju/juju/cmd/modelcmd"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/jujuclient"
)

// NewDestroyCommand returns a command to destroy a controller.
func NewDestroyCommand() cmd.Command {
	// Even though this command is all about destroying a controller we end up
	// needing environment endpoints so we can fall back to the client destroy
	// environment method. This shouldn't really matter in practice as the
	// user trying to take down the controller will need to have access to the
	// controller environment anyway.
	return modelcmd.WrapController(
		&destroyCommand{},
		modelcmd.ControllerSkipFlags,
		modelcmd.ControllerSkipDefault,
	)
}

// destroyCommand destroys the specified controller.
type destroyCommand struct {
	destroyCommandBase
	destroyModels bool
}

// usageDetails has backticks which we want to keep for markdown processing.
// TODO(cheryl): Do we want the usage, options, examples, and see also text in
// backticks for markdown?
var usageDetails = `
All models (initial model plus all workload/hosted) associated with the
controller will first need to be destroyed, either in advance, or by
specifying `[1:] + "`--destroy-all-models`." + `

Examples:
juju destroy-controller --destroy-all-models mycontroller

See also:
kill-controller`

var usageSummary = `
Destroys a controller.`[1:]

var destroySysMsg = `
WARNING! This command will destroy the %q controller.
This includes all machines, services, data and other resources.

Continue [y/N]? `[1:]

// destroyControllerAPI defines the methods on the controller API endpoint
// that the destroy command calls.
type destroyControllerAPI interface {
	Close() error
	ModelConfig() (map[string]interface{}, error)
	DestroyController(destroyModels bool) error
	ListBlockedModels() ([]params.ModelBlockInfo, error)
	ModelStatus(models ...names.ModelTag) ([]base.ModelStatus, error)
	AllModels() ([]base.UserModel, error)
}

// destroyClientAPI defines the methods on the client API endpoint that the
// destroy command might call.
type destroyClientAPI interface {
	Close() error
	ModelGet() (map[string]interface{}, error)
	DestroyModel() error
}

// Info implements Command.Info.
func (c *destroyCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "destroy-controller",
		Args:    "<controller name>",
		Purpose: usageSummary,
		Doc:     usageDetails,
	}
}

// SetFlags implements Command.SetFlags.
func (c *destroyCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.destroyModels, "destroy-all-models", false, "Destroy all hosted models in the controller")
	c.destroyCommandBase.SetFlags(f)
}

// Run implements Command.Run
func (c *destroyCommand) Run(ctx *cmd.Context) error {
	controllerName := c.ControllerName()
	store := c.ClientStore()
	controllerDetails, err := store.ControllerByName(controllerName)
	if err != nil {
		return errors.Annotate(err, "cannot read controller info")
	}

	if !c.assumeYes {
		if err = confirmDestruction(ctx, c.ControllerName()); err != nil {
			return err
		}
	}

	// Attempt to connect to the API.  If we can't, fail the destroy.  Users will
	// need to use the controller kill command if we can't connect.
	api, err := c.getControllerAPI()
	if err != nil {
		return c.ensureUserFriendlyErrorLog(errors.Annotate(err, "cannot connect to API"), ctx, nil)
	}
	defer api.Close()

	// Obtain controller environ so we can clean up afterwards.
	controllerEnviron, err := c.getControllerEnviron(store, controllerName, api)
	if err != nil {
		return errors.Annotate(err, "getting controller environ")
	}

	for {
		// Attempt to destroy the controller.
		ctx.Infof("Destroying controller")
		var hasHostedModels bool
		err = api.DestroyController(c.destroyModels)
		if err != nil {
			if params.IsCodeHasHostedModels(err) {
				hasHostedModels = true
			} else {
				return c.ensureUserFriendlyErrorLog(
					errors.Annotate(err, "cannot destroy controller"),
					ctx, api,
				)
			}
		}

		updateStatus := newTimedStatusUpdater(ctx, api, controllerDetails.ControllerUUID)
		ctrStatus, modelsStatus := updateStatus(0)
		if !c.destroyModels {
			if err := c.checkNoAliveHostedModels(ctx, modelsStatus); err != nil {
				return errors.Trace(err)
			}
			if hasHostedModels && !hasUnDeadModels(modelsStatus) {
				// When we called DestroyController before, we were
				// informed that there were hosted models remaining.
				// When we checked just now, there were none. We should
				// try destroying again.
				continue
			}
		}

		// Even if we've not just requested for hosted models to be destroyed,
		// there may be some being destroyed already. We need to wait for them.
		ctx.Infof("Waiting for hosted model resources to be reclaimed")
		for ; hasUnDeadModels(modelsStatus); ctrStatus, modelsStatus = updateStatus(2 * time.Second) {
			ctx.Infof(fmtCtrStatus(ctrStatus))
			for _, model := range modelsStatus {
				ctx.Verbosef(fmtModelStatus(model))
			}
		}
		ctx.Infof("All hosted models reclaimed, cleaning up controller machines")
		return environs.Destroy(c.ControllerName(), controllerEnviron, store)
	}
}

// checkNoAliveHostedModels ensures that the given set of hosted models
// contains none that are Alive. If there are, an message is printed
// out to
func (c *destroyCommand) checkNoAliveHostedModels(ctx *cmd.Context, models []modelData) error {
	if !hasAliveModels(models) {
		return nil
	}
	// The user did not specify --destroy-all-models,
	// and there are models still alive.
	var buf bytes.Buffer
	for _, model := range models {
		if model.Life != params.Alive {
			continue
		}
		buf.WriteString(fmtModelStatus(model))
		buf.WriteRune('\n')
	}
	return errors.Errorf(`cannot destroy controller %q

The controller has live hosted models. If you want
to destroy all hosted models in the controller,
run this command again with the --destroy-all-models
flag.

Models:
%s`, c.ControllerName(), buf.String())
}

// ensureUserFriendlyErrorLog ensures that error will be logged and displayed
// in a user-friendly manner with readable and digestable error message.
func (c *destroyCommand) ensureUserFriendlyErrorLog(destroyErr error, ctx *cmd.Context, api destroyControllerAPI) error {
	if destroyErr == nil {
		return nil
	}
	if params.IsCodeOperationBlocked(destroyErr) {
		logger.Errorf(destroyControllerBlockedMsg)
		if api != nil {
			models, err := api.ListBlockedModels()
			var bytes []byte
			if err == nil {
				bytes, err = formatTabularBlockedModels(models)
			}
			if err != nil {
				logger.Errorf("Unable to list blocked models: %s", err)
				return cmd.ErrSilent
			}
			ctx.Infof(string(bytes))
		}
		return cmd.ErrSilent
	}
	if params.IsCodeHasHostedModels(destroyErr) {
		return destroyErr
	}
	logger.Errorf(stdFailureMsg, c.ControllerName())
	return destroyErr
}

const destroyControllerBlockedMsg = `there are blocks preventing controller destruction
To remove all blocks in the controller, please run:

    juju controller remove-blocks

`

// TODO(axw) this should only be printed out if we couldn't
// connect to the controller.
const stdFailureMsg = `failed to destroy controller %q

If the controller is unusable, then you may run

    juju kill-controller

to forcibly destroy the controller. Upon doing so, review
your cloud provider console for any resources that need
to be cleaned up.

`

func formatTabularBlockedModels(value interface{}) ([]byte, error) {
	models, ok := value.([]params.ModelBlockInfo)
	if !ok {
		return nil, errors.Errorf("expected value of type %T, got %T", models, value)
	}

	var out bytes.Buffer
	const (
		// To format things into columns.
		minwidth = 0
		tabwidth = 1
		padding  = 2
		padchar  = ' '
		flags    = 0
	)
	tw := tabwriter.NewWriter(&out, minwidth, tabwidth, padding, padchar, flags)
	fmt.Fprintf(tw, "NAME\tMODEL UUID\tOWNER\tBLOCKS\n")
	for _, model := range models {
		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", model.Name, model.UUID, model.OwnerTag, blocksToStr(model.Blocks))
	}
	tw.Flush()
	return out.Bytes(), nil
}

func blocksToStr(blocks []string) string {
	result := ""
	sep := ""
	for _, blk := range blocks {
		result = result + sep + block.OperationFromType(blk)
		sep = ","
	}

	return result
}

// destroyCommandBase provides common attributes and methods that both the controller
// destroy and controller kill commands require.
type destroyCommandBase struct {
	modelcmd.ControllerCommandBase
	assumeYes bool

	// The following fields are for mocking out
	// api behavior for testing.
	api       destroyControllerAPI
	apierr    error
	clientapi destroyClientAPI
}

func (c *destroyCommandBase) getControllerAPI() (destroyControllerAPI, error) {
	if c.api != nil {
		return c.api, c.apierr
	}
	root, err := c.NewAPIRoot()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return controller.NewClient(root), nil
}

// SetFlags implements Command.SetFlags.
func (c *destroyCommandBase) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.assumeYes, "y", false, "Do not ask for confirmation")
	f.BoolVar(&c.assumeYes, "yes", false, "")
}

// Init implements Command.Init.
func (c *destroyCommandBase) Init(args []string) error {
	switch len(args) {
	case 0:
		return errors.New("no controller specified")
	case 1:
		return c.SetControllerName(args[0])
	default:
		return cmd.CheckEmpty(args[1:])
	}
}

// getControllerEnviron returns the Environ for the controller model.
//
// getControllerEnviron gets the information required to get the
// Environ by first checking the config store, then querying the
// API if the information is not in the store.
func (c *destroyCommandBase) getControllerEnviron(
	store jujuclient.ClientStore, controllerName string, sysAPI destroyControllerAPI,
) (_ environs.Environ, err error) {
	cfg, err := modelcmd.NewGetBootstrapConfigFunc(store)(controllerName)
	if errors.IsNotFound(err) {
		if sysAPI == nil {
			return nil, errors.New(
				"unable to get bootstrap information from client store or API",
			)
		}
		bootstrapConfig, err := sysAPI.ModelConfig()
		if err != nil {
			return nil, errors.Annotate(err, "getting bootstrap config from API")
		}
		cfg, err = config.New(config.NoDefaults, bootstrapConfig)
		if err != nil {
			return nil, errors.Trace(err)
		}
	} else if err != nil {
		return nil, errors.Annotate(err, "getting bootstrap config from client store")
	}
	return environs.New(cfg)
}

func confirmDestruction(ctx *cmd.Context, controllerName string) error {
	// Get confirmation from the user that they want to continue
	fmt.Fprintf(ctx.Stdout, destroySysMsg, controllerName)

	scanner := bufio.NewScanner(ctx.Stdin)
	scanner.Scan()
	err := scanner.Err()
	if err != nil && err != io.EOF {
		return errors.Annotate(err, "controller destruction aborted")
	}
	answer := strings.ToLower(scanner.Text())
	if answer != "y" && answer != "yes" {
		return errors.New("controller destruction aborted")
	}

	return nil
}
<commit_msg>Indent destroy-controller<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package controller

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/names"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/api/controller"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/juju/block"
	"github.com/juju/juju/cmd/modelcmd"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/jujuclient"
)

// NewDestroyCommand returns a command to destroy a controller.
func NewDestroyCommand() cmd.Command {
	// Even though this command is all about destroying a controller we end up
	// needing environment endpoints so we can fall back to the client destroy
	// environment method. This shouldn't really matter in practice as the
	// user trying to take down the controller will need to have access to the
	// controller environment anyway.
	return modelcmd.WrapController(
		&destroyCommand{},
		modelcmd.ControllerSkipFlags,
		modelcmd.ControllerSkipDefault,
	)
}

// destroyCommand destroys the specified controller.
type destroyCommand struct {
	destroyCommandBase
	destroyModels bool
}

// usageDetails has backticks which we want to keep for markdown processing.
// TODO(cheryl): Do we want the usage, options, examples, and see also text in
// backticks for markdown?
var usageDetails = `
All models (initial model plus all workload/hosted) associated with the
controller will first need to be destroyed, either in advance, or by
specifying `[1:] + "`--destroy-all-models`." + `

Examples:
    juju destroy-controller --destroy-all-models mycontroller

See also:
    kill-controller`

var usageSummary = `
Destroys a controller.`[1:]

var destroySysMsg = `
WARNING! This command will destroy the %q controller.
This includes all machines, services, data and other resources.

Continue [y/N]? `[1:]

// destroyControllerAPI defines the methods on the controller API endpoint
// that the destroy command calls.
type destroyControllerAPI interface {
	Close() error
	ModelConfig() (map[string]interface{}, error)
	DestroyController(destroyModels bool) error
	ListBlockedModels() ([]params.ModelBlockInfo, error)
	ModelStatus(models ...names.ModelTag) ([]base.ModelStatus, error)
	AllModels() ([]base.UserModel, error)
}

// destroyClientAPI defines the methods on the client API endpoint that the
// destroy command might call.
type destroyClientAPI interface {
	Close() error
	ModelGet() (map[string]interface{}, error)
	DestroyModel() error
}

// Info implements Command.Info.
func (c *destroyCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "destroy-controller",
		Args:    "<controller name>",
		Purpose: usageSummary,
		Doc:     usageDetails,
	}
}

// SetFlags implements Command.SetFlags.
func (c *destroyCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.destroyModels, "destroy-all-models", false, "Destroy all hosted models in the controller")
	c.destroyCommandBase.SetFlags(f)
}

// Run implements Command.Run
func (c *destroyCommand) Run(ctx *cmd.Context) error {
	controllerName := c.ControllerName()
	store := c.ClientStore()
	controllerDetails, err := store.ControllerByName(controllerName)
	if err != nil {
		return errors.Annotate(err, "cannot read controller info")
	}

	if !c.assumeYes {
		if err = confirmDestruction(ctx, c.ControllerName()); err != nil {
			return err
		}
	}

	// Attempt to connect to the API.  If we can't, fail the destroy.  Users will
	// need to use the controller kill command if we can't connect.
	api, err := c.getControllerAPI()
	if err != nil {
		return c.ensureUserFriendlyErrorLog(errors.Annotate(err, "cannot connect to API"), ctx, nil)
	}
	defer api.Close()

	// Obtain controller environ so we can clean up afterwards.
	controllerEnviron, err := c.getControllerEnviron(store, controllerName, api)
	if err != nil {
		return errors.Annotate(err, "getting controller environ")
	}

	for {
		// Attempt to destroy the controller.
		ctx.Infof("Destroying controller")
		var hasHostedModels bool
		err = api.DestroyController(c.destroyModels)
		if err != nil {
			if params.IsCodeHasHostedModels(err) {
				hasHostedModels = true
			} else {
				return c.ensureUserFriendlyErrorLog(
					errors.Annotate(err, "cannot destroy controller"),
					ctx, api,
				)
			}
		}

		updateStatus := newTimedStatusUpdater(ctx, api, controllerDetails.ControllerUUID)
		ctrStatus, modelsStatus := updateStatus(0)
		if !c.destroyModels {
			if err := c.checkNoAliveHostedModels(ctx, modelsStatus); err != nil {
				return errors.Trace(err)
			}
			if hasHostedModels && !hasUnDeadModels(modelsStatus) {
				// When we called DestroyController before, we were
				// informed that there were hosted models remaining.
				// When we checked just now, there were none. We should
				// try destroying again.
				continue
			}
		}

		// Even if we've not just requested for hosted models to be destroyed,
		// there may be some being destroyed already. We need to wait for them.
		ctx.Infof("Waiting for hosted model resources to be reclaimed")
		for ; hasUnDeadModels(modelsStatus); ctrStatus, modelsStatus = updateStatus(2 * time.Second) {
			ctx.Infof(fmtCtrStatus(ctrStatus))
			for _, model := range modelsStatus {
				ctx.Verbosef(fmtModelStatus(model))
			}
		}
		ctx.Infof("All hosted models reclaimed, cleaning up controller machines")
		return environs.Destroy(c.ControllerName(), controllerEnviron, store)
	}
}

// checkNoAliveHostedModels ensures that the given set of hosted models
// contains none that are Alive. If there are, an message is printed
// out to
func (c *destroyCommand) checkNoAliveHostedModels(ctx *cmd.Context, models []modelData) error {
	if !hasAliveModels(models) {
		return nil
	}
	// The user did not specify --destroy-all-models,
	// and there are models still alive.
	var buf bytes.Buffer
	for _, model := range models {
		if model.Life != params.Alive {
			continue
		}
		buf.WriteString(fmtModelStatus(model))
		buf.WriteRune('\n')
	}
	return errors.Errorf(`cannot destroy controller %q

The controller has live hosted models. If you want
to destroy all hosted models in the controller,
run this command again with the --destroy-all-models
flag.

Models:
%s`, c.ControllerName(), buf.String())
}

// ensureUserFriendlyErrorLog ensures that error will be logged and displayed
// in a user-friendly manner with readable and digestable error message.
func (c *destroyCommand) ensureUserFriendlyErrorLog(destroyErr error, ctx *cmd.Context, api destroyControllerAPI) error {
	if destroyErr == nil {
		return nil
	}
	if params.IsCodeOperationBlocked(destroyErr) {
		logger.Errorf(destroyControllerBlockedMsg)
		if api != nil {
			models, err := api.ListBlockedModels()
			var bytes []byte
			if err == nil {
				bytes, err = formatTabularBlockedModels(models)
			}
			if err != nil {
				logger.Errorf("Unable to list blocked models: %s", err)
				return cmd.ErrSilent
			}
			ctx.Infof(string(bytes))
		}
		return cmd.ErrSilent
	}
	if params.IsCodeHasHostedModels(destroyErr) {
		return destroyErr
	}
	logger.Errorf(stdFailureMsg, c.ControllerName())
	return destroyErr
}

const destroyControllerBlockedMsg = `there are blocks preventing controller destruction
To remove all blocks in the controller, please run:

    juju controller remove-blocks

`

// TODO(axw) this should only be printed out if we couldn't
// connect to the controller.
const stdFailureMsg = `failed to destroy controller %q

If the controller is unusable, then you may run

    juju kill-controller

to forcibly destroy the controller. Upon doing so, review
your cloud provider console for any resources that need
to be cleaned up.

`

func formatTabularBlockedModels(value interface{}) ([]byte, error) {
	models, ok := value.([]params.ModelBlockInfo)
	if !ok {
		return nil, errors.Errorf("expected value of type %T, got %T", models, value)
	}

	var out bytes.Buffer
	const (
		// To format things into columns.
		minwidth = 0
		tabwidth = 1
		padding  = 2
		padchar  = ' '
		flags    = 0
	)
	tw := tabwriter.NewWriter(&out, minwidth, tabwidth, padding, padchar, flags)
	fmt.Fprintf(tw, "NAME\tMODEL UUID\tOWNER\tBLOCKS\n")
	for _, model := range models {
		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", model.Name, model.UUID, model.OwnerTag, blocksToStr(model.Blocks))
	}
	tw.Flush()
	return out.Bytes(), nil
}

func blocksToStr(blocks []string) string {
	result := ""
	sep := ""
	for _, blk := range blocks {
		result = result + sep + block.OperationFromType(blk)
		sep = ","
	}

	return result
}

// destroyCommandBase provides common attributes and methods that both the controller
// destroy and controller kill commands require.
type destroyCommandBase struct {
	modelcmd.ControllerCommandBase
	assumeYes bool

	// The following fields are for mocking out
	// api behavior for testing.
	api       destroyControllerAPI
	apierr    error
	clientapi destroyClientAPI
}

func (c *destroyCommandBase) getControllerAPI() (destroyControllerAPI, error) {
	if c.api != nil {
		return c.api, c.apierr
	}
	root, err := c.NewAPIRoot()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return controller.NewClient(root), nil
}

// SetFlags implements Command.SetFlags.
func (c *destroyCommandBase) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.assumeYes, "y", false, "Do not ask for confirmation")
	f.BoolVar(&c.assumeYes, "yes", false, "")
}

// Init implements Command.Init.
func (c *destroyCommandBase) Init(args []string) error {
	switch len(args) {
	case 0:
		return errors.New("no controller specified")
	case 1:
		return c.SetControllerName(args[0])
	default:
		return cmd.CheckEmpty(args[1:])
	}
}

// getControllerEnviron returns the Environ for the controller model.
//
// getControllerEnviron gets the information required to get the
// Environ by first checking the config store, then querying the
// API if the information is not in the store.
func (c *destroyCommandBase) getControllerEnviron(
	store jujuclient.ClientStore, controllerName string, sysAPI destroyControllerAPI,
) (_ environs.Environ, err error) {
	cfg, err := modelcmd.NewGetBootstrapConfigFunc(store)(controllerName)
	if errors.IsNotFound(err) {
		if sysAPI == nil {
			return nil, errors.New(
				"unable to get bootstrap information from client store or API",
			)
		}
		bootstrapConfig, err := sysAPI.ModelConfig()
		if err != nil {
			return nil, errors.Annotate(err, "getting bootstrap config from API")
		}
		cfg, err = config.New(config.NoDefaults, bootstrapConfig)
		if err != nil {
			return nil, errors.Trace(err)
		}
	} else if err != nil {
		return nil, errors.Annotate(err, "getting bootstrap config from client store")
	}
	return environs.New(cfg)
}

func confirmDestruction(ctx *cmd.Context, controllerName string) error {
	// Get confirmation from the user that they want to continue
	fmt.Fprintf(ctx.Stdout, destroySysMsg, controllerName)

	scanner := bufio.NewScanner(ctx.Stdin)
	scanner.Scan()
	err := scanner.Err()
	if err != nil && err != io.EOF {
		return errors.Annotate(err, "controller destruction aborted")
	}
	answer := strings.ToLower(scanner.Text())
	if answer != "y" && answer != "yes" {
		return errors.New("controller destruction aborted")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nofilesystem
// +build linux freebsd openbsd darwin,amd64 dragonfly

package collector

import (
	"regexp"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/alecthomas/kingpin.v2"
)

// Arch-dependent implementation must define:
// * defIgnoredMountPoints
// * defIgnoredFSTypes
// * filesystemLabelNames
// * filesystemCollector.GetStats

var (
	ignoredMountPoints = kingpin.Flag(
		"collector.filesystem.ignored-mount-points",
		"Regexp of mount points to ignore for filesystem collector.",
	).Default(defIgnoredMountPoints).String()
	ignoredFSTypes = kingpin.Flag(
		"collector.filesystem.ignored-fs-types",
		"Regexp of filesystem types to ignore for filesystem collector.",
	).Default(defIgnoredFSTypes).String()

	filesystemLabelNames = []string{"device", "mountpoint", "fstype"}
)

type filesystemCollector struct {
	ignoredMountPointsPattern     *regexp.Regexp
	ignoredFSTypesPattern         *regexp.Regexp
	sizeDesc, freeDesc, availDesc *prometheus.Desc
	filesDesc, filesFreeDesc      *prometheus.Desc
	roDesc, deviceErrorDesc       *prometheus.Desc
	logger                        log.Logger
}

type filesystemLabels struct {
	device, mountPoint, fsType, options string
}

type filesystemStats struct {
	labels            filesystemLabels
	size, free, avail float64
	files, filesFree  float64
	ro, deviceError   float64
}

func init() {
	registerCollector("filesystem", defaultEnabled, NewFilesystemCollector)
}

// NewFilesystemCollector returns a new Collector exposing filesystems stats.
func NewFilesystemCollector(logger log.Logger) (Collector, error) {
	subsystem := "filesystem"
	level.Info(logger).Log("msg", "Parsed flag --collector.filesystem.ignored-mount-points", "flag", *ignoredMountPoints)
	mountPointPattern := regexp.MustCompile(*ignoredMountPoints)
	level.Info(logger).Log("msg", "Parsed flag --collector.filesystem.ignored-fs-types", "flag", *ignoredMountPoints)
	filesystemsTypesPattern := regexp.MustCompile(*ignoredFSTypes)

	sizeDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "size_bytes"),
		"Filesystem size in bytes.",
		filesystemLabelNames, nil,
	)

	freeDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "free_bytes"),
		"Filesystem free space in bytes.",
		filesystemLabelNames, nil,
	)

	availDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "avail_bytes"),
		"Filesystem space available to non-root users in bytes.",
		filesystemLabelNames, nil,
	)

	filesDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "files"),
		"Filesystem total file nodes.",
		filesystemLabelNames, nil,
	)

	filesFreeDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "files_free"),
		"Filesystem total free file nodes.",
		filesystemLabelNames, nil,
	)

	roDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "readonly"),
		"Filesystem read-only status.",
		filesystemLabelNames, nil,
	)

	deviceErrorDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "device_error"),
		"Whether an error occurred while getting statistics for the given device.",
		filesystemLabelNames, nil,
	)

	return &filesystemCollector{
		ignoredMountPointsPattern: mountPointPattern,
		ignoredFSTypesPattern:     filesystemsTypesPattern,
		sizeDesc:                  sizeDesc,
		freeDesc:                  freeDesc,
		availDesc:                 availDesc,
		filesDesc:                 filesDesc,
		filesFreeDesc:             filesFreeDesc,
		roDesc:                    roDesc,
		deviceErrorDesc:           deviceErrorDesc,
		logger:                    logger,
	}, nil
}

func (c *filesystemCollector) Update(ch chan<- prometheus.Metric) error {
	stats, err := c.GetStats()
	if err != nil {
		return err
	}
	// Make sure we expose a metric once, even if there are multiple mounts
	seen := map[filesystemLabels]bool{}
	for _, s := range stats {
		if seen[s.labels] {
			continue
		}
		seen[s.labels] = true

		ch <- prometheus.MustNewConstMetric(
			c.deviceErrorDesc, prometheus.GaugeValue,
			s.deviceError, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		if s.deviceError > 0 {
			continue
		}

		ch <- prometheus.MustNewConstMetric(
			c.sizeDesc, prometheus.GaugeValue,
			s.size, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.freeDesc, prometheus.GaugeValue,
			s.free, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.availDesc, prometheus.GaugeValue,
			s.avail, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.filesDesc, prometheus.GaugeValue,
			s.files, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.filesFreeDesc, prometheus.GaugeValue,
			s.filesFree, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.roDesc, prometheus.GaugeValue,
			s.ro, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
	}
	return nil
}
<commit_msg>collector/filesystem: fixing logging message<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nofilesystem
// +build linux freebsd openbsd darwin,amd64 dragonfly

package collector

import (
	"regexp"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/alecthomas/kingpin.v2"
)

// Arch-dependent implementation must define:
// * defIgnoredMountPoints
// * defIgnoredFSTypes
// * filesystemLabelNames
// * filesystemCollector.GetStats

var (
	ignoredMountPoints = kingpin.Flag(
		"collector.filesystem.ignored-mount-points",
		"Regexp of mount points to ignore for filesystem collector.",
	).Default(defIgnoredMountPoints).String()
	ignoredFSTypes = kingpin.Flag(
		"collector.filesystem.ignored-fs-types",
		"Regexp of filesystem types to ignore for filesystem collector.",
	).Default(defIgnoredFSTypes).String()

	filesystemLabelNames = []string{"device", "mountpoint", "fstype"}
)

type filesystemCollector struct {
	ignoredMountPointsPattern     *regexp.Regexp
	ignoredFSTypesPattern         *regexp.Regexp
	sizeDesc, freeDesc, availDesc *prometheus.Desc
	filesDesc, filesFreeDesc      *prometheus.Desc
	roDesc, deviceErrorDesc       *prometheus.Desc
	logger                        log.Logger
}

type filesystemLabels struct {
	device, mountPoint, fsType, options string
}

type filesystemStats struct {
	labels            filesystemLabels
	size, free, avail float64
	files, filesFree  float64
	ro, deviceError   float64
}

func init() {
	registerCollector("filesystem", defaultEnabled, NewFilesystemCollector)
}

// NewFilesystemCollector returns a new Collector exposing filesystems stats.
func NewFilesystemCollector(logger log.Logger) (Collector, error) {
	subsystem := "filesystem"
	level.Info(logger).Log("msg", "Parsed flag --collector.filesystem.ignored-mount-points", "flag", *ignoredMountPoints)
	mountPointPattern := regexp.MustCompile(*ignoredMountPoints)
	level.Info(logger).Log("msg", "Parsed flag --collector.filesystem.ignored-fs-types", "flag", *ignoredFSTypes)
	filesystemsTypesPattern := regexp.MustCompile(*ignoredFSTypes)

	sizeDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "size_bytes"),
		"Filesystem size in bytes.",
		filesystemLabelNames, nil,
	)

	freeDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "free_bytes"),
		"Filesystem free space in bytes.",
		filesystemLabelNames, nil,
	)

	availDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "avail_bytes"),
		"Filesystem space available to non-root users in bytes.",
		filesystemLabelNames, nil,
	)

	filesDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "files"),
		"Filesystem total file nodes.",
		filesystemLabelNames, nil,
	)

	filesFreeDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "files_free"),
		"Filesystem total free file nodes.",
		filesystemLabelNames, nil,
	)

	roDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "readonly"),
		"Filesystem read-only status.",
		filesystemLabelNames, nil,
	)

	deviceErrorDesc := prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, "device_error"),
		"Whether an error occurred while getting statistics for the given device.",
		filesystemLabelNames, nil,
	)

	return &filesystemCollector{
		ignoredMountPointsPattern: mountPointPattern,
		ignoredFSTypesPattern:     filesystemsTypesPattern,
		sizeDesc:                  sizeDesc,
		freeDesc:                  freeDesc,
		availDesc:                 availDesc,
		filesDesc:                 filesDesc,
		filesFreeDesc:             filesFreeDesc,
		roDesc:                    roDesc,
		deviceErrorDesc:           deviceErrorDesc,
		logger:                    logger,
	}, nil
}

func (c *filesystemCollector) Update(ch chan<- prometheus.Metric) error {
	stats, err := c.GetStats()
	if err != nil {
		return err
	}
	// Make sure we expose a metric once, even if there are multiple mounts
	seen := map[filesystemLabels]bool{}
	for _, s := range stats {
		if seen[s.labels] {
			continue
		}
		seen[s.labels] = true

		ch <- prometheus.MustNewConstMetric(
			c.deviceErrorDesc, prometheus.GaugeValue,
			s.deviceError, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		if s.deviceError > 0 {
			continue
		}

		ch <- prometheus.MustNewConstMetric(
			c.sizeDesc, prometheus.GaugeValue,
			s.size, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.freeDesc, prometheus.GaugeValue,
			s.free, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.availDesc, prometheus.GaugeValue,
			s.avail, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.filesDesc, prometheus.GaugeValue,
			s.files, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.filesFreeDesc, prometheus.GaugeValue,
			s.filesFree, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
		ch <- prometheus.MustNewConstMetric(
			c.roDesc, prometheus.GaugeValue,
			s.ro, s.labels.device, s.labels.mountPoint, s.labels.fsType,
		)
	}
	return nil
}
<|endoftext|>


<commit_before>package genapi

import (
	"fmt"
	"net"

	"github.com/armon/go-proxyproto"
)

type proxyListener struct {
	net.Listener
	allowed []*net.IPNet
}

func newProxyListener(l net.Listener, allowed []string) (net.Listener, error) {
	a := make([]*net.IPNet, 0, len(allowed))
	for i := range allowed {
		if allowed[i] == "" {
			continue
		}
		_, parsed, err := net.ParseCIDR(allowed[i])
		if err != nil {
			return proxyListener{}, err
		}
		a = append(a, parsed)
	}
	if len(a) == 0 {
		return l, nil
	}
	return proxyListener{
		Listener: l,
		allowed:  a,
	}, nil
}

func (p proxyListener) Accept() (net.Conn, error) {
	c, err := p.Listener.Accept()
	if err != nil {
		return c, err
	}

	// The next two steps should never error. If they do something is crazy
	// wrong

	ipStr, _, err := net.SplitHostPort(c.RemoteAddr().String())
	if err != nil {
		return nil, fmt.Errorf("invalid RemoteAddr on accepted conn: %s", err)
	}

	ip := net.ParseIP(ipStr)
	if ip == nil {
		return nil, fmt.Errorf("invalid ip on accepted conn: %q", ipStr)
	}

	for _, cidr := range p.allowed {
		if cidr.Contains(ip) {
			return proxyproto.NewConn(c), nil
		}
	}
	return c, nil
}
<commit_msg>proxyproto changed its api slightly<commit_after>package genapi

import (
	"fmt"
	"net"

	"github.com/armon/go-proxyproto"
)

type proxyListener struct {
	net.Listener
	allowed []*net.IPNet
}

func newProxyListener(l net.Listener, allowed []string) (net.Listener, error) {
	a := make([]*net.IPNet, 0, len(allowed))
	for i := range allowed {
		if allowed[i] == "" {
			continue
		}
		_, parsed, err := net.ParseCIDR(allowed[i])
		if err != nil {
			return proxyListener{}, err
		}
		a = append(a, parsed)
	}
	if len(a) == 0 {
		return l, nil
	}
	return proxyListener{
		Listener: l,
		allowed:  a,
	}, nil
}

func (p proxyListener) Accept() (net.Conn, error) {
	c, err := p.Listener.Accept()
	if err != nil {
		return c, err
	}

	// The next two steps should never error. If they do something is crazy
	// wrong

	ipStr, _, err := net.SplitHostPort(c.RemoteAddr().String())
	if err != nil {
		return nil, fmt.Errorf("invalid RemoteAddr on accepted conn: %s", err)
	}

	ip := net.ParseIP(ipStr)
	if ip == nil {
		return nil, fmt.Errorf("invalid ip on accepted conn: %q", ipStr)
	}

	for _, cidr := range p.allowed {
		if cidr.Contains(ip) {
			return proxyproto.NewConn(c, 0), nil
		}
	}
	return c, nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"
)

var gitHubAPIBaseUrl = "https://api.github.com/"

type GitHubAPIKeyModel struct {
	Key                string
	RemainingUses      int
	RequestsPerHour    int
	RateLimitResetTime time.Time
}

func (apiKey *GitHubAPIKeyModel) incrementUsage(remainingRequests string, rateLimitResetTime string) {
	rateLimitResetInt, _ := strconv.ParseInt(rateLimitResetTime, 0, 64)
	rateLimitResetTimestamp := time.Unix(rateLimitResetInt, 0)
	remainingRequestsInt, _ := strconv.Atoi(remainingRequests)
	apiKey.RemainingUses = remainingRequestsInt
	apiKey.RateLimitResetTime = rateLimitResetTimestamp
	log.Printf("Decrementing APIKeyModel usage to %d uses \n", remainingRequestsInt)
}

func (apiKey *GitHubAPIKeyModel) prime() {
	githubTestURL := gitHubAPIBaseUrl + "repos/a/b" + "?access_token=" + apiKey.Key
	log.Printf("Preparing to prime APIKeyModel with key %s and url %s", apiKey.Key, githubTestURL)

	resp, err := http.Get(githubTestURL)
	if err != nil {
		log.Println("Could not prime APIKey, fatal error in github api request")
		log.Fatal(err)
	}
	defer resp.Body.Close()

	responseHeader := resp.Header
	remaingRequests := responseHeader.Get("X-RateLimit-Remaining")
	rateLimitResetTime := responseHeader.Get("X-RateLimit-Reset")
	apiKey.incrementUsage(remaingRequests, rateLimitResetTime)
}

func (apiKey *GitHubAPIKeyModel) reset() {
	apiKey.RemainingUses = 5000
}

func (apiKey *GitHubAPIKeyModel) print() {
	fmt.Printf("%+v", apiKey)
}
<commit_msg>Light refactor to centralize more responsability to API model<commit_after>package common

import (
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"
)

var gitHubAPIBaseUrl = "https://api.github.com/"

type GitHubAPIKeyModel struct {
	Key                string
	RemainingUses      int
	RequestsPerHour    int
	RateLimitResetTime time.Time
}

func (apiKey *GitHubAPIKeyModel) incrementUsageFromResponseHeader(respHeader http.Header) {
	remaingRequests := respHeader.Get("X-RateLimit-Remaining")
	rateLimitResetTime := respHeader.Get("X-RateLimit-Reset")

	remainingRequestsInt, _ := strconv.Atoi(remaingRequests)
	rateLimitResetInt, _ := strconv.ParseInt(rateLimitResetTime, 0, 64)
	rateLimitResetTimestamp := time.Unix(rateLimitResetInt, 0)

	apiKey.RemainingUses = remainingRequestsInt
	apiKey.RateLimitResetTime = rateLimitResetTimestamp

	log.Printf("Rate limit remaining requests %s \n", remaingRequests)
	log.Printf("Rate limit reset time %s \n", rateLimitResetTime)
	log.Printf("Decrementing APIKeyModel usage to %d uses \n", remainingRequestsInt)
}

// TODO consider passing url endpoint to prime, or maybe a enum for more accuracy when pinging GH
func (apiKey *GitHubAPIKeyModel) prime() {
	githubTestURL := gitHubAPIBaseUrl + "repos/a/b" + "?access_token=" + apiKey.Key
	log.Printf("Preparing to prime APIKeyModel with key %s and url %s", apiKey.Key, githubTestURL)

	resp, err := http.Get(githubTestURL)
	if err != nil {
		log.Println("Could not prime APIKey, fatal error in github api request")
		log.Fatal(err)
	}
	defer resp.Body.Close()

	apiKey.incrementUsageFromResponseHeader(resp.Header)
}

func (apiKey *GitHubAPIKeyModel) reset() {
	apiKey.RemainingUses = 5000
}

func (apiKey *GitHubAPIKeyModel) print() {
	fmt.Printf("%+v", apiKey)
}
<|endoftext|>


<commit_before>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugo

// CurrentVersion represents the current build version.
// This should be the only one.
var CurrentVersion = Version{
	Major:      0,
	Minor:      105,
	PatchLevel: 0,
	Suffix:     "-DEV",
}
<commit_msg>releaser: Bump versions for release of 0.104.1<commit_after>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugo

// CurrentVersion represents the current build version.
// This should be the only one.
var CurrentVersion = Version{
	Major:      0,
	Minor:      104,
	PatchLevel: 1,
	Suffix:     "",
}
<|endoftext|>


<commit_before>package i18n

import (
	"encoding/xml"
	"errors"
	"io/ioutil"
	"os"
	"sort"
)

type Xliff struct {
	XMLName xml.Name `xml:"xliff"`
	File    File     `xml:"file"`
	Version string   `xml:"version,attr"`
	Xmlns   string   `xml:"xmlns,attr"`
}
type File struct {
	Body           Body   `xml:"body"`
	Original       string `xml:"original,attr"`
	Datatype       string `xml:"datatype,attr"`
	SourceLanguage string `xml:"source-language,attr"`
	TargetLanguage string `xml:"target-language,attr,omitempty"`
}
type Body struct {
	TransList []TransUnit `xml:"trans-unit"`
}
type TransUnit struct {
	Source string   `xml:"source"`
	Target string   `xml:"target"`
	Note   []string `xml:"note"`
}

func XliffParser(fp *os.File) (Catalog, error) {
	defer fp.Close()
	xmldata, err := ioutil.ReadAll(fp)
	if err != nil {
		return nil, errors.New("Couldn't read file")
	}

	q := Xliff{}
	err = xml.Unmarshal(xmldata, &q)
	if err != nil {
		return nil, err
	}

	catalog := make(Catalog)

	for _, transunit := range q.File.Body.TransList {
		catalog[transunit.Source] = transunit.Target
	}

	return catalog, nil
}
func (b *Body) Add(source, target string) {
	transunit := TransUnit{
		Source: source,
		Target: target,
	}
	b.TransList = append(b.TransList, transunit)
}

type Alphabetically []TransUnit

func (a Alphabetically) Len() int           { return len(a) }
func (a Alphabetically) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Alphabetically) Less(i, j int) bool { return a[i].Source < a[j].Source }

func (b *Body) SortAlphabetically() {
	sort.Sort(Alphabetically(b.TransList))
}

func CreateXliff(catalog Catalog, sourceLanguage, targetLanguage string) []byte {

	xliff := Xliff{
		File: File{
			Body:           Body{},
			Original:       "file.ext",
			Datatype:       "plaintext",
			SourceLanguage: sourceLanguage,
			TargetLanguage: targetLanguage,
		},
		Version: "1.2",
		Xmlns:   "urn:oasis:names:tc:xliff:document:1.2",
	}
	for source, target := range catalog {
		xliff.File.Body.Add(source, target)
	}

	xliff.File.Body.SortAlphabetically()

	xml, err := xml.Marshal(xliff)
	if err != nil {
		panic(err)
	}

	return xml
}
<commit_msg>Add id to xliff trans-unit<commit_after>package i18n

import (
	"crypto/md5"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"sort"
)

type Xliff struct {
	XMLName xml.Name `xml:"xliff"`
	File    File     `xml:"file"`
	Version string   `xml:"version,attr"`
	Xmlns   string   `xml:"xmlns,attr"`
}
type File struct {
	Body           Body   `xml:"body"`
	Original       string `xml:"original,attr"`
	Datatype       string `xml:"datatype,attr"`
	SourceLanguage string `xml:"source-language,attr"`
	TargetLanguage string `xml:"target-language,attr,omitempty"`
}
type Body struct {
	TransList []TransUnit `xml:"trans-unit"`
}
type TransUnit struct {
	Id     string   `xml:"id,attr,omitempty"`
	Source string   `xml:"source"`
	Target string   `xml:"target"`
	Note   []string `xml:"note"`
}

func XliffParser(fp *os.File) (Catalog, error) {
	defer fp.Close()
	xmldata, err := ioutil.ReadAll(fp)
	if err != nil {
		return nil, errors.New("Couldn't read file")
	}

	q := Xliff{}
	err = xml.Unmarshal(xmldata, &q)
	if err != nil {
		return nil, err
	}

	catalog := make(Catalog)

	for _, transunit := range q.File.Body.TransList {
		catalog[transunit.Source] = transunit.Target
	}

	return catalog, nil
}
func (b *Body) Add(source, target string) {
	transunit := TransUnit{
		Id:     fmt.Sprintf("%x", md5.Sum([]byte(source))),
		Source: source,
		Target: target,
	}
	b.TransList = append(b.TransList, transunit)
}

type Alphabetically []TransUnit

func (a Alphabetically) Len() int           { return len(a) }
func (a Alphabetically) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Alphabetically) Less(i, j int) bool { return a[i].Source < a[j].Source }

func (b *Body) SortAlphabetically() {
	sort.Sort(Alphabetically(b.TransList))
}

func CreateXliff(catalog Catalog, sourceLanguage, targetLanguage string) []byte {

	xliff := Xliff{
		File: File{
			Body:           Body{},
			Original:       "file.ext",
			Datatype:       "plaintext",
			SourceLanguage: sourceLanguage,
			TargetLanguage: targetLanguage,
		},
		Version: "1.2",
		Xmlns:   "urn:oasis:names:tc:xliff:document:1.2",
	}
	for source, target := range catalog {
		xliff.File.Body.Add(source, target)
	}

	xliff.File.Body.SortAlphabetically()

	xml, err := xml.Marshal(xliff)
	if err != nil {
		panic(err)
	}

	return xml
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"

	"cloud.google.com/go/internal/version"
	"golang.org/x/net/context"
	pb "google.golang.org/genproto/googleapis/datastore/v1"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

// datastoreClient is a wrapper for the pb.DatastoreClient that includes gRPC
// metadata to be sent in each request for server-side traffic management.
type datastoreClient struct {
	// Embed so we still implement the DatastoreClient interface,
	// if the interface adds more methods.
	pb.DatastoreClient

	c  pb.DatastoreClient
	md metadata.MD
}

func newDatastoreClient(conn *grpc.ClientConn, projectID string) pb.DatastoreClient {
	return &datastoreClient{
		c: pb.NewDatastoreClient(conn),
		md: metadata.Pairs(
			resourcePrefixHeader, "projects/"+projectID,
			"x-goog-api-client", fmt.Sprintf("gl-go/%s gccl/%s grpc/", version.Go(), version.Repo)),
	}
}

func (dc *datastoreClient) Lookup(ctx context.Context, in *pb.LookupRequest, opts ...grpc.CallOption) (*pb.LookupResponse, error) {
	return dc.c.Lookup(metadata.NewOutgoingContext(ctx, dc.md), in, opts...)
}

func (dc *datastoreClient) RunQuery(ctx context.Context, in *pb.RunQueryRequest, opts ...grpc.CallOption) (*pb.RunQueryResponse, error) {
	return dc.c.RunQuery(metadata.NewOutgoingContext(ctx, dc.md), in, opts...)
}

func (dc *datastoreClient) BeginTransaction(ctx context.Context, in *pb.BeginTransactionRequest, opts ...grpc.CallOption) (*pb.BeginTransactionResponse, error) {
	return dc.c.BeginTransaction(metadata.NewOutgoingContext(ctx, dc.md), in, opts...)
}

func (dc *datastoreClient) Commit(ctx context.Context, in *pb.CommitRequest, opts ...grpc.CallOption) (*pb.CommitResponse, error) {
	return dc.c.Commit(metadata.NewOutgoingContext(ctx, dc.md), in, opts...)
}

func (dc *datastoreClient) Rollback(ctx context.Context, in *pb.RollbackRequest, opts ...grpc.CallOption) (*pb.RollbackResponse, error) {
	return dc.c.Rollback(metadata.NewOutgoingContext(ctx, dc.md), in, opts...)
}

func (dc *datastoreClient) AllocateIds(ctx context.Context, in *pb.AllocateIdsRequest, opts ...grpc.CallOption) (*pb.AllocateIdsResponse, error) {
	return dc.c.AllocateIds(metadata.NewOutgoingContext(ctx, dc.md), in, opts...)
}
<commit_msg>datastore: add retry<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"

	gax "github.com/googleapis/gax-go"

	"cloud.google.com/go/internal"
	"cloud.google.com/go/internal/version"
	"golang.org/x/net/context"
	pb "google.golang.org/genproto/googleapis/datastore/v1"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"
)

// datastoreClient is a wrapper for the pb.DatastoreClient that includes gRPC
// metadata to be sent in each request for server-side traffic management.
type datastoreClient struct {
	// Embed so we still implement the DatastoreClient interface,
	// if the interface adds more methods.
	pb.DatastoreClient

	c  pb.DatastoreClient
	md metadata.MD
}

func newDatastoreClient(conn *grpc.ClientConn, projectID string) pb.DatastoreClient {
	return &datastoreClient{
		c: pb.NewDatastoreClient(conn),
		md: metadata.Pairs(
			resourcePrefixHeader, "projects/"+projectID,
			"x-goog-api-client", fmt.Sprintf("gl-go/%s gccl/%s grpc/", version.Go(), version.Repo)),
	}
}

func (dc *datastoreClient) Lookup(ctx context.Context, in *pb.LookupRequest, opts ...grpc.CallOption) (res *pb.LookupResponse, err error) {
	err = dc.invoke(ctx, func(ctx context.Context) error {
		res, err = dc.c.Lookup(ctx, in, opts...)
		return err
	})
	return res, err
}

func (dc *datastoreClient) RunQuery(ctx context.Context, in *pb.RunQueryRequest, opts ...grpc.CallOption) (res *pb.RunQueryResponse, err error) {
	err = dc.invoke(ctx, func(ctx context.Context) error {
		res, err = dc.c.RunQuery(ctx, in, opts...)
		return err
	})
	return res, err
}

func (dc *datastoreClient) BeginTransaction(ctx context.Context, in *pb.BeginTransactionRequest, opts ...grpc.CallOption) (res *pb.BeginTransactionResponse, err error) {
	err = dc.invoke(ctx, func(ctx context.Context) error {
		res, err = dc.c.BeginTransaction(ctx, in, opts...)
		return err
	})
	return res, err
}

func (dc *datastoreClient) Commit(ctx context.Context, in *pb.CommitRequest, opts ...grpc.CallOption) (res *pb.CommitResponse, err error) {
	err = dc.invoke(ctx, func(ctx context.Context) error {
		res, err = dc.c.Commit(ctx, in, opts...)
		return err
	})
	return res, err
}

func (dc *datastoreClient) Rollback(ctx context.Context, in *pb.RollbackRequest, opts ...grpc.CallOption) (res *pb.RollbackResponse, err error) {
	err = dc.invoke(ctx, func(ctx context.Context) error {
		res, err = dc.c.Rollback(ctx, in, opts...)
		return err
	})
	return res, err
}

func (dc *datastoreClient) AllocateIds(ctx context.Context, in *pb.AllocateIdsRequest, opts ...grpc.CallOption) (res *pb.AllocateIdsResponse, err error) {
	err = dc.invoke(ctx, func(ctx context.Context) error {
		res, err = dc.c.AllocateIds(ctx, in, opts...)
		return err
	})
	return res, err
}

func (dc *datastoreClient) invoke(ctx context.Context, f func(ctx context.Context) error) error {
	ctx = metadata.NewOutgoingContext(ctx, dc.md)
	return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
		err = f(ctx)
		return !shouldRetry(err), err
	})
}

func shouldRetry(err error) bool {
	if err == nil {
		return false
	}
	s, ok := status.FromError(err)
	if !ok {
		return false
	}
	// See https://cloud.google.com/datastore/docs/concepts/errors.
	return s.Code() == codes.Unavailable || s.Code() == codes.DeadlineExceeded
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestConfig(t *testing.T) {
	config := GetConfig()
	if (config.App.Env != "development") {
		t.Error("Failed to initialize config")
	}
}
<commit_msg>be concise<commit_after>package main

import "testing"

func TestConfig(t *testing.T) {
	config := GetConfig()
	if (config.App.Env != "development") {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package conio

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"unicode/utf16"
	"unsafe"
)

const (
	RIGHT_ALT_PRESSED  = 1
	LEFT_ALT_PRESSED   = 2
	RIGHT_CTRL_PRESSED = 4
	LEFT_CTRL_PRESSED  = 8
	CTRL_PRESSED       = RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED
	ALT_PRESSED        = RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED
)

type inputRecordT struct {
	eventType uint16
	_         uint16
	// _KEY_EVENT_RECORD {
	bKeyDown         int32
	wRepeartCount    uint16
	wVirtualKeyCode  uint16
	wVirtualScanCode uint16
	unicodeChar      uint16
	// }
	dwControlKeyState uint32
}

var getConsoleMode = kernel32.NewProc("GetConsoleMode")
var setConsoleMode = kernel32.NewProc("SetConsoleMode")
var readConsoleInput = kernel32.NewProc("ReadConsoleInputW")

var hConin syscall.Handle

func init() {
	var err error
	hConin, err = syscall.Open("CONIN$", syscall.O_RDWR, 0)
	if err != nil {
		panic(fmt.Sprintf("conio: %v", err))
	}
}

type keyInfo struct {
	KeyCode    rune
	ScanCode   uint16
	ShiftState uint32
}

func ctrlCHandler(ch chan os.Signal) {
	for _ = range ch {
		keyBuffer = append(keyBuffer, keyInfo{3, 0, LEFT_CTRL_PRESSED})
	}
}

func DisableCtrlC() {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	go ctrlCHandler(ch)
}

func getKeys() []keyInfo {
	var numberOfEventsRead uint32
	var events [10]inputRecordT
	var orgConMode uint32

	result := make([]keyInfo, 0, 0)

	getConsoleMode.Call(uintptr(hConin),
		uintptr(unsafe.Pointer(&orgConMode)))
	setConsoleMode.Call(uintptr(hConin), 0)
	readConsoleInput.Call(
		uintptr(hConin),
		uintptr(unsafe.Pointer(&events[0])),
		uintptr(len(events)),
		uintptr(unsafe.Pointer(&numberOfEventsRead)))
	setConsoleMode.Call(uintptr(hConin), uintptr(orgConMode))
	for i := uint32(0); i < numberOfEventsRead; i++ {
		if events[i].eventType == KEY_EVENT && events[i].bKeyDown != 0 {
			var keycode rune
			if events[i].unicodeChar == 0 {
				keycode = rune(0)
			} else {
				keycode = utf16.Decode([]uint16{events[i].unicodeChar})[0]
			}
			result = append(result, keyInfo{
				keycode,
				events[i].wVirtualKeyCode,
				events[i].dwControlKeyState,
			})
		}
	}
	return result
}

var keyBuffer []keyInfo
var keyBufferRead = 0

func getKey() (rune, uint16, uint32) {
	for keyBuffer == nil || keyBufferRead >= len(keyBuffer) {
		keyBuffer = getKeys()
		keyBufferRead = 0
	}
	r := keyBuffer[keyBufferRead]
	keyBufferRead++
	return r.KeyCode, r.ScanCode, r.ShiftState
}

func GetKey() (rune, uint16, uint32) {
	code,scan,shift := getKey()
	if code < 0xDC00 || 0xDFFF < code {
		return code,scan,shift
	}
	code2,_,_ := getKey()
	return utf16.DecodeRune(code,code2),scan,shift
}

func GetCh() rune {
	for {
		ch, _, _ := GetKey()
		if ch != 0 {
			return ch
		}
	}
}
<commit_msg>go fmt X-(<commit_after>package conio

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"unicode/utf16"
	"unsafe"
)

const (
	RIGHT_ALT_PRESSED  = 1
	LEFT_ALT_PRESSED   = 2
	RIGHT_CTRL_PRESSED = 4
	LEFT_CTRL_PRESSED  = 8
	CTRL_PRESSED       = RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED
	ALT_PRESSED        = RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED
)

type inputRecordT struct {
	eventType uint16
	_         uint16
	// _KEY_EVENT_RECORD {
	bKeyDown         int32
	wRepeartCount    uint16
	wVirtualKeyCode  uint16
	wVirtualScanCode uint16
	unicodeChar      uint16
	// }
	dwControlKeyState uint32
}

var getConsoleMode = kernel32.NewProc("GetConsoleMode")
var setConsoleMode = kernel32.NewProc("SetConsoleMode")
var readConsoleInput = kernel32.NewProc("ReadConsoleInputW")

var hConin syscall.Handle

func init() {
	var err error
	hConin, err = syscall.Open("CONIN$", syscall.O_RDWR, 0)
	if err != nil {
		panic(fmt.Sprintf("conio: %v", err))
	}
}

type keyInfo struct {
	KeyCode    rune
	ScanCode   uint16
	ShiftState uint32
}

func ctrlCHandler(ch chan os.Signal) {
	for _ = range ch {
		keyBuffer = append(keyBuffer, keyInfo{3, 0, LEFT_CTRL_PRESSED})
	}
}

func DisableCtrlC() {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	go ctrlCHandler(ch)
}

func getKeys() []keyInfo {
	var numberOfEventsRead uint32
	var events [10]inputRecordT
	var orgConMode uint32

	result := make([]keyInfo, 0, 0)

	getConsoleMode.Call(uintptr(hConin),
		uintptr(unsafe.Pointer(&orgConMode)))
	setConsoleMode.Call(uintptr(hConin), 0)
	readConsoleInput.Call(
		uintptr(hConin),
		uintptr(unsafe.Pointer(&events[0])),
		uintptr(len(events)),
		uintptr(unsafe.Pointer(&numberOfEventsRead)))
	setConsoleMode.Call(uintptr(hConin), uintptr(orgConMode))
	for i := uint32(0); i < numberOfEventsRead; i++ {
		if events[i].eventType == KEY_EVENT && events[i].bKeyDown != 0 {
			var keycode rune
			if events[i].unicodeChar == 0 {
				keycode = rune(0)
			} else {
				keycode = utf16.Decode([]uint16{events[i].unicodeChar})[0]
			}
			result = append(result, keyInfo{
				keycode,
				events[i].wVirtualKeyCode,
				events[i].dwControlKeyState,
			})
		}
	}
	return result
}

var keyBuffer []keyInfo
var keyBufferRead = 0

func getKey() (rune, uint16, uint32) {
	for keyBuffer == nil || keyBufferRead >= len(keyBuffer) {
		keyBuffer = getKeys()
		keyBufferRead = 0
	}
	r := keyBuffer[keyBufferRead]
	keyBufferRead++
	return r.KeyCode, r.ScanCode, r.ShiftState
}

func GetKey() (rune, uint16, uint32) {
	code, scan, shift := getKey()
	if code < 0xDC00 || 0xDFFF < code {
		return code, scan, shift
	}
	code2, _, _ := getKey()
	return utf16.DecodeRune(code, code2), scan, shift
}

func GetCh() rune {
	for {
		ch, _, _ := GetKey()
		if ch != 0 {
			return ch
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"image"
	"image/jpeg"
	"image/png"
	"net/http"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	"github.com/coreos/go-log/log"
	"github.com/spf13/cobra"
)

var (
	baseDir  string
	threadId = regexp.MustCompile(`thread-(\d*)-`)
	imageId  = regexp.MustCompile(`([^\/]+)\.(png|jpg)`)
)

func worker(destDir string, linkChan chan string, wg *sync.WaitGroup) {
	defer wg.Done()

	for target := range linkChan {
		resp, err := http.Get(target)
		if err != nil {
			log.Println(err)
			continue
		}
		defer resp.Body.Close()

		m, _, err := image.Decode(resp.Body)
		if err != nil {
			log.Println(err)
			continue
		}

		// Ignore small images
		bounds := m.Bounds()
		if bounds.Size().X > 300 && bounds.Size().Y > 300 {
			imgInfo := imageId.FindStringSubmatch(target)
			out, _ := os.Create(destDir + "/" + imgInfo[1] + "." + imgInfo[2])
			defer out.Close()
			switch imgInfo[2] {
			case "jpg":
				jpeg.Encode(out, m, nil)
			case "png":
				png.Encode(out, m)
			}
		}
	}
}

func crawler(target string, workerNum int) {
	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1#thread_subject").Text()
	dir := fmt.Sprint("%v/%v - %v", baseDir, threadId.FindStringSubmatch(target)[1], title)

	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

// [todo] - Holy shit function, should refactor it!
func printGoogleResult(keyword string, page int) (hrefs []string) {
	client := &http.Client{}
	queryUrl := fmt.Sprintf("https://www.google.com.tw/search?espv=210&es_sm=119&q=%v+site:ck101.com&start=%v", keyword, page*10)
	req, err := http.NewRequest("GET", queryUrl, nil)
	if err != nil {
		panic(err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")

	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		panic(err)
	}

	hrefs = make([]string, 0)

	// Print result list
	doc.Find("li.g h3.r a").Each(func(i int, s *goquery.Selection) {
		title := s.Text()
		href, exist := s.Attr("href")
		if exist {
			hrefs = append(hrefs, href)
			fmt.Printf("[%v] %v\n", i, title)
		}
	})

	// Print pages
	for i := page - 3; i <= page+2; i++ {
		if i >= 0 {
			if i == page {
				fmt.Printf("[%v] ", i)
			} else {
				fmt.Printf("%v ", i)
			}
		}
	}
	fmt.Println()

	return hrefs
}

func main() {
	usr, _ := user.Current()
	baseDir = fmt.Sprintf("%v/Pictures/iloveck101", usr.HomeDir)

	var postUrl string
	var workerNum int

	rootCmd := &cobra.Command{
		Use:   "iloveck101",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			crawler(postUrl, workerNum)
		},
	}
	rootCmd.Flags().StringVarP(&postUrl, "url", "u", "http://ck101.com/thread-2876990-1-1.html", "Url of post")
	rootCmd.Flags().IntVarP(&workerNum, "worker", "w", 10, "Number of workers")

	searchCmd := &cobra.Command{
		Use:   "search",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			page := 0
			keyword := args[0]
			hrefs := printGoogleResult(keyword, page)

			scanner := bufio.NewScanner(os.Stdin)
			quit := false

			for !quit {
				fmt.Print("ck101> ")

				if !scanner.Scan() {
					break
				}

				line := scanner.Text()
				parts := strings.Split(line, " ")
				cmd := parts[0]
				args := parts[1:]

				switch cmd {
				case "quit":
					quit = true
				case "n":
					page = page + 1
					hrefs = printGoogleResult(keyword, page)
				case "p":
					if page > 0 {
						page = page - 1
					}
					hrefs = printGoogleResult(keyword, page)
				case "s":
					page = 0
					hrefs = printGoogleResult(args[0], page)
				case "d":
					index, err := strconv.ParseUint(args[0], 0, 0)
					if err != nil {
						fmt.Println(err)
						continue
					}
					if int(index) >= len(hrefs) {
						fmt.Println("Invalid index")
						continue
					}

					// Only support url with format ck101.com/thread-xxx
					if threadId.Match([]byte(hrefs[index])) {
						crawler(hrefs[index], 10)
						fmt.Println("Done!")
					} else {
						fmt.Println("Unsupport url:", hrefs[index])
					}
				default:
					fmt.Println("Unrecognized command:", cmd, args)
				}
			}
		},
	}

	rootCmd.AddCommand(searchCmd)
	rootCmd.Execute()
}
<commit_msg>Add log levels.<commit_after>package main

import (
	"bufio"
	"fmt"
	"image"
	"image/jpeg"
	"image/png"
	"net/http"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	logging "github.com/op/go-logging"
	"github.com/spf13/cobra"
)

var (
	baseDir  string
	threadId = regexp.MustCompile(`thread-(\d*)-`)
	imageId  = regexp.MustCompile(`([^\/]+)\.(png|jpg)`)
	log      = logging.MustGetLogger("iloveck101")
)

func worker(destDir string, linkChan chan string, wg *sync.WaitGroup) {
	defer wg.Done()

	for target := range linkChan {
		resp, err := http.Get(target)
		if err != nil {
			log.Debug("Http.Get\nerror: %s\ntarget: %s", err, target)
			continue
		}
		defer resp.Body.Close()

		m, _, err := image.Decode(resp.Body)
		if err != nil {
			log.Debug("image.Decode\nerror: %s\ntarget: %s", err, target)
			continue
		}

		// Ignore small images
		bounds := m.Bounds()
		if bounds.Size().X > 300 && bounds.Size().Y > 300 {
			imgInfo := imageId.FindStringSubmatch(target)
			out, err := os.Create(destDir + "/" + imgInfo[1] + "." + imgInfo[2])
			if err != nil {
				log.Debug("os.Create\nerror: %s", err)
				continue
			}
			defer out.Close()
			switch imgInfo[2] {
			case "jpg":
				jpeg.Encode(out, m, nil)
			case "png":
				png.Encode(out, m)
			}
		}
	}
}

func crawler(target string, workerNum int) {
	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1#thread_subject").Text()
	dir := fmt.Sprintf("%v/%v - %v", baseDir, threadId.FindStringSubmatch(target)[1], title)

	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

// [todo] - Holy shit function, should refactor it!
func printGoogleResult(keyword string, page int) (hrefs []string) {
	client := &http.Client{}
	queryUrl := fmt.Sprintf("https://www.google.com.tw/search?espv=210&es_sm=119&q=%v+site:ck101.com&start=%v", keyword, page*10)
	req, err := http.NewRequest("GET", queryUrl, nil)
	if err != nil {
		panic(err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")

	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		panic(err)
	}

	hrefs = make([]string, 0)

	// Print result list
	doc.Find("li.g h3.r a").Each(func(i int, s *goquery.Selection) {
		title := s.Text()
		href, exist := s.Attr("href")
		if exist {
			hrefs = append(hrefs, href)
			fmt.Printf("[%v] %v\n", i, title)
		}
	})

	// Print pages
	fmt.Print("Pages: ")
	for i := page - 3; i <= page+2; i++ {
		if i >= 0 {
			if i == page {
				fmt.Printf("[%v] ", i)
			} else {
				fmt.Printf("%v ", i)
			}
		}
	}
	fmt.Printf("(n:next, p:prev)\n")

	return hrefs
}

func main() {
	var format = logging.MustStringFormatter("%{level} %{message}")
	logging.SetFormatter(format)
	logging.SetLevel(logging.INFO, "iloveck101")

	usr, _ := user.Current()
	baseDir = fmt.Sprintf("%v/Pictures/iloveck101", usr.HomeDir)

	var postUrl string
	var workerNum int

	rootCmd := &cobra.Command{
		Use:   "iloveck101",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			crawler(postUrl, workerNum)
		},
	}
	rootCmd.Flags().StringVarP(&postUrl, "url", "u", "http://ck101.com/thread-2876990-1-1.html", "Url of post")
	rootCmd.Flags().IntVarP(&workerNum, "worker", "w", 25, "Number of workers")

	searchCmd := &cobra.Command{
		Use:   "search",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			page := 0
			keyword := args[0]
			hrefs := printGoogleResult(keyword, page)

			scanner := bufio.NewScanner(os.Stdin)
			quit := false

			for !quit {
				fmt.Print("ck101> ")

				if !scanner.Scan() {
					break
				}

				line := scanner.Text()
				parts := strings.Split(line, " ")
				cmd := parts[0]
				args := parts[1:]

				switch cmd {
				case "quit":
					quit = true
				case "n":
					page = page + 1
					hrefs = printGoogleResult(keyword, page)
				case "p":
					if page > 0 {
						page = page - 1
					}
					hrefs = printGoogleResult(keyword, page)
				case "s":
					page = 0
					hrefs = printGoogleResult(args[0], page)
				case "d":
					index, err := strconv.ParseUint(args[0], 0, 0)
					if err != nil {
						fmt.Println(err)
						continue
					}
					if int(index) >= len(hrefs) {
						fmt.Println("Invalid index")
						continue
					}

					// Only support url with format ck101.com/thread-xxx
					if threadId.Match([]byte(hrefs[index])) {
						crawler(hrefs[index], 25)
						fmt.Println("Done!")
					} else {
						fmt.Println("Unsupport url:", hrefs[index])
					}
				default:
					fmt.Println("Unrecognized command:", cmd, args)
				}
			}
		},
	}

	rootCmd.AddCommand(searchCmd)
	rootCmd.Execute()
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager_test

import (
	"fmt"
	"reflect"
	"testing"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
	"sigs.k8s.io/structured-merge-diff/v4/merge"
	"sigs.k8s.io/yaml"
)

// TestApplyUsingLastAppliedAnnotation tests that applying to an object
// created with the client-side apply last-applied annotation
// will not give conflicts
func TestApplyUsingLastAppliedAnnotation(t *testing.T) {
	f := NewDefaultTestFieldManager(schema.FromAPIVersionAndKind("apps/v1", "Deployment"))

	tests := []struct {
		lastApplied       []byte
		original          []byte
		applied           []byte
		fieldManager      string
		expectConflictSet *fieldpath.Set
	}{
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v1
      - name: my-c2
        image: my-image2
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app # missing from last-applied
spec:
  replicas: 100 # does not match last-applied
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v2 # does no match last-applied
      # note that second container in last-applied is missing
`),
			applied: []byte(`
# test conflicts due to fields not allowed by last-applied

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # NOT allowed: update label
spec:
  replicas: 333 # NOT allowed: update replicas
  selector:
    matchLabels:
      app: my-new-label # allowed: update label
  template:
    metadata:
      labels:
        app: my-new-label # allowed: update-label
    spec:
      containers:
      - name: my-c
        image: my-image-new # NOT allowed: update image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "replicas"),
				fieldpath.MakePathOrDie("spec", "template", "spec", "containers", fieldpath.KeyByFields("name", "my-c"), "image"),
			),
		},
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100 # does not match last applied
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label
spec:
  replicas: 3 # expect conflict
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
# applied object matches original

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
# test allowed update with no conflicts

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # update label
spec:
  replicas: 333 # update replicas
  selector:
    matchLabels:
      app: my-new-label # update label
  template:
    metadata:
      labels:
        app: my-new-label # update-label
    spec:
      containers:
      - name: my-c
        image: my-image
`),
		},
		{
			fieldManager: "not_kubectl",
			lastApplied: []byte(`
# expect conflicts because field manager is NOT kubectl

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v1
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100 # does not match last-applied
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v2 # does no match last-applied
`),
			applied: []byte(`
# test conflicts due to fields not allowed by last-applied

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # update label
spec:
  replicas: 333 # update replicas
  selector:
    matchLabels:
      app: my-new-label # update label
  template:
    metadata:
      labels:
        app: my-new-label # update-label
    spec:
      containers:
      - name: my-c
        image: my-image-new # update image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "replicas"),
				fieldpath.MakePathOrDie("spec", "selector", "matchLabels", "app"),
				fieldpath.MakePathOrDie("spec", "template", "metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "template", "spec", "containers", fieldpath.KeyByFields("name", "my-c"), "image"),
			),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
# test allowed update with no conflicts

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-new-image # update image
`),
		},
		{
			fieldManager: "not_kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`

# expect changes to fail because field manager is not kubectl

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # update label
spec:
  replicas: 3 # update replicas
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-new-image # update image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "replicas"),
				fieldpath.MakePathOrDie("spec", "template", "spec", "containers", fieldpath.KeyByFields("name", "my-c"), "image"),
			),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
		},
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			original: []byte(`
apiVersion: apps/v1 # expect conflict due to apiVersion mismatch with last-applied
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVerison: foo
kind: bar
spec: expect conflict due to invalid object
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
		{
			fieldManager: "kubectl",
			// last-applied is empty
			lastApplied: []byte{},
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
	}

	for i, test := range tests {
		t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
			f.Reset()

			originalObj := &unstructured.Unstructured{Object: map[string]interface{}{}}
			if err := yaml.Unmarshal(test.original, &originalObj.Object); err != nil {
				t.Errorf("error decoding YAML: %v", err)
			}

			if test.lastApplied == nil {
				test.lastApplied = test.original
			}
			if err := setLastAppliedFromEncoded(originalObj, test.lastApplied); err != nil {
				t.Errorf("failed to set last applied: %v", err)
			}

			if err := f.Update(originalObj, "test_client_side_apply"); err != nil {
				t.Errorf("failed to apply object: %v", err)
			}

			appliedObj := &unstructured.Unstructured{Object: map[string]interface{}{}}
			if err := yaml.Unmarshal(test.applied, &appliedObj.Object); err != nil {
				t.Errorf("error decoding YAML: %v", err)
			}

			err := f.Apply(appliedObj, test.fieldManager, false)

			if test.expectConflictSet == nil {
				if err != nil {
					t.Errorf("expected no error but got %v", err)
				}
				return
			}

			if err == nil || !apierrors.IsConflict(err) {
				t.Errorf("expected to get conflicts but got %v", err)
			}

			expectedConflicts := merge.Conflicts{}
			test.expectConflictSet.Iterate(func(p fieldpath.Path) {
				expectedConflicts = append(expectedConflicts, merge.Conflict{
					Manager: `{"manager":"test_client_side_apply","operation":"Update","apiVersion":"apps/v1"}`,
					Path:    p,
				})
			})
			expectedConflictErr := internal.NewConflictError(expectedConflicts)
			if !reflect.DeepEqual(expectedConflictErr, err) {
				t.Errorf("expected to get\n%+v\nbut got\n%+v", expectedConflictErr, err)
			}
		})
	}
}
<commit_msg>Tag LabelSelector with +structType=atomic<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager_test

import (
	"fmt"
	"reflect"
	"testing"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
	"sigs.k8s.io/structured-merge-diff/v4/merge"
	"sigs.k8s.io/yaml"
)

// TestApplyUsingLastAppliedAnnotation tests that applying to an object
// created with the client-side apply last-applied annotation
// will not give conflicts
func TestApplyUsingLastAppliedAnnotation(t *testing.T) {
	f := NewDefaultTestFieldManager(schema.FromAPIVersionAndKind("apps/v1", "Deployment"))

	tests := []struct {
		lastApplied       []byte
		original          []byte
		applied           []byte
		fieldManager      string
		expectConflictSet *fieldpath.Set
	}{
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v1
      - name: my-c2
        image: my-image2
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app # missing from last-applied
spec:
  replicas: 100 # does not match last-applied
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v2 # does no match last-applied
      # note that second container in last-applied is missing
`),
			applied: []byte(`
# test conflicts due to fields not allowed by last-applied

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # NOT allowed: update label
spec:
  replicas: 333 # NOT allowed: update replicas
  selector:
    matchLabels:
      app: my-new-label # allowed: update label
  template:
    metadata:
      labels:
        app: my-new-label # allowed: update-label
    spec:
      containers:
      - name: my-c
        image: my-image-new # NOT allowed: update image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "replicas"),
				fieldpath.MakePathOrDie("spec", "template", "spec", "containers", fieldpath.KeyByFields("name", "my-c"), "image"),
			),
		},
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100 # does not match last applied
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label
spec:
  replicas: 3 # expect conflict
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
# applied object matches original

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
# test allowed update with no conflicts

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # update label
spec:
  replicas: 333 # update replicas
  selector:
    matchLabels:
      app: my-new-label # update label
  template:
    metadata:
      labels:
        app: my-new-label # update-label
    spec:
      containers:
      - name: my-c
        image: my-image
`),
		},
		{
			fieldManager: "not_kubectl",
			lastApplied: []byte(`
# expect conflicts because field manager is NOT kubectl

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v1
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100 # does not match last-applied
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image-v2 # does no match last-applied
`),
			applied: []byte(`
# test conflicts due to fields not allowed by last-applied

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # update label
spec:
  replicas: 333 # update replicas
  selector:
    matchLabels:
      app: my-new-label # update label
  template:
    metadata:
      labels:
        app: my-new-label # update-label
    spec:
      containers:
      - name: my-c
        image: my-image-new # update image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "replicas"),
				fieldpath.MakePathOrDie("spec", "selector"), // selector is atomic
				fieldpath.MakePathOrDie("spec", "template", "metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "template", "spec", "containers", fieldpath.KeyByFields("name", "my-c"), "image"),
			),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`
# test allowed update with no conflicts

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-new-image # update image
`),
		},
		{
			fieldManager: "not_kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-app
spec:
  replicas: 100
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-image
`),
			applied: []byte(`

# expect changes to fail because field manager is not kubectl

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
  labels:
    app: my-new-label # update label
spec:
  replicas: 3 # update replicas
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
      - name: my-c
        image: my-new-image # update image
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("metadata", "labels", "app"),
				fieldpath.MakePathOrDie("spec", "replicas"),
				fieldpath.MakePathOrDie("spec", "template", "spec", "containers", fieldpath.KeyByFields("name", "my-c"), "image"),
			),
		},
		{
			fieldManager: "kubectl",
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
		},
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			original: []byte(`
apiVersion: apps/v1 # expect conflict due to apiVersion mismatch with last-applied
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
		{
			fieldManager: "kubectl",
			lastApplied: []byte(`
apiVerison: foo
kind: bar
spec: expect conflict due to invalid object
`),
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
		{
			fieldManager: "kubectl",
			// last-applied is empty
			lastApplied: []byte{},
			original: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 3
`),
			applied: []byte(`
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-deployment
spec:
  replicas: 100 # update replicas
`),
			expectConflictSet: fieldpath.NewSet(
				fieldpath.MakePathOrDie("spec", "replicas"),
			),
		},
	}

	for i, test := range tests {
		t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
			f.Reset()

			originalObj := &unstructured.Unstructured{Object: map[string]interface{}{}}
			if err := yaml.Unmarshal(test.original, &originalObj.Object); err != nil {
				t.Errorf("error decoding YAML: %v", err)
			}

			if test.lastApplied == nil {
				test.lastApplied = test.original
			}
			if err := setLastAppliedFromEncoded(originalObj, test.lastApplied); err != nil {
				t.Errorf("failed to set last applied: %v", err)
			}

			if err := f.Update(originalObj, "test_client_side_apply"); err != nil {
				t.Errorf("failed to apply object: %v", err)
			}

			appliedObj := &unstructured.Unstructured{Object: map[string]interface{}{}}
			if err := yaml.Unmarshal(test.applied, &appliedObj.Object); err != nil {
				t.Errorf("error decoding YAML: %v", err)
			}

			err := f.Apply(appliedObj, test.fieldManager, false)

			if test.expectConflictSet == nil {
				if err != nil {
					t.Errorf("expected no error but got %v", err)
				}
				return
			}

			if err == nil || !apierrors.IsConflict(err) {
				t.Errorf("expected to get conflicts but got %v", err)
			}

			expectedConflicts := merge.Conflicts{}
			test.expectConflictSet.Iterate(func(p fieldpath.Path) {
				expectedConflicts = append(expectedConflicts, merge.Conflict{
					Manager: `{"manager":"test_client_side_apply","operation":"Update","apiVersion":"apps/v1"}`,
					Path:    p,
				})
			})
			expectedConflictErr := internal.NewConflictError(expectedConflicts)
			if !reflect.DeepEqual(expectedConflictErr, err) {
				t.Errorf("expected to get\n%+v\nbut got\n%+v", expectedConflictErr, err)
			}
		})
	}
}
<|endoftext|>