<commit_before>//go:build linux && cgo
// +build linux,cgo

package devmapper

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/sirupsen/logrus"
)

type directLVMConfig struct {
	Device              string
	ThinpPercent        uint64
	ThinpMetaPercent    uint64
	AutoExtendPercent   uint64
	AutoExtendThreshold uint64
	MetaDataSize        string
}

var (
	errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
	errThinpPercentTooBig  = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
	errMissingSetupDevice  = errors.New("must provide device path in `dm.directlvm_device` in order to configure direct-lvm")
)

func validateLVMConfig(cfg directLVMConfig) error {
	if cfg.Device == "" {
		return errMissingSetupDevice
	}
	if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
		return errThinpPercentMissing
	}

	if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
		return errThinpPercentTooBig
	}
	return nil
}

func checkDevAvailable(dev string) error {
	lvmScan, err := exec.LookPath("lvmdiskscan")
	if err != nil {
		logrus.Debug("could not find lvmdiskscan")
		return nil
	}

	out, err := exec.Command(lvmScan).CombinedOutput()
	if err != nil {
		logrus.WithError(err).Error(string(out))
		return nil
	}

	if !bytes.Contains(out, []byte(dev)) {
		return fmt.Errorf("%s is not available for use with devicemapper", dev)
	}
	return nil
}

func checkDevInVG(dev string) error {
	pvDisplay, err := exec.LookPath("pvdisplay")
	if err != nil {
		logrus.Debug("could not find pvdisplay")
		return nil
	}

	out, err := exec.Command(pvDisplay, dev).CombinedOutput()
	if err != nil {
		logrus.WithError(err).Error(string(out))
		return nil
	}

	scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
	for scanner.Scan() {
		fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
		if len(fields) > 1 {
			// got "VG Name" line"
			vg := strings.TrimSpace(fields[1])
			if len(vg) > 0 {
				return fmt.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
			}
			logrus.Error(fields)
			break
		}
	}
	return nil
}

func checkDevHasFS(dev string) error {
	blkid, err := exec.LookPath("blkid")
	if err != nil {
		logrus.Debug("could not find blkid")
		return nil
	}

	out, err := exec.Command(blkid, dev).CombinedOutput()
	if err != nil {
		logrus.WithError(err).Error(string(out))
		return nil
	}

	fields := bytes.Fields(out)
	for _, f := range fields {
		kv := bytes.Split(f, []byte{'='})
		if bytes.Equal(kv[0], []byte("TYPE")) {
			v := bytes.Trim(kv[1], "\"")
			if len(v) > 0 {
				return fmt.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
			}
			return nil
		}
	}
	return nil
}

func verifyBlockDevice(dev string, force bool) error {
	absPath, err := filepath.Abs(dev)
	if err != nil {
		return fmt.Errorf("unable to get absolute path for %s: %s", dev, err)
	}
	realPath, err := filepath.EvalSymlinks(absPath)
	if err != nil {
		return fmt.Errorf("failed to canonicalise path for %s: %s", dev, err)
	}
	if err := checkDevAvailable(absPath); err != nil {
		logrus.Infof("block device '%s' not available, checking '%s'", absPath, realPath)
		if err := checkDevAvailable(realPath); err != nil {
			return fmt.Errorf("neither '%s' nor '%s' are in the output of lvmdiskscan, can't use device", absPath, realPath)
		}
	}
	if err := checkDevInVG(realPath); err != nil {
		return err
	}

	if force {
		return nil
	}

	if err := checkDevHasFS(realPath); err != nil {
		return err
	}
	return nil
}

func readLVMConfig(root string) (directLVMConfig, error) {
	var cfg directLVMConfig

	p := filepath.Join(root, "setup-config.json")
	b, err := ioutil.ReadFile(p)
	if err != nil {
		if os.IsNotExist(err) {
			return cfg, nil
		}
		return cfg, fmt.Errorf("reading existing setup config: %w", err)
	}

	// check if this is just an empty file, no need to produce a json error later if so
	if len(b) == 0 {
		return cfg, nil
	}

	err = json.Unmarshal(b, &cfg)
	return cfg, fmt.Errorf("unmarshaling previous device setup config: %w", err)
}

func writeLVMConfig(root string, cfg directLVMConfig) error {
	p := filepath.Join(root, "setup-config.json")
	b, err := json.Marshal(cfg)
	if err != nil {
		return fmt.Errorf("marshalling direct lvm config: %w", err)
	}
	if err := ioutil.WriteFile(p, b, 0600); err != nil {
		return fmt.Errorf("writing direct lvm config to file: %w", err)
	}
	return nil
}

func setupDirectLVM(cfg directLVMConfig) error {
	lvmProfileDir := "/etc/lvm/profile"
	binaries := []string{"pvcreate", "vgcreate", "lvcreate", "lvconvert", "lvchange", "thin_check"}

	for _, bin := range binaries {
		if _, err := exec.LookPath(bin); err != nil {
			return fmt.Errorf("looking up command `"+bin+"` while setting up direct lvm: %w", err)
		}
	}

	err := os.MkdirAll(lvmProfileDir, 0755)
	if err != nil {
		return fmt.Errorf("creating lvm profile directory: %w", err)
	}

	if cfg.AutoExtendPercent == 0 {
		cfg.AutoExtendPercent = 20
	}

	if cfg.AutoExtendThreshold == 0 {
		cfg.AutoExtendThreshold = 80
	}

	if cfg.ThinpPercent == 0 {
		cfg.ThinpPercent = 95
	}
	if cfg.ThinpMetaPercent == 0 {
		cfg.ThinpMetaPercent = 1
	}
	if cfg.MetaDataSize == "" {
		cfg.MetaDataSize = "128k"
	}

	out, err := exec.Command("pvcreate", "--metadatasize", cfg.MetaDataSize, "-f", cfg.Device).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	out, err = exec.Command("vgcreate", "storage", cfg.Device).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpool", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}
	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpoolmeta", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	out, err = exec.Command("lvconvert", "-y", "--zero", "n", "-c", "512K", "--thinpool", "storage/thinpool", "--poolmetadata", "storage/thinpoolmeta").CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
	err = ioutil.WriteFile(lvmProfileDir+"/storage-thinpool.profile", []byte(profile), 0600)
	if err != nil {
		return fmt.Errorf("writing storage thinp autoextend profile: %w", err)
	}

	out, err = exec.Command("lvchange", "--metadataprofile", "storage-thinpool", "storage/thinpool").CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %w", string(out), err)
	}
	return nil
}
<commit_msg>fix return error in device mapper<commit_after>//go:build linux && cgo
// +build linux,cgo

package devmapper

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/sirupsen/logrus"
)

type directLVMConfig struct {
	Device              string
	ThinpPercent        uint64
	ThinpMetaPercent    uint64
	AutoExtendPercent   uint64
	AutoExtendThreshold uint64
	MetaDataSize        string
}

var (
	errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
	errThinpPercentTooBig  = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
	errMissingSetupDevice  = errors.New("must provide device path in `dm.directlvm_device` in order to configure direct-lvm")
)

func validateLVMConfig(cfg directLVMConfig) error {
	if cfg.Device == "" {
		return errMissingSetupDevice
	}
	if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
		return errThinpPercentMissing
	}

	if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
		return errThinpPercentTooBig
	}
	return nil
}

func checkDevAvailable(dev string) error {
	lvmScan, err := exec.LookPath("lvmdiskscan")
	if err != nil {
		logrus.Debug("could not find lvmdiskscan")
		return nil
	}

	out, err := exec.Command(lvmScan).CombinedOutput()
	if err != nil {
		logrus.WithError(err).Error(string(out))
		return nil
	}

	if !bytes.Contains(out, []byte(dev)) {
		return fmt.Errorf("%s is not available for use with devicemapper", dev)
	}
	return nil
}

func checkDevInVG(dev string) error {
	pvDisplay, err := exec.LookPath("pvdisplay")
	if err != nil {
		logrus.Debug("could not find pvdisplay")
		return nil
	}

	out, err := exec.Command(pvDisplay, dev).CombinedOutput()
	if err != nil {
		logrus.WithError(err).Error(string(out))
		return nil
	}

	scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
	for scanner.Scan() {
		fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
		if len(fields) > 1 {
			// got "VG Name" line"
			vg := strings.TrimSpace(fields[1])
			if len(vg) > 0 {
				return fmt.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
			}
			logrus.Error(fields)
			break
		}
	}
	return nil
}

func checkDevHasFS(dev string) error {
	blkid, err := exec.LookPath("blkid")
	if err != nil {
		logrus.Debug("could not find blkid")
		return nil
	}

	out, err := exec.Command(blkid, dev).CombinedOutput()
	if err != nil {
		logrus.WithError(err).Error(string(out))
		return nil
	}

	fields := bytes.Fields(out)
	for _, f := range fields {
		kv := bytes.Split(f, []byte{'='})
		if bytes.Equal(kv[0], []byte("TYPE")) {
			v := bytes.Trim(kv[1], "\"")
			if len(v) > 0 {
				return fmt.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
			}
			return nil
		}
	}
	return nil
}

func verifyBlockDevice(dev string, force bool) error {
	absPath, err := filepath.Abs(dev)
	if err != nil {
		return fmt.Errorf("unable to get absolute path for %s: %s", dev, err)
	}
	realPath, err := filepath.EvalSymlinks(absPath)
	if err != nil {
		return fmt.Errorf("failed to canonicalise path for %s: %s", dev, err)
	}
	if err := checkDevAvailable(absPath); err != nil {
		logrus.Infof("block device '%s' not available, checking '%s'", absPath, realPath)
		if err := checkDevAvailable(realPath); err != nil {
			return fmt.Errorf("neither '%s' nor '%s' are in the output of lvmdiskscan, can't use device", absPath, realPath)
		}
	}
	if err := checkDevInVG(realPath); err != nil {
		return err
	}

	if force {
		return nil
	}

	if err := checkDevHasFS(realPath); err != nil {
		return err
	}
	return nil
}

func readLVMConfig(root string) (directLVMConfig, error) {
	var cfg directLVMConfig

	p := filepath.Join(root, "setup-config.json")
	b, err := ioutil.ReadFile(p)
	if err != nil {
		if os.IsNotExist(err) {
			return cfg, nil
		}
		return cfg, fmt.Errorf("reading existing setup config: %w", err)
	}

	// check if this is just an empty file, no need to produce a json error later if so
	if len(b) == 0 {
		return cfg, nil
	}
	if err := json.Unmarshal(b, &cfg); err != nil {
		return cfg, fmt.Errorf("unmarshaling previous device setup config: %w", err)
	}
	return cfg, nil
}

func writeLVMConfig(root string, cfg directLVMConfig) error {
	p := filepath.Join(root, "setup-config.json")
	b, err := json.Marshal(cfg)
	if err != nil {
		return fmt.Errorf("marshalling direct lvm config: %w", err)
	}
	if err := ioutil.WriteFile(p, b, 0600); err != nil {
		return fmt.Errorf("writing direct lvm config to file: %w", err)
	}
	return nil
}

func setupDirectLVM(cfg directLVMConfig) error {
	lvmProfileDir := "/etc/lvm/profile"
	binaries := []string{"pvcreate", "vgcreate", "lvcreate", "lvconvert", "lvchange", "thin_check"}

	for _, bin := range binaries {
		if _, err := exec.LookPath(bin); err != nil {
			return fmt.Errorf("looking up command `"+bin+"` while setting up direct lvm: %w", err)
		}
	}

	err := os.MkdirAll(lvmProfileDir, 0755)
	if err != nil {
		return fmt.Errorf("creating lvm profile directory: %w", err)
	}

	if cfg.AutoExtendPercent == 0 {
		cfg.AutoExtendPercent = 20
	}

	if cfg.AutoExtendThreshold == 0 {
		cfg.AutoExtendThreshold = 80
	}

	if cfg.ThinpPercent == 0 {
		cfg.ThinpPercent = 95
	}
	if cfg.ThinpMetaPercent == 0 {
		cfg.ThinpMetaPercent = 1
	}
	if cfg.MetaDataSize == "" {
		cfg.MetaDataSize = "128k"
	}

	out, err := exec.Command("pvcreate", "--metadatasize", cfg.MetaDataSize, "-f", cfg.Device).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	out, err = exec.Command("vgcreate", "storage", cfg.Device).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpool", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}
	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpoolmeta", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	out, err = exec.Command("lvconvert", "-y", "--zero", "n", "-c", "512K", "--thinpool", "storage/thinpool", "--poolmetadata", "storage/thinpoolmeta").CombinedOutput()
	if err != nil {
		return fmt.Errorf("%v: %w", string(out), err)
	}

	profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
	err = ioutil.WriteFile(lvmProfileDir+"/storage-thinpool.profile", []byte(profile), 0600)
	if err != nil {
		return fmt.Errorf("writing storage thinp autoextend profile: %w", err)
	}

	out, err = exec.Command("lvchange", "--metadataprofile", "storage-thinpool", "storage/thinpool").CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %w", string(out), err)
	}
	return nil
}
<|endoftext|>


<commit_before>package saga

import (
	"errors"
	"fmt"
)

/*
 * Data Structure representation of the current state of the Saga.
 */
type SagaState struct {
	sagaId string
	job    []byte

	//map of taskId to StartTask message logged
	taskStarted map[string]bool

	//map of taskId to EndTask message logged
	taskCompleted map[string]bool

	//map of taskId to results in EndTask message
	taskResults map[string][]byte

	//map of taskId to StartCompTask message logged
	compTaskStarted map[string]bool

	//map of taskId to EndCompTask message logged
	compTaskCompleted map[string]bool

	//map of taskId to reulst returned as part of EndCompTask message
	compTaskResults map[string][]byte

	//bool if AbortSaga message logged
	sagaAborted bool

	//bool if EndSaga message logged
	sagaCompleted bool
}

/*
 * Initialize a Default Empty Saga
 */
func initializeSagaState() *SagaState {
	return &SagaState{
		sagaId:            "",
		job:               nil,
		taskStarted:       make(map[string]bool),
		taskCompleted:     make(map[string]bool),
		taskResults:       make(map[string][]byte),
		compTaskStarted:   make(map[string]bool),
		compTaskCompleted: make(map[string]bool),
		compTaskResults:   make(map[string][]byte),
		sagaAborted:       false,
		sagaCompleted:     false,
	}
}

/*
 * Returns the Id of the Saga this state represents
 */
func (state *SagaState) SagaId() string {
	return state.sagaId
}

/*
 * Returns the Job associated with this Saga
 */
func (state *SagaState) Job() []byte {
	return state.job
}

/*
 * Returns true if the specified Task has been started,
 * fasle otherwise
 */
func (state *SagaState) IsTaskStarted(taskId string) bool {
	started, _ := state.taskStarted[taskId]
	return started
}

/*
 * Returns true if the specified Task has been completed,
 * fasle otherwise
 */
func (state *SagaState) IsTaskCompleted(taskId string) bool {
	completed, _ := state.taskCompleted[taskId]
	return completed
}

/*
 * Returns true if the specified Compensating Task has been started,
 * fasle otherwise
 */
func (state *SagaState) IsCompTaskStarted(taskId string) bool {
	started, _ := state.compTaskStarted[taskId]
	return started
}

/*
 * Returns true if the specified Compensating Task has been completed,
 * fasle otherwise
 */
func (state *SagaState) IsCompTaskCompleted(taskId string) bool {
	completed, _ := state.compTaskCompleted[taskId]
	return completed
}

/*
 * Returns true if this Saga has been Aborted, false otherwise
 */
func (state *SagaState) IsSagaAborted() bool {
	return state.sagaAborted
}

/*
 * Returns true if this Saga has been Completed, false otherwise
 */
func (state *SagaState) IsSagaCompleted() bool {
	return state.sagaCompleted
}

/*
 * Applies the supplied message to the supplied sagaState.  Does not mutate supplied Saga State
 * Instead returns a new SagaState which has the update applied
 *
 * Returns an InvalidSagaState Error if applying the message would result in an invalid Saga State
 * Returns an InvalidSagaMessage Error if the message is Invalid
 */
func updateSagaState(s *SagaState, msg sagaMessage) (*SagaState, error) {

	//first copy current state, and then apply update so we don't mutate the passed in SagaState
	state := copySagaState(s)

	if msg.sagaId != state.sagaId {
		return nil, fmt.Errorf("InvalidSagaState: sagaId %s & SagaMessage sagaId %s do not match", state.sagaId, msg.sagaId)
	}

	switch msg.msgType {

	case EndSaga:

		//A Successfully Completed Saga must have StartTask/EndTask pairs for all messages or
		//an aborted Saga must have StartTask/StartCompTask/EndCompTask pairs for all messages
		for taskId := range state.taskStarted {

			if state.sagaAborted {
				if !(state.compTaskStarted[taskId] && state.compTaskCompleted[taskId]) {
					return nil, errors.New(fmt.Sprintf("InvalidSagaState: End Saga Message cannot be applied to an aborted Saga where Task %s has not completed its compensating Tasks", taskId))
				}
			} else {
				if !state.taskCompleted[taskId] {
					return nil, errors.New(fmt.Sprintf("InvalidSagaState: End Saga Message cannot be applied to a Saga where Task %s has not completed", taskId))
				}
			}
		}

		state.sagaCompleted = true

	case AbortSaga:
		state.sagaAborted = true

	case StartTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		state.taskStarted[msg.taskId] = true

	case EndTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		// All EndTask Messages must have a preceding StartTask Message
		if !state.taskStarted[msg.taskId] {
			return nil, fmt.Errorf("Invalid Saga State: Cannot have a EndTask %s Message Before a StartTask %s Message", msg.taskId, msg.taskId)
		}

		state.taskCompleted[msg.taskId] = true

	case StartCompTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		//In order to apply compensating transactions a saga must first be aborted
		if !state.sagaAborted {
			return nil, fmt.Errorf("Invalid SagaState: Cannot have a StartCompTask %s Message when Saga has not been Aborted", msg.taskId)
		}

		// All StartCompTask Messages must have a preceding StartTask Message
		if !state.taskStarted[msg.taskId] {
			return nil, fmt.Errorf("Invalid Saga State: Cannot have a StartCompTask %s Message Before a StartTask %s Message", msg.taskId, msg.taskId)
		}

		state.compTaskStarted[msg.taskId] = true

	case EndCompTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		//in order to apply compensating transactions a saga must first be aborted
		if !state.sagaAborted {
			return nil, fmt.Errorf("Invalid SagaState: Cannot have a EndCompTask %s Message when Saga has not been Aborted", msg.taskId)
		}

		// All EndCompTask Messages must have a preceding StartTask Message
		if !state.taskStarted[msg.taskId] {
			return nil, fmt.Errorf("Invalid Saga State: Cannot have a StartCompTask %s Message Before a StartTask %s Message", msg.taskId, msg.taskId)
		}

		// All EndCompTask Messages must have a preceding StartCompTask Message
		if !state.compTaskStarted[msg.taskId] {
			return nil, fmt.Errorf("Invalid Saga State: Cannot have a EndCompTask %s Message Before a StartCompTaks %s Message", msg.taskId, msg.taskId)
		}

		state.compTaskCompleted[msg.taskId] = true
	}

	return state, nil
}

/*
 * Creates a deep copy of mutable saga state.  Does not Deepcopy
 * Job field since this is never mutated after creation
 */
func copySagaState(s *SagaState) *SagaState {

	newS := &SagaState{
		sagaId:        s.sagaId,
		sagaAborted:   s.sagaAborted,
		sagaCompleted: s.sagaCompleted,
	}

	newS.taskStarted = make(map[string]bool)
	for key, value := range s.taskStarted {
		newS.taskStarted[key] = value
	}

	newS.taskCompleted = make(map[string]bool)
	for key, value := range s.taskCompleted {
		newS.taskCompleted[key] = value
	}

	newS.compTaskStarted = make(map[string]bool)
	for key, value := range s.compTaskStarted {
		newS.compTaskStarted[key] = value
	}

	newS.compTaskCompleted = make(map[string]bool)
	for key, value := range s.compTaskCompleted {
		newS.compTaskCompleted[key] = value
	}

	//don't need to deep copy job, since its only set on create.
	newS.job = s.job

	return newS
}

/*
 * Validates that a SagaId Is valid. Returns error if valid, nil otherwise
 */
func validateSagaId(sagaId string) error {
	if sagaId == "" {
		return fmt.Errorf("Invalid Saga Message: sagaId cannot be the empty string")
	} else {
		return nil
	}
}

/*
 * Validates that a TaskId Is valid. Returns error if valid, nil otherwise
 */
func validateTaskId(taskId string) error {
	if taskId == "" {
		return fmt.Errorf("Invalid Saga Message: taskId cannot be the empty string")
	} else {
		return nil
	}
}

/*
 * Initialize a SagaState for the specified saga, and default data.
 */
func sagaStateFactory(sagaId string, job []byte) (*SagaState, error) {

	state := initializeSagaState()

	err := validateSagaId(sagaId)
	if err != nil {
		return nil, err
	}

	state.sagaId = sagaId
	state.job = job

	return state, nil
}
<commit_msg>compressing sagastate (#14)<commit_after>package saga

import (
	"errors"
	"fmt"
)

type flag byte

const (
	TaskStarted flag = 1 << iota
	TaskCompleted
	CompTaskStarted
	CompTaskCompleted
)

/*
 * Data Structure representation of the current state of the Saga.
 */
type SagaState struct {
	sagaId string
	job    []byte

	//map of taskId to Flag specifying task progress
	taskState map[string]flag

	//map of taskId to results in EndTask message
	taskResults map[string][]byte

	//map of taskId to reulst returned as part of EndCompTask message
	compTaskResults map[string][]byte

	//bool if AbortSaga message logged
	sagaAborted bool

	//bool if EndSaga message logged
	sagaCompleted bool
}

/*
 * Initialize a Default Empty Saga
 */
func initializeSagaState() *SagaState {
	return &SagaState{
		sagaId:          "",
		job:             nil,
		taskState:       make(map[string]flag),
		taskResults:     make(map[string][]byte),
		compTaskResults: make(map[string][]byte),
		sagaAborted:     false,
		sagaCompleted:   false,
	}
}

/*
 * Returns the Id of the Saga this state represents
 */
func (state *SagaState) SagaId() string {
	return state.sagaId
}

/*
 * Returns the Job associated with this Saga
 */
func (state *SagaState) Job() []byte {
	return state.job
}

/*
 * Returns true if the specified Task has been started,
 * fasle otherwise
 */
func (state *SagaState) IsTaskStarted(taskId string) bool {
	flags, _ := state.taskState[taskId]
	return flags&TaskStarted != 0
}

/*
 * Returns true if the specified Task has been completed,
 * fasle otherwise
 */
func (state *SagaState) IsTaskCompleted(taskId string) bool {
	flags, _ := state.taskState[taskId]
	return flags&TaskCompleted != 0
}

/*
 * Returns true if the specified Compensating Task has been started,
 * fasle otherwise
 */
func (state *SagaState) IsCompTaskStarted(taskId string) bool {
	flags, _ := state.taskState[taskId]
	return flags&CompTaskStarted != 0
}

/*
 * Returns true if the specified Compensating Task has been completed,
 * fasle otherwise
 */
func (state *SagaState) IsCompTaskCompleted(taskId string) bool {
	flags, _ := state.taskState[taskId]
	return flags&CompTaskCompleted != 0
}

/*
 * Returns true if this Saga has been Aborted, false otherwise
 */
func (state *SagaState) IsSagaAborted() bool {
	return state.sagaAborted
}

/*
 * Returns true if this Saga has been Completed, false otherwise
 */
func (state *SagaState) IsSagaCompleted() bool {
	return state.sagaCompleted
}

/*
 * Applies the supplied message to the supplied sagaState.  Does not mutate supplied Saga State
 * Instead returns a new SagaState which has the update applied to it
 *
 * Returns an Error if applying the message would result in an invalid Saga State
 */
func updateSagaState(s *SagaState, msg sagaMessage) (*SagaState, error) {

	//first copy current state, and then apply update so we don't mutate the passed in SagaState
	state := copySagaState(s)

	if msg.sagaId != state.sagaId {
		return nil, fmt.Errorf("InvalidSagaMessage: sagaId %s & SagaMessage sagaId %s do not match", state.sagaId, msg.sagaId)
	}

	switch msg.msgType {

	case EndSaga:

		//A Successfully Completed Saga must have StartTask/EndTask pairs for all messages or
		//an aborted Saga must have StartTask/StartCompTask/EndCompTask pairs for all messages
		for taskId := range state.taskState {

			if state.sagaAborted {
				if !(state.IsCompTaskStarted(taskId) && state.IsCompTaskCompleted(taskId)) {
					return nil, errors.New(fmt.Sprintf("InvalidSagaState: End Saga Message cannot be applied to an aborted Saga where Task %s has not completed its compensating Tasks", taskId))
				}
			} else {
				if !state.IsTaskCompleted(taskId) {
					return nil, errors.New(fmt.Sprintf("InvalidSagaState: End Saga Message cannot be applied to a Saga where Task %s has not completed", taskId))
				}
			}
		}

		state.sagaCompleted = true

	case AbortSaga:
		state.sagaAborted = true

	case StartTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		state.taskState[msg.taskId] = TaskStarted

	case EndTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		// All EndTask Messages must have a preceding StartTask Message
		if !state.IsTaskStarted(msg.taskId) {
			return nil, fmt.Errorf("InvalidSagaState: Cannot have a EndTask %s Message Before a StartTask %s Message", msg.taskId, msg.taskId)
		}

		state.taskState[msg.taskId] = state.taskState[msg.taskId] | TaskCompleted

	case StartCompTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		//In order to apply compensating transactions a saga must first be aborted
		if !state.IsSagaAborted() {
			return nil, fmt.Errorf("InvalidSagaState: Cannot have a StartCompTask %s Message when Saga has not been Aborted", msg.taskId)
		}

		// All StartCompTask Messages must have a preceding StartTask Message
		if !state.IsTaskStarted(msg.taskId) {
			return nil, fmt.Errorf("InvalidSaga State: Cannot have a StartCompTask %s Message Before a StartTask %s Message", msg.taskId, msg.taskId)
		}

		state.taskState[msg.taskId] = state.taskState[msg.taskId] | CompTaskStarted

	case EndCompTask:
		err := validateTaskId(msg.taskId)
		if err != nil {
			return nil, err
		}

		//in order to apply compensating transactions a saga must first be aborted
		if !state.IsSagaAborted() {
			return nil, fmt.Errorf("InvalidSagaState: Cannot have a EndCompTask %s Message when Saga has not been Aborted", msg.taskId)
		}

		// All EndCompTask Messages must have a preceding StartTask Message
		if !state.IsTaskStarted(msg.taskId) {
			return nil, fmt.Errorf("InvalidSagaState: Cannot have a StartCompTask %s Message Before a StartTask %s Message", msg.taskId, msg.taskId)
		}

		// All EndCompTask Messages must have a preceding StartCompTask Message
		if !state.IsCompTaskStarted(msg.taskId) {
			return nil, fmt.Errorf("InvalidSagaState: Cannot have a EndCompTask %s Message Before a StartCompTaks %s Message", msg.taskId, msg.taskId)
		}

		state.taskState[msg.taskId] = state.taskState[msg.taskId] | CompTaskCompleted
	}

	return state, nil
}

/*
 * Creates a deep copy of mutable saga state.  Does not Deepcopy
 * Job field since this is never mutated after creation
 */
func copySagaState(s *SagaState) *SagaState {

	newS := &SagaState{
		sagaId:        s.sagaId,
		sagaAborted:   s.sagaAborted,
		sagaCompleted: s.sagaCompleted,
	}

	newS.taskState = make(map[string]flag)
	for key, value := range s.taskState {
		newS.taskState[key] = value
	}

	//don't need to deep copy job, since its only set on create.
	newS.job = s.job

	return newS
}

/*
 * Validates that a SagaId Is valid. Returns error if valid, nil otherwise
 */
func validateSagaId(sagaId string) error {
	if sagaId == "" {
		return fmt.Errorf("Invalid Saga Message: sagaId cannot be the empty string")
	} else {
		return nil
	}
}

/*
 * Validates that a TaskId Is valid. Returns error if valid, nil otherwise
 */
func validateTaskId(taskId string) error {
	if taskId == "" {
		return fmt.Errorf("Invalid Saga Message: taskId cannot be the empty string")
	} else {
		return nil
	}
}

/*
 * Initialize a SagaState for the specified saga, and default data.
 */
func sagaStateFactory(sagaId string, job []byte) (*SagaState, error) {

	state := initializeSagaState()

	err := validateSagaId(sagaId)
	if err != nil {
		return nil, err
	}

	state.sagaId = sagaId
	state.job = job

	return state, nil
}
<|endoftext|>


<commit_before>package srvconf

import (
	"crypto/tls"
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"time"

	pb "github.com/pandemicsyn/syndicate/api/proto"
	"github.com/shirou/gopsutil/disk"
	"github.com/shirou/gopsutil/mem"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

var (
	ErrSRVLookupFailed = errors.New("srv lookup failed")
)

// lookup returned records are sorted by priority and randomized by weight within a priority.
func lookup(service string) ([]*net.SRV, error) {
	_, addrs, err := net.LookupSRV("", "", service)
	if err != nil {
		log.Println("srv:", service)
		log.Println(err)
		return nil, ErrSRVLookupFailed
	}
	return addrs, nil
}

type SRVLoader struct {
	Record       string
	SyndicateURL string
}

func GetHardwareProfile() (*pb.HardwareProfile, error) {
	v, err := mem.VirtualMemory()
	if err != nil {
		return &pb.HardwareProfile{}, err
	}
	d, err := disk.DiskPartitions(true)
	if err != nil {
		return &pb.HardwareProfile{}, err
	}
	hw := &pb.HardwareProfile{
		Disks:    make([]*pb.Disk, len(d)),
		Cpus:     int64(runtime.NumCPU()),
		Memtotal: v.Total,
		Memfree:  v.Free,
	}
	for k := range d {
		fmt.Println(d[k].Mountpoint)
		usage, err := disk.DiskUsage(hw.Disks[k].Path)
		if err != nil {
			continue
		}
		hw.Disks[k] = &pb.Disk{
			Path:   d[k].Mountpoint,
			Device: d[k].Device,
		}
		hw.Disks[k].Size = usage.Total
		hw.Disks[k].Used = usage.Used
	}
	return hw, nil
}

func (s *SRVLoader) getConfig() (*pb.NodeConfig, error) {
	nconfig := &pb.NodeConfig{}
	var opts []grpc.DialOption
	var creds credentials.TransportAuthenticator
	creds = credentials.NewTLS(&tls.Config{
		InsecureSkipVerify: true,
	})
	opts = append(opts, grpc.WithTransportCredentials(creds))
	conn, err := grpc.Dial(s.SyndicateURL, opts...)
	if err != nil {
		return nconfig, fmt.Errorf("Failed to dial ring server for config: %s", err)
	}
	defer conn.Close()

	client := pb.NewSyndicateClient(conn)

	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)

	rr := &pb.RegisterRequest{}
	rr.Hostname, _ = os.Hostname()
	addrs, _ := net.InterfaceAddrs()
	for k, _ := range addrs {
		rr.Addrs = append(rr.Addrs, addrs[k].String())
	}
	rr.Hardware, err = GetHardwareProfile()
	if err != nil {
		return nconfig, err
	}
	rr.Tiers = []string{rr.Hostname}

	nconfig, err = client.RegisterNode(ctx, rr)
	return nconfig, err
}

func (s *SRVLoader) Load() (nodeconfig *pb.NodeConfig, err error) {
	if s.SyndicateURL == "" {
		serviceAddrs, err := lookup(s.Record)
		if err != nil {
			return &pb.NodeConfig{}, err
		}
		s.SyndicateURL = fmt.Sprintf("%s:%d", serviceAddrs[0].Target, serviceAddrs[0].Port)
	}
	nodeconfig, err = s.getConfig()
	if err != nil {
		if err == ErrSRVLookupFailed {
			return nodeconfig, err
		}
		return nodeconfig, err
	}
	return nodeconfig, nil
}
<commit_msg>Bug fix<commit_after>package srvconf

import (
	"crypto/tls"
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"time"

	pb "github.com/pandemicsyn/syndicate/api/proto"
	"github.com/shirou/gopsutil/disk"
	"github.com/shirou/gopsutil/mem"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

var (
	ErrSRVLookupFailed = errors.New("srv lookup failed")
)

// lookup returned records are sorted by priority and randomized by weight within a priority.
func lookup(service string) ([]*net.SRV, error) {
	_, addrs, err := net.LookupSRV("", "", service)
	if err != nil {
		log.Println("srv:", service)
		log.Println(err)
		return nil, ErrSRVLookupFailed
	}
	return addrs, nil
}

type SRVLoader struct {
	Record       string
	SyndicateURL string
}

func GetHardwareProfile() (*pb.HardwareProfile, error) {
	v, err := mem.VirtualMemory()
	if err != nil {
		return &pb.HardwareProfile{}, err
	}
	d, err := disk.DiskPartitions(true)
	if err != nil {
		return &pb.HardwareProfile{}, err
	}
	hw := &pb.HardwareProfile{
		Disks:    make([]*pb.Disk, len(d)),
		Cpus:     int64(runtime.NumCPU()),
		Memtotal: v.Total,
		Memfree:  v.Free,
	}
	for k := range d {
		usage, err := disk.DiskUsage(d[k].Mountpoint)
		if err != nil {
			continue
		}
		hw.Disks[k] = &pb.Disk{
			Path:   d[k].Mountpoint,
			Device: d[k].Device,
		}
		hw.Disks[k].Size = usage.Total
		hw.Disks[k].Used = usage.Used
	}
	return hw, nil
}

func (s *SRVLoader) getConfig() (*pb.NodeConfig, error) {
	nconfig := &pb.NodeConfig{}
	var opts []grpc.DialOption
	var creds credentials.TransportAuthenticator
	creds = credentials.NewTLS(&tls.Config{
		InsecureSkipVerify: true,
	})
	opts = append(opts, grpc.WithTransportCredentials(creds))
	conn, err := grpc.Dial(s.SyndicateURL, opts...)
	if err != nil {
		return nconfig, fmt.Errorf("Failed to dial ring server for config: %s", err)
	}
	defer conn.Close()

	client := pb.NewSyndicateClient(conn)

	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)

	rr := &pb.RegisterRequest{}
	rr.Hostname, _ = os.Hostname()
	addrs, _ := net.InterfaceAddrs()
	for k, _ := range addrs {
		rr.Addrs = append(rr.Addrs, addrs[k].String())
	}
	rr.Hardware, err = GetHardwareProfile()
	if err != nil {
		return nconfig, err
	}
	rr.Tiers = []string{rr.Hostname}

	nconfig, err = client.RegisterNode(ctx, rr)
	return nconfig, err
}

func (s *SRVLoader) Load() (nodeconfig *pb.NodeConfig, err error) {
	if s.SyndicateURL == "" {
		serviceAddrs, err := lookup(s.Record)
		if err != nil {
			return &pb.NodeConfig{}, err
		}
		s.SyndicateURL = fmt.Sprintf("%s:%d", serviceAddrs[0].Target, serviceAddrs[0].Port)
	}
	nodeconfig, err = s.getConfig()
	if err != nil {
		if err == ErrSRVLookupFailed {
			return nodeconfig, err
		}
		return nodeconfig, err
	}
	return nodeconfig, nil
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package helper

import (
	"regexp"
)

var (
	temporaryFileRegexp  *regexp.Regexp
	persistentFileRegexp *regexp.Regexp
	anyFileRegexp        *regexp.Regexp
	placeholderRegexp = regexp.MustCompile(`\[storage:[\d]+\]`)
)

func init() {
	Init()
}

func Init() {
	ruleEnd := ExtensionRegexpEnd()
	temporaryFileRegexp = regexp.MustCompile(UploadURLPath + `[\w-]+/0/[\w]+` + ruleEnd)
	persistentFileRegexp = regexp.MustCompile(UploadURLPath + `[\w-]+/([^0]|[0-9]{2,})/[\w]+` + ruleEnd)
	anyFileRegexp = regexp.MustCompile(UploadURLPath + `[\w-]+/([\w-]+/)+[\w-]+` + ruleEnd)
}

// ParseTemporaryFileName 从文本中解析出临时文件名称
var ParseTemporaryFileName = func(s string) []string {
	files := temporaryFileRegexp.FindAllString(s, -1)
	return files
}

// ParsePersistentFileName 从文本中解析出正式文件名称
var ParsePersistentFileName = func(s string) []string {
	files := persistentFileRegexp.FindAllString(s, -1)
	return files
}

// ParseAnyFileName 从文本中解析出任意上传文件名称
var ParseAnyFileName = func(s string) []string {
	files := anyFileRegexp.FindAllString(s, -1)
	return files
}

// ReplaceAnyFileName 从文本中替换任意上传文件名称
var ReplaceAnyFileName = func(s string, repl func(string) string) string {
	return anyFileRegexp.ReplaceAllStringFunc(s, repl)
}

// ReplacePlaceholder 从文本中替换占位符
var ReplacePlaceholder = func(s string, repl func(string) string) string {
	return placeholderRegexp.ReplaceAllStringFunc(s, func(find string) string{
		id := find[9:len(find)-1]
		return repl(id)
	})
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package helper

import (
	"regexp"
)

var (
	temporaryFileRegexp  *regexp.Regexp
	persistentFileRegexp *regexp.Regexp
	anyFileRegexp        *regexp.Regexp
	placeholderRegexp    = regexp.MustCompile(`\[storage:[\d]+\]`)
)

func init() {
	Init(UploadURLPath)
}

func Init(pathPrefix string) {
	ruleEnd := ExtensionRegexpEnd()
	temporaryFileRegexp = regexp.MustCompile(pathPrefix + `[\w-]+/0/[\w]+` + ruleEnd)
	persistentFileRegexp = regexp.MustCompile(pathPrefix + `[\w-]+/([^0]|[0-9]{2,})/[\w]+` + ruleEnd)
	anyFileRegexp = regexp.MustCompile(pathPrefix + `[\w-]+/([\w-]+/)+[\w-]+` + ruleEnd)
}

// ParseTemporaryFileName 从文本中解析出临时文件名称
var ParseTemporaryFileName = func(s string) []string {
	files := temporaryFileRegexp.FindAllString(s, -1)
	return files
}

// ParsePersistentFileName 从文本中解析出正式文件名称
var ParsePersistentFileName = func(s string) []string {
	files := persistentFileRegexp.FindAllString(s, -1)
	return files
}

// ParseAnyFileName 从文本中解析出任意上传文件名称
var ParseAnyFileName = func(s string) []string {
	files := anyFileRegexp.FindAllString(s, -1)
	return files
}

// ReplaceAnyFileName 从文本中替换任意上传文件名称
var ReplaceAnyFileName = func(s string, repl func(string) string) string {
	return anyFileRegexp.ReplaceAllStringFunc(s, repl)
}

// ReplacePlaceholder 从文本中替换占位符
var ReplacePlaceholder = func(s string, repl func(string) string) string {
	return placeholderRegexp.ReplaceAllStringFunc(s, func(find string) string {
		id := find[9 : len(find)-1]
		return repl(id)
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"strconv"
	"strings"
)

var version string = "0.15.0"

func Full() string {
	return version
}

func getSubVersion(v string, position int) int64 {
	arr := strings.Split(v, ".")
	if len(arr) < 3 {
		return 0
	}
	res, _ := strconv.ParseInt(arr[position], 10, 64)
	return res
}

func Proto(v string) int64 {
	return getSubVersion(v, 0)
}

func Major(v string) int64 {
	return getSubVersion(v, 1)
}

func Minor(v string) int64 {
	return getSubVersion(v, 2)
}

// add every case there if server will not accept client's protocol and return false
func Compat(client string) (ok bool, msg string) {
	if LessThan(client, "0.10.0") {
		return false, "Please upgrade your frpc version to at least 0.10.0"
	}
	return true, ""
}

func LessThan(client string, server string) bool {
	vc := Proto(client)
	vs := Proto(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Major(client)
	vs = Major(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Minor(client)
	vs = Minor(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}
	return false
}
<commit_msg>version: to v0.15.1<commit_after>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"strconv"
	"strings"
)

var version string = "0.15.1"

func Full() string {
	return version
}

func getSubVersion(v string, position int) int64 {
	arr := strings.Split(v, ".")
	if len(arr) < 3 {
		return 0
	}
	res, _ := strconv.ParseInt(arr[position], 10, 64)
	return res
}

func Proto(v string) int64 {
	return getSubVersion(v, 0)
}

func Major(v string) int64 {
	return getSubVersion(v, 1)
}

func Minor(v string) int64 {
	return getSubVersion(v, 2)
}

// add every case there if server will not accept client's protocol and return false
func Compat(client string) (ok bool, msg string) {
	if LessThan(client, "0.10.0") {
		return false, "Please upgrade your frpc version to at least 0.10.0"
	}
	return true, ""
}

func LessThan(client string, server string) bool {
	vc := Proto(client)
	vs := Proto(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Major(client)
	vs = Major(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Minor(client)
	vs = Minor(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package k8sTest

import (
	"fmt"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var _ = Describe("K8sUpdates", func() {

	// This test runs 8 steps as following:
	// 1 - delete all pods. Clean cilium, this can be, and should be achieved by
	// `clean-cilium-state: "true"` option that we have in configmap
	// 2 - install cilium `cilium:v1.1.4`
	// 3 - make endpoints talk with each other with policy
	// 4 - upgrade cilium to `k8s1:5000/cilium/cilium-dev:latest`
	// 5 - make endpoints talk with each other with policy
	// 6 - downgrade cilium to `cilium:v1.1.4`
	// 7 - make endpoints talk with each other with policy
	// 8 - delete all pods. Clean cilium, this can be, and should be achieved by
	// `clean-cilium-state: "true"` option that we have in configmap.
	// This makes sure the upgrade tests won't affect any other test
	// 9 - re install cilium:latest image for remaining tests.

	var (
		kubectl *helpers.Kubectl

		cleanupCallback = func() { return }
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		_ = kubectl.Delete(helpers.DNSDeployment())

		// Delete kube-dns because if not will be a restore the old endpoints
		// from master instead of create the new ones.
		_ = kubectl.DeleteResource(
			"deploy", fmt.Sprintf("-n %s kube-dns", helpers.KubeSystemNamespace))

		// Sometimes PolicyGen has a lot of pods running around without delete
		// it. Using this we are sure that we delete before this test start
		kubectl.Exec(fmt.Sprintf(
			"%s delete --all pods,svc,cnp -n %s", helpers.KubectlCmd, helpers.DefaultNamespace))

		kubectl.DeleteETCDOperator()

		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		_ = kubectl.Apply(helpers.DNSDeployment())
	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace, "cilium endpoint list")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		cleanupCallback()
		ExpectAllPodsTerminated(kubectl)
	})

	It("Tests upgrade and downgrade from a Cilium stable image to master", func() {
		var assertUpgradeSuccessful func()
		assertUpgradeSuccessful, cleanupCallback =
			InstallAndValidateCiliumUpgrades(kubectl, helpers.CiliumStableVersion, helpers.CiliumDeveloperImage)
		assertUpgradeSuccessful()
	})
})

// InstallAndValidateCiliumUpgrades installs and tests if the oldVersion can be
// upgrade to the newVersion and if the newVersion can be downgraded to the
// oldVersion.  It returns two callbacks, the first one is the assertfunction
// that need to run, and the second one are the cleanup actions
func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldVersion, newVersion string) (func(), func()) {
	canRun, err := helpers.CanRunK8sVersion(oldVersion, helpers.GetCurrentK8SEnv())
	ExpectWithOffset(1, err).To(BeNil(), "Unable to get k8s constraints for %s", oldVersion)
	if !canRun {
		Skip(fmt.Sprintf(
			"Cilium %q is not supported in K8s %q. Skipping upgrade/downgrade tests.",
			oldVersion, helpers.GetCurrentK8SEnv()))
		return func() {}, func() {}
	}

	demoPath := helpers.ManifestGet("demo.yaml")
	l7Policy := helpers.ManifestGet("l7-policy.yaml")
	apps := []string{helpers.App1, helpers.App2, helpers.App3}
	app1Service := "app1-service"

	cleanupCallback := func() {
		kubectl.Delete(l7Policy)
		kubectl.Delete(demoPath)

		// make sure that Kubedns is deleted correctly
		_ = kubectl.Delete(helpers.DNSDeployment())

		kubectl.DeleteETCDOperator()

		ExpectAllPodsTerminated(kubectl)

		// make sure we clean everything up before doing any other test
		err := kubectl.CiliumInstall(
			helpers.CiliumDefaultDSPatch,
			"cilium-cm-patch-clean-cilium-state.yaml",
		)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be deployed", newVersion)
		err = kubectl.WaitForCiliumInitContainerToFinish()
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be clean up environment", newVersion)

		_ = kubectl.DeleteResource(
			"ds", fmt.Sprintf("-n %s cilium", helpers.KubeSystemNamespace))
	}

	testfunc := func() {
		// Making sure that we deleted the  cilium ds. No assert message
		// because maybe is not present
		_ = kubectl.DeleteResource("ds", fmt.Sprintf("-n %s cilium", helpers.KubeSystemNamespace))

		// Delete kube-dns because if not will be a restore the old endpoints
		// from master instead of create the new ones.
		_ = kubectl.Delete(helpers.DNSDeployment())

		ExpectAllPodsTerminated(kubectl)

		By("Installing a cleaning state of Cilium")
		err = kubectl.CiliumInstallVersion(
			helpers.CiliumDefaultDSPatch,
			"cilium-cm-patch-clean-cilium-state.yaml",
			oldVersion,
		)
		Expect(err).To(BeNil(), "Cilium %q was not able to be deployed", oldVersion)

		By("Installing kube-dns")
		_ = kubectl.Apply(helpers.DNSDeployment())

		// Deploy the etcd operator
		By("Deploying etcd-operator")
		err = kubectl.DeployETCDOperator()
		Expect(err).To(BeNil(), "Unable to deploy etcd operator")

		// Cilium is only ready if kvstore is ready, the kvstore is ready if
		// kube-dns is running.
		By("Cilium %q is installed and running", oldVersion)
		ExpectCiliumReady(kubectl)

		ExpectETCDOperatorReady(kubectl)

		By("Installing Microscope")
		microscopeErr, microscopeCancel := kubectl.MicroscopeStart()
		ExpectWithOffset(1, microscopeErr).To(BeNil(), "Microscope cannot be started")
		defer microscopeCancel()

		validatedImage := func(image string) {
			By("Checking that installed image is %q", image)

			filter := `{.items[*].status.containerStatuses[0].image}`
			data, err := kubectl.GetPods(
				helpers.KubeSystemNamespace, "-l k8s-app=cilium").Filter(filter)
			ExpectWithOffset(1, err).To(BeNil(), "Cannot get cilium pods")

			for _, val := range strings.Split(data.String(), " ") {
				ExpectWithOffset(1, val).To(ContainSubstring(image), "Cilium image didn't update correctly")
			}
		}

		validateEndpointsConnection := func() {
			err := kubectl.CiliumEndpointWaitReady()
			ExpectWithOffset(1, err).To(BeNil(), "Endpoints are not ready after timeout")

			ExpectKubeDNSReady(kubectl)

			err = kubectl.WaitForKubeDNSEntry(app1Service, helpers.DefaultNamespace)
			ExpectWithOffset(1, err).To(BeNil(), "DNS entry is not ready after timeout")

			err = kubectl.CiliumEndpointWaitReady()
			ExpectWithOffset(1, err).To(BeNil(), "Endpoints are not ready after timeout")

			appPods := helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "id")

			err = kubectl.WaitForKubeDNSEntry(app1Service, helpers.DefaultNamespace)
			ExpectWithOffset(1, err).To(BeNil(), "DNS entry is not ready after timeout")

			By("Making L7 requests between endpoints")
			res := kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail("http://%s/public", app1Service))
			ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), "Cannot curl app1-service")

			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail("http://%s/private", app1Service))
			ExpectWithOffset(1, res).ShouldNot(helpers.CMDSuccess(), "Expect a 403 from app1-service")
		}

		By("Creating some endpoints and L7 policy")
		res := kubectl.Apply(demoPath)
		ExpectWithOffset(1, res).To(helpers.CMDSuccess(), "cannot apply dempo application")

		err := kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=testapp", timeout)
		Expect(err).Should(BeNil(), "Test pods are not ready after timeout")

		ExpectKubeDNSReady(kubectl)

		_, err = kubectl.CiliumPolicyAction(
			helpers.KubeSystemNamespace, l7Policy, helpers.KubectlApply, timeout)
		Expect(err).Should(BeNil(), "cannot import l7 policy: %v", l7Policy)

		validateEndpointsConnection()

		By("Updating cilium to master image")

		waitForUpdateImage := func(image string) func() bool {
			return func() bool {
				pods, err := kubectl.GetCiliumPods(helpers.KubeSystemNamespace)
				if err != nil {
					return false
				}

				filter := `{.items[*].status.containerStatuses[0].image}`
				data, err := kubectl.GetPods(
					helpers.KubeSystemNamespace, "-l k8s-app=cilium").Filter(filter)
				if err != nil {
					return false
				}
				number := strings.Count(data.String(), image)
				if number == len(pods) {
					return true
				}
				log.Infof("Only '%v' of '%v' cilium pods updated to the new image",
					number, len(pods))
				return false
			}
		}

		By("Install Cilium pre-flight check DaemonSet")
		err = kubectl.CiliumPreFlightInstall(helpers.CiliumDefaultPreFlightPatch)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium pre-flight %q was not able to be deployed", newVersion)
		ExpectCiliumPreFlightInstallReady(kubectl)

		// Once they are installed we can remove it
		By("Removing Cilium pre-flight check DaemonSet")
		kubectl.Delete(helpers.GetK8sDescriptor(helpers.CiliumDefaultPreFlight))

		err = kubectl.CiliumInstall(helpers.CiliumDefaultDSPatch, helpers.CiliumConfigMapPatch)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be deployed", newVersion)

		err = helpers.WithTimeout(
			waitForUpdateImage(newVersion),
			"Cilium Pods are not updating correctly",
			&helpers.TimeoutConfig{Timeout: timeout})
		ExpectWithOffset(1, err).To(BeNil(), "Pods are not updating")

		err = kubectl.WaitforPods(
			helpers.KubeSystemNamespace, "-l k8s-app=cilium", timeout)
		ExpectWithOffset(1, err).Should(BeNil(), "Cilium is not ready after timeout")

		validatedImage(newVersion)

		validateEndpointsConnection()

		By("Downgrading cilium to %s image", oldVersion)

		err = kubectl.CiliumInstallVersion(
			helpers.CiliumDefaultDSPatch,
			helpers.CiliumConfigMapPatch,
			helpers.CiliumStableVersion,
		)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be deployed", helpers.CiliumStableVersion)

		err = helpers.WithTimeout(
			waitForUpdateImage(helpers.CiliumStableVersion),
			"Cilium Pods are not updating correctly",
			&helpers.TimeoutConfig{Timeout: timeout})
		ExpectWithOffset(1, err).To(BeNil(), "Pods are not updating")

		err = kubectl.WaitforPods(
			helpers.KubeSystemNamespace, "-l k8s-app=cilium", timeout)
		ExpectWithOffset(1, err).Should(BeNil(), "Cilium is not ready after timeout")

		validatedImage(helpers.CiliumStableImageVersion)

		validateEndpointsConnection()

	}
	return testfunc, cleanupCallback
}
<commit_msg>test: perform downgrade for given version<commit_after>package k8sTest

import (
	"fmt"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var _ = Describe("K8sUpdates", func() {

	// This test runs 8 steps as following:
	// 1 - delete all pods. Clean cilium, this can be, and should be achieved by
	// `clean-cilium-state: "true"` option that we have in configmap
	// 2 - install cilium `cilium:v1.1.4`
	// 3 - make endpoints talk with each other with policy
	// 4 - upgrade cilium to `k8s1:5000/cilium/cilium-dev:latest`
	// 5 - make endpoints talk with each other with policy
	// 6 - downgrade cilium to `cilium:v1.1.4`
	// 7 - make endpoints talk with each other with policy
	// 8 - delete all pods. Clean cilium, this can be, and should be achieved by
	// `clean-cilium-state: "true"` option that we have in configmap.
	// This makes sure the upgrade tests won't affect any other test
	// 9 - re install cilium:latest image for remaining tests.

	var (
		kubectl *helpers.Kubectl

		cleanupCallback = func() { return }
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		_ = kubectl.Delete(helpers.DNSDeployment())

		// Delete kube-dns because if not will be a restore the old endpoints
		// from master instead of create the new ones.
		_ = kubectl.DeleteResource(
			"deploy", fmt.Sprintf("-n %s kube-dns", helpers.KubeSystemNamespace))

		// Sometimes PolicyGen has a lot of pods running around without delete
		// it. Using this we are sure that we delete before this test start
		kubectl.Exec(fmt.Sprintf(
			"%s delete --all pods,svc,cnp -n %s", helpers.KubectlCmd, helpers.DefaultNamespace))

		kubectl.DeleteETCDOperator()

		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		_ = kubectl.Apply(helpers.DNSDeployment())
	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace, "cilium endpoint list")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		cleanupCallback()
		ExpectAllPodsTerminated(kubectl)
	})

	It("Tests upgrade and downgrade from a Cilium stable image to master", func() {
		var assertUpgradeSuccessful func()
		assertUpgradeSuccessful, cleanupCallback =
			InstallAndValidateCiliumUpgrades(kubectl, helpers.CiliumStableVersion, helpers.CiliumDeveloperImage)
		assertUpgradeSuccessful()
	})
})

// InstallAndValidateCiliumUpgrades installs and tests if the oldVersion can be
// upgrade to the newVersion and if the newVersion can be downgraded to the
// oldVersion.  It returns two callbacks, the first one is the assertfunction
// that need to run, and the second one are the cleanup actions
func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldVersion, newVersion string) (func(), func()) {
	canRun, err := helpers.CanRunK8sVersion(oldVersion, helpers.GetCurrentK8SEnv())
	ExpectWithOffset(1, err).To(BeNil(), "Unable to get k8s constraints for %s", oldVersion)
	if !canRun {
		Skip(fmt.Sprintf(
			"Cilium %q is not supported in K8s %q. Skipping upgrade/downgrade tests.",
			oldVersion, helpers.GetCurrentK8SEnv()))
		return func() {}, func() {}
	}

	demoPath := helpers.ManifestGet("demo.yaml")
	l7Policy := helpers.ManifestGet("l7-policy.yaml")
	apps := []string{helpers.App1, helpers.App2, helpers.App3}
	app1Service := "app1-service"

	cleanupCallback := func() {
		kubectl.Delete(l7Policy)
		kubectl.Delete(demoPath)

		// make sure that Kubedns is deleted correctly
		_ = kubectl.Delete(helpers.DNSDeployment())

		kubectl.DeleteETCDOperator()

		ExpectAllPodsTerminated(kubectl)

		// make sure we clean everything up before doing any other test
		err := kubectl.CiliumInstall(
			helpers.CiliumDefaultDSPatch,
			"cilium-cm-patch-clean-cilium-state.yaml",
		)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be deployed", newVersion)
		err = kubectl.WaitForCiliumInitContainerToFinish()
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be clean up environment", newVersion)

		_ = kubectl.DeleteResource(
			"ds", fmt.Sprintf("-n %s cilium", helpers.KubeSystemNamespace))
	}

	testfunc := func() {
		// Making sure that we deleted the  cilium ds. No assert message
		// because maybe is not present
		_ = kubectl.DeleteResource("ds", fmt.Sprintf("-n %s cilium", helpers.KubeSystemNamespace))

		// Delete kube-dns because if not will be a restore the old endpoints
		// from master instead of create the new ones.
		_ = kubectl.Delete(helpers.DNSDeployment())

		ExpectAllPodsTerminated(kubectl)

		By("Installing a cleaning state of Cilium")
		err = kubectl.CiliumInstallVersion(
			helpers.CiliumDefaultDSPatch,
			"cilium-cm-patch-clean-cilium-state.yaml",
			oldVersion,
		)
		Expect(err).To(BeNil(), "Cilium %q was not able to be deployed", oldVersion)

		By("Installing kube-dns")
		_ = kubectl.Apply(helpers.DNSDeployment())

		// Deploy the etcd operator
		By("Deploying etcd-operator")
		err = kubectl.DeployETCDOperator()
		Expect(err).To(BeNil(), "Unable to deploy etcd operator")

		// Cilium is only ready if kvstore is ready, the kvstore is ready if
		// kube-dns is running.
		By("Cilium %q is installed and running", oldVersion)
		ExpectCiliumReady(kubectl)

		ExpectETCDOperatorReady(kubectl)

		By("Installing Microscope")
		microscopeErr, microscopeCancel := kubectl.MicroscopeStart()
		ExpectWithOffset(1, microscopeErr).To(BeNil(), "Microscope cannot be started")
		defer microscopeCancel()

		validatedImage := func(image string) {
			By("Checking that installed image is %q", image)

			filter := `{.items[*].status.containerStatuses[0].image}`
			data, err := kubectl.GetPods(
				helpers.KubeSystemNamespace, "-l k8s-app=cilium").Filter(filter)
			ExpectWithOffset(1, err).To(BeNil(), "Cannot get cilium pods")

			for _, val := range strings.Split(data.String(), " ") {
				ExpectWithOffset(1, val).To(ContainSubstring(image), "Cilium image didn't update correctly")
			}
		}

		validateEndpointsConnection := func() {
			err := kubectl.CiliumEndpointWaitReady()
			ExpectWithOffset(1, err).To(BeNil(), "Endpoints are not ready after timeout")

			ExpectKubeDNSReady(kubectl)

			err = kubectl.WaitForKubeDNSEntry(app1Service, helpers.DefaultNamespace)
			ExpectWithOffset(1, err).To(BeNil(), "DNS entry is not ready after timeout")

			err = kubectl.CiliumEndpointWaitReady()
			ExpectWithOffset(1, err).To(BeNil(), "Endpoints are not ready after timeout")

			appPods := helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "id")

			err = kubectl.WaitForKubeDNSEntry(app1Service, helpers.DefaultNamespace)
			ExpectWithOffset(1, err).To(BeNil(), "DNS entry is not ready after timeout")

			By("Making L7 requests between endpoints")
			res := kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail("http://%s/public", app1Service))
			ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), "Cannot curl app1-service")

			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail("http://%s/private", app1Service))
			ExpectWithOffset(1, res).ShouldNot(helpers.CMDSuccess(), "Expect a 403 from app1-service")
		}

		By("Creating some endpoints and L7 policy")
		res := kubectl.Apply(demoPath)
		ExpectWithOffset(1, res).To(helpers.CMDSuccess(), "cannot apply dempo application")

		err := kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=testapp", timeout)
		Expect(err).Should(BeNil(), "Test pods are not ready after timeout")

		ExpectKubeDNSReady(kubectl)

		_, err = kubectl.CiliumPolicyAction(
			helpers.KubeSystemNamespace, l7Policy, helpers.KubectlApply, timeout)
		Expect(err).Should(BeNil(), "cannot import l7 policy: %v", l7Policy)

		validateEndpointsConnection()

		By("Updating cilium to master image")

		waitForUpdateImage := func(image string) func() bool {
			return func() bool {
				pods, err := kubectl.GetCiliumPods(helpers.KubeSystemNamespace)
				if err != nil {
					return false
				}

				filter := `{.items[*].status.containerStatuses[0].image}`
				data, err := kubectl.GetPods(
					helpers.KubeSystemNamespace, "-l k8s-app=cilium").Filter(filter)
				if err != nil {
					return false
				}
				number := strings.Count(data.String(), image)
				if number == len(pods) {
					return true
				}
				log.Infof("Only '%v' of '%v' cilium pods updated to the new image",
					number, len(pods))
				return false
			}
		}

		By("Install Cilium pre-flight check DaemonSet")
		err = kubectl.CiliumPreFlightInstall(helpers.CiliumDefaultPreFlightPatch)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium pre-flight %q was not able to be deployed", newVersion)
		ExpectCiliumPreFlightInstallReady(kubectl)

		// Once they are installed we can remove it
		By("Removing Cilium pre-flight check DaemonSet")
		kubectl.Delete(helpers.GetK8sDescriptor(helpers.CiliumDefaultPreFlight))

		err = kubectl.CiliumInstall(helpers.CiliumDefaultDSPatch, helpers.CiliumConfigMapPatch)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be deployed", newVersion)

		err = helpers.WithTimeout(
			waitForUpdateImage(newVersion),
			"Cilium Pods are not updating correctly",
			&helpers.TimeoutConfig{Timeout: timeout})
		ExpectWithOffset(1, err).To(BeNil(), "Pods are not updating")

		err = kubectl.WaitforPods(
			helpers.KubeSystemNamespace, "-l k8s-app=cilium", timeout)
		ExpectWithOffset(1, err).Should(BeNil(), "Cilium is not ready after timeout")

		validatedImage(newVersion)

		validateEndpointsConnection()

		By("Downgrading cilium to %s image", oldVersion)

		err = kubectl.CiliumInstallVersion(
			helpers.CiliumDefaultDSPatch,
			helpers.CiliumConfigMapPatch,
			oldVersion,
		)
		ExpectWithOffset(1, err).To(BeNil(), "Cilium %q was not able to be deployed", oldVersion)

		err = helpers.WithTimeout(
			waitForUpdateImage(oldVersion),
			"Cilium Pods are not updating correctly",
			&helpers.TimeoutConfig{Timeout: timeout})
		ExpectWithOffset(1, err).To(BeNil(), "Pods are not updating")

		err = kubectl.WaitforPods(
			helpers.KubeSystemNamespace, "-l k8s-app=cilium", timeout)
		ExpectWithOffset(1, err).Should(BeNil(), "Cilium is not ready after timeout")

		validatedImage(oldVersion)

		validateEndpointsConnection()

	}
	return testfunc, cleanupCallback
}
<|endoftext|>


<commit_before>package ql

// These are a set of helpers that just call LoadValue and return the value.
// They return (_, ErrNotFound) if nothing was found.
//
// The inclusion of these helpers in the package is not an obvious choice:
// Benefits:
//  - slight increase in code clarity/conciseness b/c you can use ":=" to define the variable
//
//    count, err := d.Select("COUNT(*)").From("users").Where("x = ?", x).ReturnInt64()
//
//    vs
//
//    var count int64
//    err := d.Select("COUNT(*)").From("users").Where("x = ?", x).LoadValue(&count)
//
// Downsides:
//  - very small increase in code cost, although it's not complex code
//  - increase in conceptual model / API footprint when presenting the package to new users
//  - no functionality that you can't achieve calling .LoadValue directly.
//  - There's a lot of possible types. Do we want to include ALL of them? u?int{8,16,32,64}?,
//    strings, null varieties, etc.
//  - Let's just do the common, non-null varieties.

// ReturnInt64 executes the SelectBuilder and returns the value as an int64.
func (b *SelectBuilder) ReturnInt64() (int64, error) {
	var v int64
	err := b.LoadValue(&v)
	return v, err
}

// ReturnInt64s executes the SelectBuilder and returns the value as a slice of int64s.
func (b *SelectBuilder) ReturnInt64s() ([]int64, error) {
	var v []int64
	_, err := b.LoadValues(&v)
	return v, err
}

// ReturnUint64 executes the SelectBuilder and returns the value as an uint64.
func (b *SelectBuilder) ReturnUint64() (uint64, error) {
	var v uint64
	err := b.LoadValue(&v)
	return v, err
}

// ReturnUint64s executes the SelectBuilder and returns the value as a slice of uint64s.
func (b *SelectBuilder) ReturnUint64s() ([]uint64, error) {
	var v []uint64
	_, err := b.LoadValues(&v)
	return v, err
}

// ReturnString executes the SelectBuilder and returns the value as a string.
func (b *SelectBuilder) ReturnString() (string, error) {
	var v string
	err := b.LoadValue(&v)
	return v, err
}

// ReturnStrings executes the SelectBuilder and returns the value as a slice of strings.
func (b *SelectBuilder) ReturnStrings() ([]string, error) {
	var v []string
	_, err := b.LoadValues(&v)
	return v, err
}
<commit_msg>add Return{{I,Ui}nt64,String}{,s} to Query<commit_after>package ql

// These are a set of helpers that just call LoadValue and return the value.
// They return (_, ErrNotFound) if nothing was found.
//
// The inclusion of these helpers in the package is not an obvious choice:
// Benefits:
//  - slight increase in code clarity/conciseness b/c you can use ":=" to define the variable
//
//    count, err := d.Select("COUNT(*)").From("users").Where("x = ?", x).ReturnInt64()
//
//    vs
//
//    var count int64
//    err := d.Select("COUNT(*)").From("users").Where("x = ?", x).LoadValue(&count)
//
// Downsides:
//  - very small increase in code cost, although it's not complex code
//  - increase in conceptual model / API footprint when presenting the package to new users
//  - no functionality that you can't achieve calling .LoadValue directly.
//  - There's a lot of possible types. Do we want to include ALL of them? u?int{8,16,32,64}?,
//    strings, null varieties, etc.
//  - Let's just do the common, non-null varieties.

// ReturnInt64 executes the SelectBuilder and returns the value as an int64.
func (l loader) ReturnInt64() (int64, error) {
	var v int64
	err := l.LoadValue(&v)
	return v, err
}

// ReturnInt64s executes the SelectBuilder and returns the value as a slice of int64s.
func (l loader) ReturnInt64s() ([]int64, error) {
	var v []int64
	_, err := l.LoadValues(&v)
	return v, err
}

// ReturnUint64 executes the SelectBuilder and returns the value as an uint64.
func (l loader) ReturnUint64() (uint64, error) {
	var v uint64
	err := l.LoadValue(&v)
	return v, err
}

// ReturnUint64s executes the SelectBuilder and returns the value as a slice of uint64s.
func (l loader) ReturnUint64s() ([]uint64, error) {
	var v []uint64
	_, err := l.LoadValues(&v)
	return v, err
}

// ReturnString executes the SelectBuilder and returns the value as a string.
func (l loader) ReturnString() (string, error) {
	var v string
	err := l.LoadValue(&v)
	return v, err
}

// ReturnStrings executes the SelectBuilder and returns the value as a slice of strings.
func (l loader) ReturnStrings() ([]string, error) {
	var v []string
	_, err := l.LoadValues(&v)
	return v, err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// JobLayer associates a Layer with a Job.
type JobLayer struct {
	Name string `json:"name"`
}

// JobVolume associates one or more Volumes with a Job.
type JobVolume struct {
	Name string `json:"name"`
}

// ResultSource describes a mechanism for providing a Job's result back to the client. This can be
// either the singleton constant StdoutResult or a FileResult with a path.
type ResultSource interface {
	IsResultSource()
}

type stdoutResult struct{}

func (r stdoutResult) IsResultSource() {}

// StdoutResult is a singleton ResultSource that indicates that a Job will return its result to
// the client over stdout.
var StdoutResult = stdoutResult{}

// FileResult is a ResultSource that indicates that a Job will return its result to the client by
// placing it in a file at a certain path within its container.
type FileResult struct {
	Path string
}

func (r FileResult) String() string {
	return "file:" + r.Path
}

// IsResultSource is a marker method for the ResultSource interface.
func (r FileResult) IsResultSource() {}

// ResultType indicates how a Job's output should be interpreted by the client. Must be one of
// BinaryResult or PickleResult.
type ResultType struct {
	name string
}

func (s ResultType) String() string {
	return s.name
}

var (
	// BinaryResult indicates that the client should not attempt to interpret the result payload, but
	// provide it as raw bytes.
	BinaryResult = ResultType{name: "binary"}

	// PickleResult indicates that the result contains pickled Python objects.
	PickleResult = ResultType{name: "pickle"}
)

// JobStatus describes the current status of a submitted job.
type JobStatus struct {
	name      string
	completed bool
}

func (s JobStatus) String() string {
	return s.name
}

// IsFinished returns true if the current status indicates that the job has completed execution,
// successfully or otherwise.
func (s JobStatus) IsFinished() bool {
	return s.completed
}

// MarshalJSON encodes a JobStatus as a JSON string.
func (s JobStatus) MarshalJSON() ([]byte, error) {
	return json.Marshal(s.name)
}

var (
	// StatusWaiting indicates that a job has been submitted, but has not yet entered the queue.
	StatusWaiting = JobStatus{name: "waiting"}

	// StatusQueued indicates that a job has been placed into the execution queue.
	StatusQueued = JobStatus{name: "queued"}

	// StatusProcessing indicates that the job is running.
	StatusProcessing = JobStatus{name: "processing"}

	// StatusDone indicates that the job has completed successfully.
	StatusDone = JobStatus{name: "done"}

	// StatusError indicates that the job threw some kind of exception or otherwise returned a non-zero
	// exit code.
	StatusError = JobStatus{name: "error"}

	// StatusKilled indicates that the user requested that the job be terminated.
	StatusKilled = JobStatus{name: "killed"}

	// StatusStalled indicates that the job has gotten stuck (usually fetching dependencies).
	StatusStalled = JobStatus{name: "stalled"}
)

// Collected contains various metrics about the running job.
type Collected struct {
	CPUTimeUser     uint64 `json:"cputime_user,omitempty"`
	CPUTimeSystem   uint64 `json:"cputime_system,omitempty"`
	MemoryFailCount uint64 `json:"memory_failcnt,omitempty"`
	MemoryMaxUsage  uint64 `json:"memory_max_usage,omitempty"`
}

// Job is a user-submitted compute task to be executed in an appropriate Docker container.
type Job struct {
	Command      string            `json:"cmd"`
	Name         *string           `json:"name,omitempty"`
	Core         string            `json:"core"`
	Multicore    int               `json:"multicore"`
	Restartable  bool              `json:"restartable"`
	Tags         map[string]string `json:"tags"`
	Layers       []JobLayer        `json:"layer"`
	Volumes      []JobVolume       `json:"vol"`
	Environment  map[string]string `json:"env"`
	ResultSource ResultSource      `json:"-"`
	ResultType   ResultType        `json:"-"`
	MaxRuntime   int               `json:"max_runtime"`
	Stdin        []byte            `json:"stdin"`

	Profile   *bool   `json:"profile,omitempty"`
	DependsOn *string `json:"depends_on,omitempty"`
}

// SubmittedJob is a Job that has already been submitted.
type SubmittedJob struct {
	Job

	CreatedAt  JSONTime `json:"created_at"`
	StartedAt  JSONTime `json:"started_at,omitempty"`
	FinishedAt JSONTime `json:"finished_at,omitempty"`

	Status        JobStatus `json:"status"`
	Result        string    `json:"result"`
	ReturnCode    string    `json:"return_code"`
	Runtime       uint64    `json:"runtime"`
	QueueDelay    uint64    `json:"queue_delay"`
	OverheadDelay uint64    `json:"overhead_delay"`
	Stderr        string    `json:"stderr"`
	Stdout        string    `json:"stdout"`

	Collected Collected `json:"collected,omitempty"`
}

// JobHandler dispatches API calls to /job based on request type.
func JobHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		JobListHandler(c, w, r)
	case "POST":
		JobSubmitHandler(c, w, r)
	default:
		RhoError{
			Code:    "3",
			Message: "Method not supported",
			Hint:    "Use GET or POST against this endpoint.",
			Retry:   false,
		}.Report(http.StatusMethodNotAllowed, w)
	}
}

// JobSubmitHandler enqueues a new job associated with the authenticated account.
func JobSubmitHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	type RequestJob struct {
		Job

		RawResultSource string `json:"result_source"`
		RawResultType   string `json:"result_type"`
	}

	type Request struct {
		Jobs []RequestJob `json:"jobs"`
	}

	type Response struct {
		JIDs []uint `json:"jids"`
	}

	account, err := Authenticate(c, w, r)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("Authentication failure.")
		return
	}

	// body, err := ioutil.ReadAll(r.Body)
	// log.WithFields(log.Fields{
	// 	"body": string(body),
	// }).Info("Request body")

	var req Request
	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		log.WithFields(log.Fields{
			"error":   err,
			"account": account.Name,
		}).Error("Unable to parse JSON.")

		RhoError{
			Code:    "5",
			Message: "Unable to parse job payload as JSON.",
			Hint:    "Please supply valid JSON in your request.",
			Retry:   false,
		}.Report(http.StatusBadRequest, w)
		return
	}

	jids := make([]uint, len(req.Jobs))
	for index, rjob := range req.Jobs {
		job := rjob.Job

		// Interpret the deferred fields.
		if rjob.RawResultSource == "stdout" {
			job.ResultSource = StdoutResult
		} else if strings.HasPrefix(rjob.RawResultSource, "file:") {
			path := rjob.RawResultSource[len("file:") : len(rjob.RawResultSource)-1]
			job.ResultSource = FileResult{Path: path}
		} else {
			log.WithFields(log.Fields{
				"account":       account.Name,
				"result_source": rjob.RawResultSource,
			}).Error("Invalid result_source in a submitted job.")

			RhoError{
				Code:    "6",
				Message: "Invalid result_source.",
				Hint:    `"result_source" must be either "stdout" or "file:{path}".`,
				Retry:   false,
			}.Report(http.StatusBadRequest, w)
			return
		}

		switch rjob.RawResultType {
		case BinaryResult.name:
			job.ResultType = BinaryResult
		case PickleResult.name:
			job.ResultType = PickleResult
		default:
			log.WithFields(log.Fields{
				"account":     account.Name,
				"result_type": rjob.RawResultType,
			}).Error("Invalid result_type in a submitted job.")

			RhoError{
				Code:    "7",
				Message: "Invalid result_type.",
				Hint:    `"result_type" must be either "binary" or "pickle".`,
				Retry:   false,
			}.Report(http.StatusBadRequest, w)
			return
		}

		jids[index] = uint(index)
		log.WithFields(log.Fields{
			"job":     job,
			"account": account.Name,
		}).Info("Succesfully submitted a job.")
	}

	response := Response{JIDs: jids}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(response)
}

// JobListHandler provides updated details about one or more jobs currently submitted to the
// cluster.
func JobListHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, `[]`)
}

// JobKillHandler allows a user to prematurely terminate a running job.
func JobKillHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	//
}

// JobKillAllHandler allows a user to terminate all jobs associated with their account.
func JobKillAllHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	//
}

// JobQueueStatsHandler allows a user to view statistics about the jobs that they have submitted.
func JobQueueStatsHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	//
}
<commit_msg>Add bson tags to Job structs.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// JobLayer associates a Layer with a Job.
type JobLayer struct {
	Name string `json:"name",bson:"name"`
}

// JobVolume associates one or more Volumes with a Job.
type JobVolume struct {
	Name string `json:"name",bson:"name"`
}

// ResultSource describes a mechanism for providing a Job's result back to the client. This can be
// either the singleton constant StdoutResult or a FileResult with a path.
type ResultSource interface {
	IsResultSource()
}

type stdoutResult struct{}

func (r stdoutResult) IsResultSource() {}

// StdoutResult is a singleton ResultSource that indicates that a Job will return its result to
// the client over stdout.
var StdoutResult = stdoutResult{}

// FileResult is a ResultSource that indicates that a Job will return its result to the client by
// placing it in a file at a certain path within its container.
type FileResult struct {
	Path string
}

func (r FileResult) String() string {
	return "file:" + r.Path
}

// IsResultSource is a marker method for the ResultSource interface.
func (r FileResult) IsResultSource() {}

// ResultType indicates how a Job's output should be interpreted by the client. Must be one of
// BinaryResult or PickleResult.
type ResultType struct {
	name string
}

func (s ResultType) String() string {
	return s.name
}

var (
	// BinaryResult indicates that the client should not attempt to interpret the result payload, but
	// provide it as raw bytes.
	BinaryResult = ResultType{name: "binary"}

	// PickleResult indicates that the result contains pickled Python objects.
	PickleResult = ResultType{name: "pickle"}
)

// JobStatus describes the current status of a submitted job.
type JobStatus struct {
	name      string
	completed bool
}

func (s JobStatus) String() string {
	return s.name
}

// IsFinished returns true if the current status indicates that the job has completed execution,
// successfully or otherwise.
func (s JobStatus) IsFinished() bool {
	return s.completed
}

// MarshalJSON encodes a JobStatus as a JSON string.
func (s JobStatus) MarshalJSON() ([]byte, error) {
	return json.Marshal(s.name)
}

var (
	// StatusWaiting indicates that a job has been submitted, but has not yet entered the queue.
	StatusWaiting = JobStatus{name: "waiting"}

	// StatusQueued indicates that a job has been placed into the execution queue.
	StatusQueued = JobStatus{name: "queued"}

	// StatusProcessing indicates that the job is running.
	StatusProcessing = JobStatus{name: "processing"}

	// StatusDone indicates that the job has completed successfully.
	StatusDone = JobStatus{name: "done"}

	// StatusError indicates that the job threw some kind of exception or otherwise returned a non-zero
	// exit code.
	StatusError = JobStatus{name: "error"}

	// StatusKilled indicates that the user requested that the job be terminated.
	StatusKilled = JobStatus{name: "killed"}

	// StatusStalled indicates that the job has gotten stuck (usually fetching dependencies).
	StatusStalled = JobStatus{name: "stalled"}
)

// Collected contains various metrics about the running job.
type Collected struct {
	CPUTimeUser     uint64 `json:"cputime_user,omitempty"`
	CPUTimeSystem   uint64 `json:"cputime_system,omitempty"`
	MemoryFailCount uint64 `json:"memory_failcnt,omitempty"`
	MemoryMaxUsage  uint64 `json:"memory_max_usage,omitempty"`
}

// Job is a user-submitted compute task to be executed in an appropriate Docker container.
type Job struct {
	Command      string            `json:"cmd",bson:"cmd"`
	Name         *string           `json:"name,omitempty",bson:"name,omitempty"`
	Core         string            `json:"core",bson:"core"`
	Multicore    int               `json:"multicore",bson:"multicore"`
	Restartable  bool              `json:"restartable",bson:"restartable"`
	Tags         map[string]string `json:"tags",bson:"tags"`
	Layers       []JobLayer        `json:"layer",bson:"layer"`
	Volumes      []JobVolume       `json:"vol",bson:"vol"`
	Environment  map[string]string `json:"env",bson:"env"`
	ResultSource ResultSource      `json:"-",bson:"-"`
	ResultType   ResultType        `json:"-",bson:"-"`
	MaxRuntime   int               `json:"max_runtime",bson:"max_runtime"`
	Stdin        []byte            `json:"stdin",bson:"stdin"`

	Profile   *bool   `json:"profile,omitempty",bson:"profile,omitempty"`
	DependsOn *string `json:"depends_on,omitempty",bson:"depends_on,omitempty"`
}

// SubmittedJob is a Job that has already been submitted.
type SubmittedJob struct {
	Job

	CreatedAt  JSONTime `json:"created_at",bson:"created_at"`
	StartedAt  JSONTime `json:"started_at,omitempty",bson:"started_at"`
	FinishedAt JSONTime `json:"finished_at,omitempty",bson:"finished_at"`

	Status        JobStatus `json:"status",bson:"status"`
	Result        string    `json:"result",bson:"result"`
	ReturnCode    string    `json:"return_code",bson:"return_code"`
	Runtime       uint64    `json:"runtime",bson:"runtime"`
	QueueDelay    uint64    `json:"queue_delay",bson:"queue_delay"`
	OverheadDelay uint64    `json:"overhead_delay",bson:"overhead_delay"`
	Stderr        string    `json:"stderr",bson:"stderr"`
	Stdout        string    `json:"stdout",bson:"stdout"`

	Collected Collected `json:"collected,omitempty",bson:"collected,omitempty"`

	Account string `json:"-",bson:"account"`
}

// JobHandler dispatches API calls to /job based on request type.
func JobHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		JobListHandler(c, w, r)
	case "POST":
		JobSubmitHandler(c, w, r)
	default:
		RhoError{
			Code:    "3",
			Message: "Method not supported",
			Hint:    "Use GET or POST against this endpoint.",
			Retry:   false,
		}.Report(http.StatusMethodNotAllowed, w)
	}
}

// JobSubmitHandler enqueues a new job associated with the authenticated account.
func JobSubmitHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	type RequestJob struct {
		Job

		RawResultSource string `json:"result_source"`
		RawResultType   string `json:"result_type"`
	}

	type Request struct {
		Jobs []RequestJob `json:"jobs"`
	}

	type Response struct {
		JIDs []uint `json:"jids"`
	}

	account, err := Authenticate(c, w, r)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("Authentication failure.")
		return
	}

	// body, err := ioutil.ReadAll(r.Body)
	// log.WithFields(log.Fields{
	// 	"body": string(body),
	// }).Info("Request body")

	var req Request
	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		log.WithFields(log.Fields{
			"error":   err,
			"account": account.Name,
		}).Error("Unable to parse JSON.")

		RhoError{
			Code:    "5",
			Message: "Unable to parse job payload as JSON.",
			Hint:    "Please supply valid JSON in your request.",
			Retry:   false,
		}.Report(http.StatusBadRequest, w)
		return
	}

	jids := make([]uint, len(req.Jobs))
	for index, rjob := range req.Jobs {
		job := rjob.Job

		// Interpret the deferred fields.
		if rjob.RawResultSource == "stdout" {
			job.ResultSource = StdoutResult
		} else if strings.HasPrefix(rjob.RawResultSource, "file:") {
			path := rjob.RawResultSource[len("file:") : len(rjob.RawResultSource)-1]
			job.ResultSource = FileResult{Path: path}
		} else {
			log.WithFields(log.Fields{
				"account":       account.Name,
				"result_source": rjob.RawResultSource,
			}).Error("Invalid result_source in a submitted job.")

			RhoError{
				Code:    "6",
				Message: "Invalid result_source.",
				Hint:    `"result_source" must be either "stdout" or "file:{path}".`,
				Retry:   false,
			}.Report(http.StatusBadRequest, w)
			return
		}

		switch rjob.RawResultType {
		case BinaryResult.name:
			job.ResultType = BinaryResult
		case PickleResult.name:
			job.ResultType = PickleResult
		default:
			log.WithFields(log.Fields{
				"account":     account.Name,
				"result_type": rjob.RawResultType,
			}).Error("Invalid result_type in a submitted job.")

			RhoError{
				Code:    "7",
				Message: "Invalid result_type.",
				Hint:    `"result_type" must be either "binary" or "pickle".`,
				Retry:   false,
			}.Report(http.StatusBadRequest, w)
			return
		}

		jids[index] = uint(index)
		log.WithFields(log.Fields{
			"job":     job,
			"account": account.Name,
		}).Info("Succesfully submitted a job.")
	}

	response := Response{JIDs: jids}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(response)
}

// JobListHandler provides updated details about one or more jobs currently submitted to the
// cluster.
func JobListHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, `[]`)
}

// JobKillHandler allows a user to prematurely terminate a running job.
func JobKillHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	//
}

// JobKillAllHandler allows a user to terminate all jobs associated with their account.
func JobKillAllHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	//
}

// JobQueueStatsHandler allows a user to view statistics about the jobs that they have submitted.
func JobQueueStatsHandler(c *Context, w http.ResponseWriter, r *http.Request) {
	//
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"net/http"
	"fmt"
	"log"
	"encoding/json"
	elastigo "github.com/mattbaird/elastigo/lib"
)

var (
	http_listen = flag.String("l", ":8080", "[host]:[port] where the HTTP is listening")
	es_server = flag.String("es", "localhost", "ElasticSearch host")
	es_conn *elastigo.Conn
)

func initElastics(host string) {
	es_conn = elastigo.NewConn()
	es_conn.Domain = host
}

func handler(w http.ResponseWriter, r *http.Request) {
	q := r.FormValue("q")

	searchJson := fmt.Sprintf(`{
		"query": {
			"fuzzy_like_this": {
				"fields": ["Path"],
				"like_text": "%s",
				"fuzziness": 1
			}
		}
	}`, q)
	results, err := es_conn.Search("torture", "file", nil, searchJson)
	if err != nil {
		log.Print(err)
	}

	output, err := json.Marshal(results.Hits)
	w.Write(output)
}

func main() {
	initElastics(*es_server)

	http.Handle("/", http.FileServer(http.Dir("static")))
	http.HandleFunc("/s", handler)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Add forgotten host-from-flag thingy<commit_after>package main

import (
	"flag"
	"net/http"
	"fmt"
	"log"
	"encoding/json"
	elastigo "github.com/mattbaird/elastigo/lib"
)

var (
	http_listen = flag.String("l", ":8080", "[host]:[port] where the HTTP is listening")
	es_server = flag.String("es", "localhost", "ElasticSearch host")
	es_conn *elastigo.Conn
)

func initElastics(host string) {
	es_conn = elastigo.NewConn()
	es_conn.Domain = host
}

func handler(w http.ResponseWriter, r *http.Request) {
	q := r.FormValue("q")

	searchJson := fmt.Sprintf(`{
		"query": {
			"fuzzy_like_this": {
				"fields": ["Path"],
				"like_text": "%s",
				"fuzziness": 1
			}
		}
	}`, q)
	results, err := es_conn.Search("torture", "file", nil, searchJson)
	if err != nil {
		log.Print(err)
	}

	output, err := json.Marshal(results.Hits)
	w.Write(output)
}

func main() {
	initElastics(*es_server)

	http.Handle("/", http.FileServer(http.Dir("static")))
	http.HandleFunc("/s", handler)
	http.ListenAndServe(*http_listen, nil)
}
<|endoftext|>