<commit_before>package main

import termbox "github.com/nsf/termbox-go"

type KeybindMode uint8
const (
	KEYBIND_MODE_COMMON = iota
	KEYBIND_MODE_CONVERSATION
	KEYBIND_MODE_HOME_TIMELINE
	KEYBIND_MODE_INPUT
	KEYBIND_MODE_CONFIRM
	KEYBIND_MODE_MENTION_VIEW
	KEYBIND_MODE_USER_TIMELINE
	KEYBIND_MODE_LIST_VIEW
)

type Action uint8

type keybind struct {
	Key    termbox.Key
	Ch     rune
	Action Action
}

const NO_ACTION = 0xff - 1

/* common event action list */
const (
	ACTION_LIKE_TWEET = iota
	ACTION_MENTION
	ACTION_RETWEET
	ACTION_OPEN_IMAGES
	ACTION_NEXT_TWEET
	ACTION_PREVIOUS_TWEET
	ACTION_PAGE_DOWN
	ACTION_PAGE_UP
	ACTION_MOVE_TO_TOP_TWEET
	ACTION_MOVE_TO_BOTTOM_TWEET
	ACTION_TURN_INPUT_MODE
	ACTION_TURN_COMMAND_MODE
	ACTION_TURN_HOME_TIMELINE_MODE
	ACTION_TURN_CONVERSATION_VIEW_MODE
	ACTION_TURN_MENTION_VIEW_MODE
	ACTION_TURN_USER_TIMELINE_MODE
	ACTION_QUIT
	ACTION_OPEN_URL
	ACTION_SHOW_HELP
)
var commonKeybindList = []keybind {
	{ termbox.KeyCtrlS		,	0	,	ACTION_TURN_INPUT_MODE				},
	{ termbox.KeyCtrlW		,	0	,	ACTION_MENTION						},
	{ termbox.KeyCtrlF		,	0	,	ACTION_LIKE_TWEET					},
	{ termbox.KeyCtrlV		,	0	,	ACTION_RETWEET						},
	{ termbox.KeyCtrlO		,	0	,	ACTION_OPEN_URL						},
	{ termbox.KeyCtrlP		,	0	,	ACTION_OPEN_IMAGES					},

	{ termbox.KeyCtrlZ		,	0	,	ACTION_TURN_HOME_TIMELINE_MODE		},
	{ termbox.KeyCtrlX		,	0	,	ACTION_TURN_MENTION_VIEW_MODE		},
	{ termbox.KeyCtrlD		,	0	,	ACTION_TURN_USER_TIMELINE_MODE		},
	{ termbox.KeyArrowRight	,	0	,	ACTION_TURN_CONVERSATION_VIEW_MODE	},
	{ 0						,  ':'	,	ACTION_TURN_COMMAND_MODE			}, /* TODO: need ModAlt field */
	{ termbox.KeyCtrlQ		,	0	,	ACTION_QUIT							},

	{ termbox.KeyArrowUp	,	0	,	ACTION_NEXT_TWEET					},
	{ termbox.KeyArrowDown	,	0	,	ACTION_PREVIOUS_TWEET				},
	{ termbox.KeyHome		,	0	,	ACTION_MOVE_TO_TOP_TWEET			},
	{ termbox.KeyPgup		,	0	,	ACTION_MOVE_TO_TOP_TWEET			},
	{ termbox.KeyEnd		,	0	,	ACTION_MOVE_TO_BOTTOM_TWEET			},
	{ termbox.KeyPgdn		,	0	,	ACTION_MOVE_TO_BOTTOM_TWEET			},
}

/* home timeline action list */
const (
	ACTION_LOAD_PREVIOUSE_TWEETS = iota
	ACTION_LOAD_NEW_TWEETS
)
var homeTimelineKeybindList = []keybind {
	{ termbox.KeySpace,	0,	ACTION_LOAD_PREVIOUSE_TWEETS	},
	{ termbox.KeyCtrlR,	0,	ACTION_LOAD_NEW_TWEETS			},
}

/* input mode action list */
const (
	ACTION_MOVE_LEFT= iota
	ACTION_MOVE_RIGHT
	ACTION_MOVE_UP
	ACTION_MOVE_DOWN
	ACTION_INSERT_SPACE
	ACTION_EXIT_INPUT_MODE
	ACTION_DELETE_RUNE
	ACTION_MOVE_LINE_TOP
	ACTION_MOVE_LINE_BOTTOM
	ACTION_TURN_CONFIRM_MODE
	ACTION_INSERT_NEW_LINE
)
var inputModeKeybindList = []keybind {
	{ termbox.KeyArrowLeft	,	0,	ACTION_MOVE_LEFT			},
	{ termbox.KeyArrowRight	,	0,	ACTION_MOVE_RIGHT			},
	{ termbox.KeyArrowUp	,	0,	ACTION_MOVE_UP				},
	{ termbox.KeyArrowDown	,	0,	ACTION_MOVE_DOWN			},
	{ termbox.KeySpace		,	0,	ACTION_INSERT_SPACE			},
	{ termbox.KeyEsc		,	0,	ACTION_EXIT_INPUT_MODE		},
	{ termbox.KeyCtrlG		,	0,	ACTION_EXIT_INPUT_MODE		},
	{ termbox.KeyBackspace	,	0,	ACTION_DELETE_RUNE			},
	{ termbox.KeyBackspace2	,	0,	ACTION_DELETE_RUNE			},
	{ termbox.KeyCtrlA		,	0,	ACTION_MOVE_LINE_TOP		},
	{ termbox.KeyCtrlE		,	0,	ACTION_MOVE_LINE_BOTTOM		},
	{ termbox.KeyCtrlJ		,	0,	ACTION_TURN_CONFIRM_MODE	},
	{ termbox.KeyEnter		,	0,	ACTION_INSERT_NEW_LINE		},
}

/* confirm mode action list */
const (
	ACTION_CANCEL_SUBMIT = iota
	ACTION_SUBMIT_TWEET
)
var confirmModeKeybindList = []keybind {
	{ termbox.KeyEsc,	0	,	ACTION_CANCEL_SUBMIT	},
	{ termbox.KeyCtrlG,	0	,	ACTION_CANCEL_SUBMIT	},
	{ termbox.KeyEnter,	0	,	ACTION_SUBMIT_TWEET		},
}

/* mention view mode action list */
const (
	ACTION_LOAD_PREVIOUSE_MENTIONS = iota
	ACTION_LOAD_NEW_MENTIONS
)
var mentionViewModeKeybindList = []keybind {
	{ termbox.KeyEnter,	0	,	ACTION_LOAD_PREVIOUSE_MENTIONS	},
	{ termbox.KeySpace,	0	,	ACTION_LOAD_PREVIOUSE_MENTIONS	},
	{ termbox.KeyCtrlR,	0	,	ACTION_LOAD_NEW_MENTIONS		},
}

/* conversation mode action list */
const (
	ACTION_EXIT_CONVERSATION_MODE = iota
)
var conversationModeKeybindList = []keybind {
	{ termbox.KeyArrowLeft,		0	,	ACTION_EXIT_CONVERSATION_MODE	},
}

/* user timeline mode action list */
const (
	ACTION_LOAD_PREVIOUSE_USER_TWEETS = iota
	ACTION_LOAD_NEW_USER_TWEETS
)
var userTimelineModeKeybindList = []keybind {
	{ termbox.KeyEnter,	0,	ACTION_LOAD_PREVIOUSE_USER_TWEETS	},
	{ termbox.KeySpace,	0,	ACTION_LOAD_PREVIOUSE_USER_TWEETS	},
	{ termbox.KeyCtrlR,	0,	ACTION_LOAD_NEW_USER_TWEETS			},
}

/* list mode actin list */
const (
	ACTION_LOAD_PREVIOUSE_LIST = iota
	ACTION_LOAD_NEW_LIST
)
var listModeKeybindList = []keybind {
	{ termbox.KeyEnter,	0,	ACTION_LOAD_PREVIOUSE_LIST	},
	{ termbox.KeySpace,	0,	ACTION_LOAD_PREVIOUSE_LIST	},
	{ termbox.KeyCtrlR,	0,	ACTION_LOAD_NEW_LIST		},
}

func (view *view) handleAction(ev termbox.Event, mode KeybindMode) (Action) {
	var action Action = NO_ACTION
	var keybindList []keybind
	switch mode {
		case KEYBIND_MODE_COMMON :
			keybindList = commonKeybindList
		case KEYBIND_MODE_CONVERSATION :
			keybindList = conversationModeKeybindList
		case KEYBIND_MODE_HOME_TIMELINE :
			keybindList = homeTimelineKeybindList
		case KEYBIND_MODE_INPUT :
			keybindList = inputModeKeybindList
		case KEYBIND_MODE_CONFIRM :
			keybindList = confirmModeKeybindList
		case KEYBIND_MODE_MENTION_VIEW :
			keybindList = mentionViewModeKeybindList
		case KEYBIND_MODE_USER_TIMELINE :
			keybindList = userTimelineModeKeybindList
		case KEYBIND_MODE_LIST_VIEW :
			keybindList = listModeKeybindList
	}
	for i := 0; i<len(keybindList); i++ { /* TODO: To handle ModAlt */
		if ev.Key == 0 {	/* kind of CTRL			*/
			if keybindList[i].Ch == ev.Ch {
				action = keybindList[i].Action
				break
			}
		} else {			/* kind of Charactor	*/
			if keybindList[i].Key == ev.Key {
				action = keybindList[i].Action
				break
			}
		}
	}
	return action
}

<commit_msg>handle ModAlt<commit_after>package main

import termbox "github.com/nsf/termbox-go"

type KeybindMode uint8
const (
	KEYBIND_MODE_COMMON = iota
	KEYBIND_MODE_CONVERSATION
	KEYBIND_MODE_HOME_TIMELINE
	KEYBIND_MODE_INPUT
	KEYBIND_MODE_CONFIRM
	KEYBIND_MODE_MENTION_VIEW
	KEYBIND_MODE_USER_TIMELINE
	KEYBIND_MODE_LIST_VIEW
)

type Action uint8
type keybind struct {
	Mod    termbox.Modifier
	Key    termbox.Key
	Ch     rune
	Action Action
}

const ( /* common event action list */
	ACTION_LIKE_TWEET = iota + 1
	ACTION_MENTION
	ACTION_RETWEET
	ACTION_OPEN_IMAGES
	ACTION_NEXT_TWEET
	ACTION_PREVIOUS_TWEET
	ACTION_PAGE_DOWN
	ACTION_PAGE_UP
	ACTION_MOVE_TO_TOP_TWEET
	ACTION_MOVE_TO_BOTTOM_TWEET
	ACTION_TURN_INPUT_MODE
	ACTION_TURN_COMMAND_MODE
	ACTION_TURN_HOME_TIMELINE_MODE
	ACTION_TURN_CONVERSATION_VIEW_MODE
	ACTION_TURN_MENTION_VIEW_MODE
	ACTION_TURN_USER_TIMELINE_MODE
	ACTION_QUIT
	ACTION_OPEN_URL
	ACTION_SHOW_HELP
)
const ( /* home timeline action list */
	ACTION_LOAD_PREVIOUSE_TWEETS = iota + 1
	ACTION_LOAD_NEW_TWEETS
)
const ( /* input mode action list */
	ACTION_MOVE_LEFT= iota + 1
	ACTION_MOVE_RIGHT
	ACTION_MOVE_UP
	ACTION_MOVE_DOWN
	ACTION_INSERT_SPACE
	ACTION_EXIT_INPUT_MODE
	ACTION_DELETE_RUNE
	ACTION_MOVE_LINE_TOP
	ACTION_MOVE_LINE_BOTTOM
	ACTION_TURN_CONFIRM_MODE
	ACTION_INSERT_NEW_LINE
)
const ( /* confirm mode action list */
	ACTION_CANCEL_SUBMIT = iota + 1
	ACTION_SUBMIT_TWEET
)
const ( /* mention view mode action list */
	ACTION_LOAD_PREVIOUSE_MENTIONS = iota + 1
	ACTION_LOAD_NEW_MENTIONS
)
const ( /* conversation mode action list */
	ACTION_EXIT_CONVERSATION_MODE = iota + 1
)
const ( /* user timeline mode action list */
	ACTION_LOAD_PREVIOUSE_USER_TWEETS = iota + 1
	ACTION_LOAD_NEW_USER_TWEETS
)
const ( /* list mode actin list */
	ACTION_LOAD_PREVIOUSE_LIST = iota + 1
	ACTION_LOAD_NEW_LIST
)

const NO_MOD    = 0
const NO_KEY    = 0
const NO_CH     = 0
const NO_ACTION = 0

/* keybind list */
var commonKeybindList = []keybind {
	{ NO_MOD		, termbox.KeyCtrlS		, NO_CH	, ACTION_TURN_INPUT_MODE			},
	{ NO_MOD		, termbox.KeyCtrlW		, NO_CH	, ACTION_MENTION					},
	{ NO_MOD		, termbox.KeyCtrlF		, NO_CH	, ACTION_LIKE_TWEET					},
	{ NO_MOD		, termbox.KeyCtrlV		, NO_CH	, ACTION_RETWEET					},
	{ NO_MOD		, termbox.KeyCtrlO		, NO_CH	, ACTION_OPEN_URL					},
	{ NO_MOD		, termbox.KeyCtrlP		, NO_CH	, ACTION_OPEN_IMAGES				},

	{ NO_MOD		, termbox.KeyCtrlZ		, NO_CH	, ACTION_TURN_HOME_TIMELINE_MODE	},
	{ NO_MOD		, termbox.KeyCtrlX		, NO_CH	, ACTION_TURN_MENTION_VIEW_MODE		},
	{ NO_MOD		, termbox.KeyCtrlD		, NO_CH	, ACTION_TURN_USER_TIMELINE_MODE	},
	{ NO_MOD		, termbox.KeyArrowRight	, NO_CH	, ACTION_TURN_CONVERSATION_VIEW_MODE},
	{ termbox.ModAlt, NO_KEY				, 'x'	, ACTION_TURN_COMMAND_MODE			}, /* TODO: need ModAlt field */
	{ NO_MOD		, termbox.KeyCtrlQ		, NO_CH	, ACTION_QUIT						},

	{ NO_MOD		, termbox.KeyArrowUp	, NO_CH	, ACTION_NEXT_TWEET					},
	{ NO_MOD		, termbox.KeyArrowDown	, NO_CH	, ACTION_PREVIOUS_TWEET				},
	{ NO_MOD		, termbox.KeyHome		, NO_CH	, ACTION_MOVE_TO_TOP_TWEET			},
	{ NO_MOD		, termbox.KeyPgup		, NO_CH	, ACTION_MOVE_TO_TOP_TWEET			},
	{ NO_MOD		, termbox.KeyEnd		, NO_CH	, ACTION_MOVE_TO_BOTTOM_TWEET		},
	{ NO_MOD		, termbox.KeyPgdn		, NO_CH	, ACTION_MOVE_TO_BOTTOM_TWEET		},
}

var homeTimelineKeybindList = []keybind {
	{ NO_MOD		, termbox.KeySpace		, NO_CH , ACTION_LOAD_PREVIOUSE_TWEETS		},
	{ NO_MOD		, termbox.KeyCtrlR		, NO_CH , ACTION_LOAD_NEW_TWEETS			},
}

var inputModeKeybindList = []keybind {
	{ NO_MOD		, termbox.KeyArrowLeft	, NO_CH , ACTION_MOVE_LEFT					},
	{ NO_MOD		, termbox.KeyArrowRight	, NO_CH , ACTION_MOVE_RIGHT					},
	{ NO_MOD		, termbox.KeyArrowUp	, NO_CH , ACTION_MOVE_UP					},
	{ NO_MOD		, termbox.KeyArrowDown	, NO_CH , ACTION_MOVE_DOWN					},
	{ NO_MOD		, termbox.KeySpace		, NO_CH , ACTION_INSERT_SPACE				},
	{ NO_MOD		, termbox.KeyEsc		, NO_CH , ACTION_EXIT_INPUT_MODE			},
	{ NO_MOD		, termbox.KeyCtrlG		, NO_CH , ACTION_EXIT_INPUT_MODE			},
	{ NO_MOD		, termbox.KeyBackspace	, NO_CH , ACTION_DELETE_RUNE				},
	{ NO_MOD		, termbox.KeyBackspace2	, NO_CH , ACTION_DELETE_RUNE				},
	{ NO_MOD		, termbox.KeyCtrlA		, NO_CH , ACTION_MOVE_LINE_TOP				},
	{ NO_MOD		, termbox.KeyCtrlE		, NO_CH , ACTION_MOVE_LINE_BOTTOM			},
	{ NO_MOD		, termbox.KeyCtrlJ		, NO_CH , ACTION_TURN_CONFIRM_MODE			},
	{ NO_MOD		, termbox.KeyEnter		, NO_CH , ACTION_INSERT_NEW_LINE			},
}

var confirmModeKeybindList = []keybind {
	{ NO_MOD		, termbox.KeyEsc		, NO_CH , ACTION_CANCEL_SUBMIT				},
	{ NO_MOD		, termbox.KeyCtrlG		, NO_CH , ACTION_CANCEL_SUBMIT				},
	{ NO_MOD		, termbox.KeyEnter		, NO_CH , ACTION_SUBMIT_TWEET				},
}

var mentionViewModeKeybindList = []keybind {
	{ NO_MOD		, termbox.KeyEnter		, NO_CH , ACTION_LOAD_PREVIOUSE_MENTIONS	},
	{ NO_MOD		, termbox.KeySpace		, NO_CH , ACTION_LOAD_PREVIOUSE_MENTIONS	},
	{ NO_MOD		, termbox.KeyCtrlR		, NO_CH , ACTION_LOAD_NEW_MENTIONS			},
}

var conversationModeKeybindList = []keybind {
	{ NO_MOD		, termbox.KeyArrowLeft	, NO_CH , ACTION_EXIT_CONVERSATION_MODE		},
}

var userTimelineModeKeybindList = []keybind {
	{ NO_MOD		, termbox.KeyEnter		, NO_CH , ACTION_LOAD_PREVIOUSE_USER_TWEETS	},
	{ NO_MOD		, termbox.KeySpace		, NO_CH , ACTION_LOAD_PREVIOUSE_USER_TWEETS	},
	{ NO_MOD		, termbox.KeyCtrlR		, NO_CH , ACTION_LOAD_NEW_USER_TWEETS		},
}

var listModeKeybindList = []keybind {
	{ NO_MOD		, termbox.KeyEnter		, NO_CH , ACTION_LOAD_PREVIOUSE_LIST		},
	{ NO_MOD		, termbox.KeySpace		, NO_CH , ACTION_LOAD_PREVIOUSE_LIST		},
	{ NO_MOD		, termbox.KeyCtrlR		, NO_CH , ACTION_LOAD_NEW_LIST				},
}

func (view *view) handleAction(ev termbox.Event, mode KeybindMode) (Action) {
	var keybindList []keybind
	switch mode {
		case KEYBIND_MODE_COMMON :
			keybindList = commonKeybindList
		case KEYBIND_MODE_CONVERSATION :
			keybindList = conversationModeKeybindList
		case KEYBIND_MODE_HOME_TIMELINE :
			keybindList = homeTimelineKeybindList
		case KEYBIND_MODE_INPUT :
			keybindList = inputModeKeybindList
		case KEYBIND_MODE_CONFIRM :
			keybindList = confirmModeKeybindList
		case KEYBIND_MODE_MENTION_VIEW :
			keybindList = mentionViewModeKeybindList
		case KEYBIND_MODE_USER_TIMELINE :
			keybindList = userTimelineModeKeybindList
		case KEYBIND_MODE_LIST_VIEW :
			keybindList = listModeKeybindList
	}
	for i := 0; i<len(keybindList); i++ {
		if (ev.Mod == keybindList[i].Mod) &&
		   (ev.Key == keybindList[i].Key) &&
		   (ev.Ch  == keybindList[i].Ch) {
				return keybindList[i].Action
		}
	}
	return NO_ACTION
}

<|endoftext|>


<commit_before>package models

import (
	"errors"
	"fmt"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	optionsKey = "opts"
)

var (
	//DefaultIsRequired ...
	DefaultIsRequired = false
	// DefaultIsExpand ...
	DefaultIsExpand = true
	// DefaultIsDontChangeValue ...
	DefaultIsDontChangeValue = false
	// DefaultIsAlwaysRun ...
	DefaultIsAlwaysRun = false
	// DefaultIsRequiresAdminUser ...
	DefaultIsRequiresAdminUser = false
	// DefaultIsSkippable ...
	DefaultIsSkippable = false
)

// -------------------
// --- Struct methods

// Normalize ...
func (env EnvironmentItemModel) Normalize() error {
	opts, err := env.GetOptions()
	if err != nil {
		return err
	}
	env[optionsKey] = opts
	return nil
}

// Validate ...
func (env EnvironmentItemModel) Validate() error {
	key, _, err := env.GetKeyValuePair()
	if err != nil {
		return err
	}
	if key == "" {
		return errors.New("Invalid environment: empty env_key")
	}

	log.Debugln("-> validate")
	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Title == nil || *options.Title == "" {
		return errors.New("Invalid environment: missing or empty title")
	}

	return nil
}

// FillMissingDeafults ...
func (env *EnvironmentItemModel) FillMissingDeafults() error {
	defaultString := ""

	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Description == nil {
		options.Description = &defaultString
	}
	if options.IsRequired == nil {
		options.IsRequired = &DefaultIsRequired
	}
	if options.IsExpand == nil {
		options.IsExpand = &DefaultIsExpand
	}
	if options.IsDontChangeValue == nil {
		options.IsDontChangeValue = &DefaultIsDontChangeValue
	}
	return nil
}

// Normalize ...
func (step StepModel) Normalize() error {
	for _, input := range step.Inputs {
		if err := input.Normalize(); err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		if err := output.Normalize(); err != nil {
			return err
		}
	}
	return nil
}

// Validate ...
func (step StepModel) Validate() error {
	if step.Title == nil || *step.Title == "" {
		return errors.New("Invalid step: missing or empty title")
	}
	if step.Summary == nil || *step.Summary == "" {
		return errors.New("Invalid step: missing or empty summary")
	}
	if step.Website == nil || *step.Website == "" {
		return errors.New("Invalid step: missing or empty website")
	}
	if step.Source.Git == nil || *step.Source.Git == "" {
		return errors.New("Invalid step: missing or empty source")
	}
	for _, input := range step.Inputs {
		err := input.Validate()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.Validate()
		if err != nil {
			return err
		}
	}
	return nil
}

// FillMissingDeafults ...
func (step *StepModel) FillMissingDeafults() error {
	defaultString := ""

	if step.Description == nil {
		step.Description = &defaultString
	}
	if step.SourceCodeURL == nil {
		step.SourceCodeURL = &defaultString
	}
	if step.SupportURL == nil {
		step.SupportURL = &defaultString
	}
	if step.IsRequiresAdminUser == nil {
		step.IsRequiresAdminUser = &DefaultIsRequiresAdminUser
	}
	if step.IsAlwaysRun == nil {
		step.IsAlwaysRun = &DefaultIsAlwaysRun
	}
	if step.IsSkippable == nil {
		step.IsSkippable = &DefaultIsSkippable
	}
	if step.RunIf == nil {
		step.RunIf = &defaultString
	}

	for _, input := range step.Inputs {
		err := input.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	return nil
}

// GetStep ...
func (collection StepCollectionModel) GetStep(id, version string) (StepModel, bool) {
	stepHash := collection.Steps
	stepVersions, found := stepHash[id]
	if !found {
		return StepModel{}, false
	}
	step, found := stepVersions.Versions[version]
	if !found {
		return StepModel{}, false
	}
	return step, true
}

// GetDownloadLocations ...
func (collection StepCollectionModel) GetDownloadLocations(id, version string) ([]DownloadLocationModel, error) {
	step, found := collection.GetStep(id, version)
	if found == false {
		return []DownloadLocationModel{}, fmt.Errorf("Collection doesn't contains step %s (%s)", id, version)
	}

	locations := []DownloadLocationModel{}
	for _, downloadLocation := range collection.DownloadLocations {
		switch downloadLocation.Type {
		case "zip":
			url := downloadLocation.Src + id + "/" + version + "/step.zip"
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  url,
			}
			locations = append(locations, location)
		case "git":
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  *step.Source.Git,
			}
			locations = append(locations, location)
		default:
			return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - Invalid download location (%#v) for step (%#v)", downloadLocation, id)
		}
	}
	if len(locations) < 1 {
		return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - No download location found for step (%#v)", id)
	}
	return locations, nil
}

// CompareVersions ...
// semantic version (X.Y.Z)
// 1 if version 2 is greater then version 1, -1 if not
// -2 & error if can't compare (if not supported component found)
func CompareVersions(version1, version2 string) (int, error) {
	version1Slice := strings.Split(version1, ".")
	version2Slice := strings.Split(version2, ".")

	lenDiff := len(version1Slice) - len(version2Slice)
	if lenDiff != 0 {
		makeDefVerComps := func(compLen int) []string {
			comps := make([]string, compLen, compLen)
			for idx := len(comps) - 1; idx >= 0; idx-- {
				comps[idx] = "0"
			}
			return comps
		}
		if lenDiff > 0 {
			// v1 slice is longer
			version2Slice = append(version2Slice, makeDefVerComps(lenDiff)...)
		} else {
			// v2 slice is longer
			version1Slice = append(version1Slice, makeDefVerComps(-lenDiff)...)
		}
	}

	cnt := len(version1Slice)
	for i, num := range version1Slice {
		num1, err := strconv.ParseInt(num, 0, 64)
		if err != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err)
			return -2, err
		}

		num2, err2 := strconv.ParseInt(version2Slice[i], 0, 64)
		if err2 != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err2)
			return -2, err
		}

		if num2 > num1 {
			return 1, nil
		}
		if i == cnt-1 {
			// last one
			if num2 == num1 {
				return 0, nil
			}
		}
	}
	return -1, nil
}

// GetLatestStepVersion ...
func (collection StepCollectionModel) GetLatestStepVersion(id string) (string, error) {
	stepHash := collection.Steps
	stepGroup, found := stepHash[id]
	if !found {
		return "", fmt.Errorf("Collection doesn't contains step %s", id)
	}

	if stepGroup.LatestVersionNumber == "" {
		return "", fmt.Errorf("Failed to find latest version of step %s", id)
	}

	return stepGroup.LatestVersionNumber, nil
}

// GetKeyValuePair ...
func (env EnvironmentItemModel) GetKeyValuePair() (string, string, error) {
	if len(env) > 2 {
		return "", "", errors.New("Invalid env: more then 2 fields ")
	}

	retKey := ""
	retValue := ""

	for key, value := range env {
		if key != optionsKey {
			if retKey != "" {
				return "", "", errors.New("Invalid env: more then 1 key-value field found!")
			}

			valueStr, ok := value.(string)
			if !ok {
				if value == nil {
					valueStr = ""
				} else {
					return "", "", fmt.Errorf("Invalid value (key:%#v) (value:%#v)", key, value)
				}
			}

			retKey = key
			retValue = valueStr
		}
	}

	if retKey == "" {
		return "", "", errors.New("Invalid env: no envKey specified!")
	}

	return retKey, retValue, nil
}

// ParseFromInterfaceMap ...
func (envSerModel *EnvironmentItemOptionsModel) ParseFromInterfaceMap(input map[interface{}]interface{}) error {
	for key, value := range input {
		keyStr, ok := key.(string)
		if !ok {
			return fmt.Errorf("Invalid key, should be a string: %#v", key)
		}
		log.Debugf("  ** processing (key:%#v) (value:%#v) (envSerModel:%#v)", key, value, envSerModel)
		switch keyStr {
		case "title":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Title = &castedValue
		case "description":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Description = &castedValue
		case "value_options":
			castedValue, ok := value.([]string)
			if !ok {
				// try with []interface{} instead and cast the
				//  items to string
				castedValue = []string{}
				interfArr, ok := value.([]interface{})
				if !ok {
					return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
				}
				for _, interfItm := range interfArr {
					castedItm, ok := interfItm.(string)
					if !ok {
						return fmt.Errorf("Invalid value in value_options (%#v), not a string: %#v", interfArr, interfItm)
					}
					castedValue = append(castedValue, castedItm)
				}
			}
			envSerModel.ValueOptions = castedValue
		case "is_required":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsRequired = &castedValue
		case "is_expand":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsExpand = &castedValue
		case "is_dont_change_value":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsDontChangeValue = &castedValue
		default:
			return fmt.Errorf("Not supported key found in options: %#v", key)
		}
	}
	return nil
}

// GetOptions ...
func (env EnvironmentItemModel) GetOptions() (EnvironmentItemOptionsModel, error) {
	value, found := env[optionsKey]
	if !found {
		return EnvironmentItemOptionsModel{}, nil
	}

	envItmCasted, ok := value.(EnvironmentItemOptionsModel)
	if ok {
		return envItmCasted, nil
	}

	log.Debugf(" * processing env:%#v", env)

	// if it's read from a file (YAML/JSON) then it's most likely not the proper type
	//  so cast it from the generic interface-interface map
	optionsInterfaceMap, ok := value.(map[interface{}]interface{})
	if !ok {
		return EnvironmentItemOptionsModel{}, fmt.Errorf("Invalid options (value:%#v) - failed to map-interface cast", value)
	}

	options := EnvironmentItemOptionsModel{}
	err := options.ParseFromInterfaceMap(optionsInterfaceMap)
	if err != nil {
		return EnvironmentItemOptionsModel{}, err
	}

	log.Debugf("Parsed options: %#v\n", options)

	return options, nil
}
<commit_msg>normalize env fix<commit_after>package models

import (
	"errors"
	"fmt"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	optionsKey = "opts"
)

var (
	//DefaultIsRequired ...
	DefaultIsRequired = false
	// DefaultIsExpand ...
	DefaultIsExpand = true
	// DefaultIsDontChangeValue ...
	DefaultIsDontChangeValue = false
	// DefaultIsAlwaysRun ...
	DefaultIsAlwaysRun = false
	// DefaultIsRequiresAdminUser ...
	DefaultIsRequiresAdminUser = false
	// DefaultIsSkippable ...
	DefaultIsSkippable = false
)

// -------------------
// --- Struct methods

// Normalize ...
func (env *EnvironmentItemModel) Normalize() error {
	opts, err := env.GetOptions()
	if err != nil {
		return err
	}
	(*env)[optionsKey] = opts
	return nil
}

// Validate ...
func (env EnvironmentItemModel) Validate() error {
	key, _, err := env.GetKeyValuePair()
	if err != nil {
		return err
	}
	if key == "" {
		return errors.New("Invalid environment: empty env_key")
	}

	log.Debugln("-> validate")
	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Title == nil || *options.Title == "" {
		return errors.New("Invalid environment: missing or empty title")
	}

	return nil
}

// FillMissingDeafults ...
func (env *EnvironmentItemModel) FillMissingDeafults() error {
	defaultString := ""

	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Description == nil {
		options.Description = &defaultString
	}
	if options.IsRequired == nil {
		options.IsRequired = &DefaultIsRequired
	}
	if options.IsExpand == nil {
		options.IsExpand = &DefaultIsExpand
	}
	if options.IsDontChangeValue == nil {
		options.IsDontChangeValue = &DefaultIsDontChangeValue
	}
	return nil
}

// Normalize ...
func (step StepModel) Normalize() error {
	for _, input := range step.Inputs {
		if err := input.Normalize(); err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		if err := output.Normalize(); err != nil {
			return err
		}
	}
	return nil
}

// Validate ...
func (step StepModel) Validate() error {
	if step.Title == nil || *step.Title == "" {
		return errors.New("Invalid step: missing or empty title")
	}
	if step.Summary == nil || *step.Summary == "" {
		return errors.New("Invalid step: missing or empty summary")
	}
	if step.Website == nil || *step.Website == "" {
		return errors.New("Invalid step: missing or empty website")
	}
	if step.Source.Git == nil || *step.Source.Git == "" {
		return errors.New("Invalid step: missing or empty source")
	}
	for _, input := range step.Inputs {
		err := input.Validate()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.Validate()
		if err != nil {
			return err
		}
	}
	return nil
}

// FillMissingDeafults ...
func (step *StepModel) FillMissingDeafults() error {
	defaultString := ""

	if step.Description == nil {
		step.Description = &defaultString
	}
	if step.SourceCodeURL == nil {
		step.SourceCodeURL = &defaultString
	}
	if step.SupportURL == nil {
		step.SupportURL = &defaultString
	}
	if step.IsRequiresAdminUser == nil {
		step.IsRequiresAdminUser = &DefaultIsRequiresAdminUser
	}
	if step.IsAlwaysRun == nil {
		step.IsAlwaysRun = &DefaultIsAlwaysRun
	}
	if step.IsSkippable == nil {
		step.IsSkippable = &DefaultIsSkippable
	}
	if step.RunIf == nil {
		step.RunIf = &defaultString
	}

	for _, input := range step.Inputs {
		err := input.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	return nil
}

// GetStep ...
func (collection StepCollectionModel) GetStep(id, version string) (StepModel, bool) {
	stepHash := collection.Steps
	stepVersions, found := stepHash[id]
	if !found {
		return StepModel{}, false
	}
	step, found := stepVersions.Versions[version]
	if !found {
		return StepModel{}, false
	}
	return step, true
}

// GetDownloadLocations ...
func (collection StepCollectionModel) GetDownloadLocations(id, version string) ([]DownloadLocationModel, error) {
	step, found := collection.GetStep(id, version)
	if found == false {
		return []DownloadLocationModel{}, fmt.Errorf("Collection doesn't contains step %s (%s)", id, version)
	}

	locations := []DownloadLocationModel{}
	for _, downloadLocation := range collection.DownloadLocations {
		switch downloadLocation.Type {
		case "zip":
			url := downloadLocation.Src + id + "/" + version + "/step.zip"
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  url,
			}
			locations = append(locations, location)
		case "git":
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  *step.Source.Git,
			}
			locations = append(locations, location)
		default:
			return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - Invalid download location (%#v) for step (%#v)", downloadLocation, id)
		}
	}
	if len(locations) < 1 {
		return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - No download location found for step (%#v)", id)
	}
	return locations, nil
}

// CompareVersions ...
// semantic version (X.Y.Z)
// 1 if version 2 is greater then version 1, -1 if not
// -2 & error if can't compare (if not supported component found)
func CompareVersions(version1, version2 string) (int, error) {
	version1Slice := strings.Split(version1, ".")
	version2Slice := strings.Split(version2, ".")

	lenDiff := len(version1Slice) - len(version2Slice)
	if lenDiff != 0 {
		makeDefVerComps := func(compLen int) []string {
			comps := make([]string, compLen, compLen)
			for idx := len(comps) - 1; idx >= 0; idx-- {
				comps[idx] = "0"
			}
			return comps
		}
		if lenDiff > 0 {
			// v1 slice is longer
			version2Slice = append(version2Slice, makeDefVerComps(lenDiff)...)
		} else {
			// v2 slice is longer
			version1Slice = append(version1Slice, makeDefVerComps(-lenDiff)...)
		}
	}

	cnt := len(version1Slice)
	for i, num := range version1Slice {
		num1, err := strconv.ParseInt(num, 0, 64)
		if err != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err)
			return -2, err
		}

		num2, err2 := strconv.ParseInt(version2Slice[i], 0, 64)
		if err2 != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err2)
			return -2, err
		}

		if num2 > num1 {
			return 1, nil
		}
		if i == cnt-1 {
			// last one
			if num2 == num1 {
				return 0, nil
			}
		}
	}
	return -1, nil
}

// GetLatestStepVersion ...
func (collection StepCollectionModel) GetLatestStepVersion(id string) (string, error) {
	stepHash := collection.Steps
	stepGroup, found := stepHash[id]
	if !found {
		return "", fmt.Errorf("Collection doesn't contains step %s", id)
	}

	if stepGroup.LatestVersionNumber == "" {
		return "", fmt.Errorf("Failed to find latest version of step %s", id)
	}

	return stepGroup.LatestVersionNumber, nil
}

// GetKeyValuePair ...
func (env EnvironmentItemModel) GetKeyValuePair() (string, string, error) {
	if len(env) > 2 {
		return "", "", errors.New("Invalid env: more then 2 fields ")
	}

	retKey := ""
	retValue := ""

	for key, value := range env {
		if key != optionsKey {
			if retKey != "" {
				return "", "", errors.New("Invalid env: more then 1 key-value field found!")
			}

			valueStr, ok := value.(string)
			if !ok {
				if value == nil {
					valueStr = ""
				} else {
					return "", "", fmt.Errorf("Invalid value (key:%#v) (value:%#v)", key, value)
				}
			}

			retKey = key
			retValue = valueStr
		}
	}

	if retKey == "" {
		return "", "", errors.New("Invalid env: no envKey specified!")
	}

	return retKey, retValue, nil
}

// ParseFromInterfaceMap ...
func (envSerModel *EnvironmentItemOptionsModel) ParseFromInterfaceMap(input map[interface{}]interface{}) error {
	for key, value := range input {
		keyStr, ok := key.(string)
		if !ok {
			return fmt.Errorf("Invalid key, should be a string: %#v", key)
		}
		log.Debugf("  ** processing (key:%#v) (value:%#v) (envSerModel:%#v)", key, value, envSerModel)
		switch keyStr {
		case "title":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Title = &castedValue
		case "description":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Description = &castedValue
		case "value_options":
			castedValue, ok := value.([]string)
			if !ok {
				// try with []interface{} instead and cast the
				//  items to string
				castedValue = []string{}
				interfArr, ok := value.([]interface{})
				if !ok {
					return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
				}
				for _, interfItm := range interfArr {
					castedItm, ok := interfItm.(string)
					if !ok {
						return fmt.Errorf("Invalid value in value_options (%#v), not a string: %#v", interfArr, interfItm)
					}
					castedValue = append(castedValue, castedItm)
				}
			}
			envSerModel.ValueOptions = castedValue
		case "is_required":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsRequired = &castedValue
		case "is_expand":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsExpand = &castedValue
		case "is_dont_change_value":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsDontChangeValue = &castedValue
		default:
			return fmt.Errorf("Not supported key found in options: %#v", key)
		}
	}
	return nil
}

// GetOptions ...
func (env EnvironmentItemModel) GetOptions() (EnvironmentItemOptionsModel, error) {
	value, found := env[optionsKey]
	if !found {
		return EnvironmentItemOptionsModel{}, nil
	}

	envItmCasted, ok := value.(EnvironmentItemOptionsModel)
	if ok {
		return envItmCasted, nil
	}

	log.Debugf(" * processing env:%#v", env)

	// if it's read from a file (YAML/JSON) then it's most likely not the proper type
	//  so cast it from the generic interface-interface map
	optionsInterfaceMap, ok := value.(map[interface{}]interface{})
	if !ok {
		return EnvironmentItemOptionsModel{}, fmt.Errorf("Invalid options (value:%#v) - failed to map-interface cast", value)
	}

	options := EnvironmentItemOptionsModel{}
	err := options.ParseFromInterfaceMap(optionsInterfaceMap)
	if err != nil {
		return EnvironmentItemOptionsModel{}, err
	}

	log.Debugf("Parsed options: %#v\n", options)

	return options, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"fmt"
	"io"
	"sort"

	"golang.org/x/net/context"
)

// Create a read proxy consisting of the contents defined by the supplied
// refreshers concatenated. See NewReadProxy for more.
//
// If rl is non-nil, it will be used as the first temporary copy of the
// contents, and must match the concatenation of the content returned by the
// refreshers.
func NewMultiReadProxy(
	fl FileLeaser,
	refreshers []Refresher,
	rl ReadLease) (rp ReadProxy) {
	// Create one wrapped read proxy per refresher.
	var wrappedProxies []readProxyAndOffset
	var size int64
	for _, r := range refreshers {
		wrapped := NewReadProxy(fl, r, nil)
		wrappedProxies = append(wrappedProxies, readProxyAndOffset{size, wrapped})
		size += wrapped.Size()
	}

	rp = &multiReadProxy{
		size:   size,
		leaser: fl,
		rps:    wrappedProxies,
		lease:  rl,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Implementation
////////////////////////////////////////////////////////////////////////

type multiReadProxy struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	// The size of the proxied content.
	size int64

	/////////////////////////
	// Dependencies
	/////////////////////////

	leaser FileLeaser

	// The wrapped read proxies, indexed by their logical starting offset.
	//
	// INVARIANT: If len(rps) != 0, rps[0].off == 0
	// INVARIANT: For each x, x.rp.Size() >= 0
	// INVARIANT: For each i>0, rps[i].off == rps[i-i].off + rps[i-i].rp.Size()
	// INVARIANT: size is the sum over the wrapped proxy sizes.
	rps []readProxyAndOffset

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A read lease for the entire contents. May be nil.
	//
	// INVARIANT: If lease != nil, size == lease.Size()
	lease ReadLease
}

func (mrp *multiReadProxy) Size() (size int64) {
	size = mrp.size
	return
}

func (mrp *multiReadProxy) ReadAt(
	ctx context.Context,
	p []byte,
	off int64) (n int, err error) {
	// Special case: we don't support negative offsets, silly user.
	if off < 0 {
		err = fmt.Errorf("Invalid offset: %v", off)
		return
	}

	// Special case: offsets at or beyond the end of our content can never yield
	// any content, and the io.ReaderAt spec allows us to return EOF. Knock them
	// out here so we know off is in range when we start below.
	if off >= mrp.Size() {
		err = io.EOF
		return
	}

	// The read proxy that contains off is the *last* read proxy whose start
	// offset is less than or equal to off. Find the first that is greater and
	// move back one.
	//
	// Because we handled the special cases above, this must be in range.
	wrappedIndex := mrp.upperBound(off) - 1

	if wrappedIndex < 0 || wrappedIndex >= len(mrp.rps) {
		panic(fmt.Sprintf("Unexpected index: %v", wrappedIndex))
	}

	// Keep going until we've got nothing left to do.
	for len(p) > 0 {
		// Have we run out of wrapped read proxies?
		if wrappedIndex == len(mrp.rps) {
			err = io.EOF
			return
		}

		// Read from the wrapped proxy, accumulating into our total before checking
		// for a read error.
		wrappedN, wrappedErr := mrp.readFromOne(ctx, wrappedIndex, p, off)
		n += wrappedN
		if wrappedErr != nil {
			err = wrappedErr
			return
		}

		// readFromOne guarantees to either fill our buffer or exhaust the wrapped
		// proxy. So advance the buffer, the offset, and the wrapped proxy index
		// and go again.
		p = p[wrappedN:]
		off += int64(wrappedN)
		wrappedIndex++
	}

	return
}

func (mrp *multiReadProxy) Upgrade(
	ctx context.Context) (rwl ReadWriteLease, err error) {
	// Create a new read/write lease to return to the user. Ensure that it is
	// destroyed if we return in error.
	rwl, err = mrp.leaser.NewFile()
	if err != nil {
		err = fmt.Errorf("NewFile: %v", err)
		return
	}

	defer func() {
		if err != nil {
			rwl.Downgrade().Revoke()
		}
	}()

	// Accumulate each wrapped read proxy in turn.
	for i, entry := range mrp.rps {
		err = mrp.upgradeOne(ctx, rwl, entry.rp)
		if err != nil {
			err = fmt.Errorf("upgradeOne(%d): %v", i, err)
			return
		}
	}

	return
}

func (mrp *multiReadProxy) Destroy() {
	// Destroy all of the wrapped proxies.
	for _, entry := range mrp.rps {
		entry.rp.Destroy()
	}

	// Destroy the lease for the entire contents, if any.
	if mrp.lease != nil {
		mrp.lease.Revoke()
	}

	// Crash early if called again.
	mrp.rps = nil
	mrp.lease = nil
}

func (mrp *multiReadProxy) CheckInvariants() {
	// INVARIANT: If len(rps) != 0, rps[0].off == 0
	if len(mrp.rps) != 0 && mrp.rps[0].off != 0 {
		panic(fmt.Sprintf("Unexpected starting point: %v", mrp.rps[0].off))
	}

	// INVARIANT: For each x, x.rp.Size() >= 0
	for _, x := range mrp.rps {
		if x.rp.Size() < 0 {
			panic(fmt.Sprintf("Negative size: %v", x.rp.Size()))
		}
	}

	// INVARIANT: For each i>0, rps[i].off == rps[i-i].off + rps[i-i].rp.Size()
	for i := range mrp.rps {
		if i > 0 && !(mrp.rps[i].off == mrp.rps[i-1].off+mrp.rps[i-1].rp.Size()) {
			panic("Offsets are not indexed correctly.")
		}
	}

	// INVARIANT: size is the sum over the wrapped proxy sizes.
	var sum int64
	for _, wrapped := range mrp.rps {
		sum += wrapped.rp.Size()
	}

	if sum != mrp.size {
		panic(fmt.Sprintf("Size mismatch: %v vs. %v", sum, mrp.size))
	}

	// INVARIANT: If lease != nil, size == lease.Size()
	if mrp.lease != nil && mrp.size != mrp.lease.Size() {
		panic(fmt.Sprintf("Size mismatch: %v vs. %v", mrp.size, mrp.lease.Size()))
	}
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type readProxyAndOffset struct {
	off int64
	rp  ReadProxy
}

// Return the index within mrp.rps of the first read proxy whose logical offset
// is greater than off. If there is none, return len(mrp.rps).
func (mrp *multiReadProxy) upperBound(off int64) (index int) {
	pred := func(i int) bool {
		return mrp.rps[i].off > off
	}

	return sort.Search(len(mrp.rps), pred)
}

// Serve a read from the wrapped proxy at the given index within our array of
// wrapped proxies. The offset is relative to the start of the multiReadProxy,
// not the wrapped proxy.
//
// Guarantees, letting wrapped be mrp.rps[i].rp and wrappedStart be
// mrp.rps[i].off:
//
//  *  If err == nil, n == len(p) || off + n == wrappedStart + wrapped.Size().
//  *  Never returns err == io.EOF.
//
// REQUIRES: index < len(mrp.rps)
// REQUIRES: mrp.rps[index].off <= off < mrp.rps[index].off + wrapped.Size()
func (mrp *multiReadProxy) readFromOne(
	ctx context.Context,
	index int,
	p []byte,
	off int64) (n int, err error) {
	// Check input requirements.
	if !(index < len(mrp.rps)) {
		panic(fmt.Sprintf("Out of range wrapped index: %v", index))
	}

	wrapped := mrp.rps[index].rp
	wrappedStart := mrp.rps[index].off
	wrappedSize := wrapped.Size()

	if !(wrappedStart <= off && off < wrappedStart+wrappedSize) {
		panic(fmt.Sprintf(
			"Offset %v not in range [%v, %v)",
			off,
			wrappedStart,
			wrappedStart+wrappedSize))
	}

	// Check guarantees on return.
	defer func() {
		if err == nil &&
			!(n == len(p) || off+int64(n) == wrappedStart+wrappedSize) {
			panic(fmt.Sprintf(
				"Failed to serve full read. "+
					"off: %d n: %d, len(p): %d, wrapped start: %d, wrapped size: %d",
				off,
				n,
				len(p),
				wrappedStart,
				wrappedSize))

			return
		}

		if err == io.EOF {
			panic("Unexpected EOF.")
		}
	}()

	// Read from the wrapped reader, translating the offset. We rely on the
	// wrapped reader to properly implement ReadAt, not returning a short read.
	wrappedOff := off - wrappedStart
	n, err = wrapped.ReadAt(ctx, p, wrappedOff)

	// Sanity check: the wrapped read proxy is supposed to return err == nil only
	// if the entire read was satisfied.
	if err == nil && n != len(p) {
		err = fmt.Errorf(
			"Wrapped proxy %d returned only %d bytes for a %d-byte read "+
				"starting at wrapped offset %d",
			index,
			n,
			len(p),
			wrappedOff)

		return
	}

	// Don't return io.EOF, as guaranteed.
	if err == io.EOF {
		// Sanity check: if we hit EOF, that should mean that we read up to the end
		// of the wrapped range.
		if int64(n) != wrappedSize-wrappedOff {
			err = fmt.Errorf(
				"Wrapped proxy %d returned unexpected EOF. n: %d, wrapped size: %d, "+
					"wrapped offset: %d",
				index,
				n,
				wrappedSize,
				wrappedOff)

			return
		}

		err = nil
	}

	return
}

// Upgrade the read proxy and copy its contents into the supplied read/write
// lease, then destroy it.
func (mrp *multiReadProxy) upgradeOne(
	ctx context.Context,
	dst ReadWriteLease,
	rp ReadProxy) (err error) {
	// Upgrade.
	src, err := rp.Upgrade(ctx)
	if err != nil {
		err = fmt.Errorf("Upgrade: %v", err)
		return
	}

	defer func() {
		src.Downgrade().Revoke()
	}()

	// Seek to the start and copy.
	_, err = src.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	_, err = io.Copy(dst, src)
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	return
}
<commit_msg>Make use after destroy clearer.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"fmt"
	"io"
	"sort"

	"golang.org/x/net/context"
)

// Create a read proxy consisting of the contents defined by the supplied
// refreshers concatenated. See NewReadProxy for more.
//
// If rl is non-nil, it will be used as the first temporary copy of the
// contents, and must match the concatenation of the content returned by the
// refreshers.
func NewMultiReadProxy(
	fl FileLeaser,
	refreshers []Refresher,
	rl ReadLease) (rp ReadProxy) {
	// Create one wrapped read proxy per refresher.
	var wrappedProxies []readProxyAndOffset
	var size int64
	for _, r := range refreshers {
		wrapped := NewReadProxy(fl, r, nil)
		wrappedProxies = append(wrappedProxies, readProxyAndOffset{size, wrapped})
		size += wrapped.Size()
	}

	rp = &multiReadProxy{
		size:   size,
		leaser: fl,
		rps:    wrappedProxies,
		lease:  rl,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Implementation
////////////////////////////////////////////////////////////////////////

type multiReadProxy struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	// The size of the proxied content.
	size int64

	/////////////////////////
	// Dependencies
	/////////////////////////

	leaser FileLeaser

	// The wrapped read proxies, indexed by their logical starting offset.
	//
	// INVARIANT: If len(rps) != 0, rps[0].off == 0
	// INVARIANT: For each x, x.rp.Size() >= 0
	// INVARIANT: For each i>0, rps[i].off == rps[i-i].off + rps[i-i].rp.Size()
	// INVARIANT: size is the sum over the wrapped proxy sizes.
	rps []readProxyAndOffset

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A read lease for the entire contents. May be nil.
	//
	// INVARIANT: If lease != nil, size == lease.Size()
	lease ReadLease

	destroyed bool
}

func (mrp *multiReadProxy) Size() (size int64) {
	size = mrp.size
	return
}

func (mrp *multiReadProxy) ReadAt(
	ctx context.Context,
	p []byte,
	off int64) (n int, err error) {
	// Special case: we don't support negative offsets, silly user.
	if off < 0 {
		err = fmt.Errorf("Invalid offset: %v", off)
		return
	}

	// Special case: offsets at or beyond the end of our content can never yield
	// any content, and the io.ReaderAt spec allows us to return EOF. Knock them
	// out here so we know off is in range when we start below.
	if off >= mrp.Size() {
		err = io.EOF
		return
	}

	// The read proxy that contains off is the *last* read proxy whose start
	// offset is less than or equal to off. Find the first that is greater and
	// move back one.
	//
	// Because we handled the special cases above, this must be in range.
	wrappedIndex := mrp.upperBound(off) - 1

	if wrappedIndex < 0 || wrappedIndex >= len(mrp.rps) {
		panic(fmt.Sprintf("Unexpected index: %v", wrappedIndex))
	}

	// Keep going until we've got nothing left to do.
	for len(p) > 0 {
		// Have we run out of wrapped read proxies?
		if wrappedIndex == len(mrp.rps) {
			err = io.EOF
			return
		}

		// Read from the wrapped proxy, accumulating into our total before checking
		// for a read error.
		wrappedN, wrappedErr := mrp.readFromOne(ctx, wrappedIndex, p, off)
		n += wrappedN
		if wrappedErr != nil {
			err = wrappedErr
			return
		}

		// readFromOne guarantees to either fill our buffer or exhaust the wrapped
		// proxy. So advance the buffer, the offset, and the wrapped proxy index
		// and go again.
		p = p[wrappedN:]
		off += int64(wrappedN)
		wrappedIndex++
	}

	return
}

func (mrp *multiReadProxy) Upgrade(
	ctx context.Context) (rwl ReadWriteLease, err error) {
	// Create a new read/write lease to return to the user. Ensure that it is
	// destroyed if we return in error.
	rwl, err = mrp.leaser.NewFile()
	if err != nil {
		err = fmt.Errorf("NewFile: %v", err)
		return
	}

	defer func() {
		if err != nil {
			rwl.Downgrade().Revoke()
		}
	}()

	// Accumulate each wrapped read proxy in turn.
	for i, entry := range mrp.rps {
		err = mrp.upgradeOne(ctx, rwl, entry.rp)
		if err != nil {
			err = fmt.Errorf("upgradeOne(%d): %v", i, err)
			return
		}
	}

	return
}

func (mrp *multiReadProxy) Destroy() {
	// Destroy all of the wrapped proxies.
	for _, entry := range mrp.rps {
		entry.rp.Destroy()
	}

	// Destroy the lease for the entire contents, if any.
	if mrp.lease != nil {
		mrp.lease.Revoke()
	}

	// Crash early if called again.
	mrp.rps = nil
	mrp.lease = nil
	mrp.destroyed = true
}

func (mrp *multiReadProxy) CheckInvariants() {
	if mrp.destroyed {
		panic("Use after destroyed")
	}

	// INVARIANT: If len(rps) != 0, rps[0].off == 0
	if len(mrp.rps) != 0 && mrp.rps[0].off != 0 {
		panic(fmt.Sprintf("Unexpected starting point: %v", mrp.rps[0].off))
	}

	// INVARIANT: For each x, x.rp.Size() >= 0
	for _, x := range mrp.rps {
		if x.rp.Size() < 0 {
			panic(fmt.Sprintf("Negative size: %v", x.rp.Size()))
		}
	}

	// INVARIANT: For each i>0, rps[i].off == rps[i-i].off + rps[i-i].rp.Size()
	for i := range mrp.rps {
		if i > 0 && !(mrp.rps[i].off == mrp.rps[i-1].off+mrp.rps[i-1].rp.Size()) {
			panic("Offsets are not indexed correctly.")
		}
	}

	// INVARIANT: size is the sum over the wrapped proxy sizes.
	var sum int64
	for _, wrapped := range mrp.rps {
		sum += wrapped.rp.Size()
	}

	if sum != mrp.size {
		panic(fmt.Sprintf("Size mismatch: %v vs. %v", sum, mrp.size))
	}

	// INVARIANT: If lease != nil, size == lease.Size()
	if mrp.lease != nil && mrp.size != mrp.lease.Size() {
		panic(fmt.Sprintf("Size mismatch: %v vs. %v", mrp.size, mrp.lease.Size()))
	}
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type readProxyAndOffset struct {
	off int64
	rp  ReadProxy
}

// Return the index within mrp.rps of the first read proxy whose logical offset
// is greater than off. If there is none, return len(mrp.rps).
func (mrp *multiReadProxy) upperBound(off int64) (index int) {
	pred := func(i int) bool {
		return mrp.rps[i].off > off
	}

	return sort.Search(len(mrp.rps), pred)
}

// Serve a read from the wrapped proxy at the given index within our array of
// wrapped proxies. The offset is relative to the start of the multiReadProxy,
// not the wrapped proxy.
//
// Guarantees, letting wrapped be mrp.rps[i].rp and wrappedStart be
// mrp.rps[i].off:
//
//  *  If err == nil, n == len(p) || off + n == wrappedStart + wrapped.Size().
//  *  Never returns err == io.EOF.
//
// REQUIRES: index < len(mrp.rps)
// REQUIRES: mrp.rps[index].off <= off < mrp.rps[index].off + wrapped.Size()
func (mrp *multiReadProxy) readFromOne(
	ctx context.Context,
	index int,
	p []byte,
	off int64) (n int, err error) {
	// Check input requirements.
	if !(index < len(mrp.rps)) {
		panic(fmt.Sprintf("Out of range wrapped index: %v", index))
	}

	wrapped := mrp.rps[index].rp
	wrappedStart := mrp.rps[index].off
	wrappedSize := wrapped.Size()

	if !(wrappedStart <= off && off < wrappedStart+wrappedSize) {
		panic(fmt.Sprintf(
			"Offset %v not in range [%v, %v)",
			off,
			wrappedStart,
			wrappedStart+wrappedSize))
	}

	// Check guarantees on return.
	defer func() {
		if err == nil &&
			!(n == len(p) || off+int64(n) == wrappedStart+wrappedSize) {
			panic(fmt.Sprintf(
				"Failed to serve full read. "+
					"off: %d n: %d, len(p): %d, wrapped start: %d, wrapped size: %d",
				off,
				n,
				len(p),
				wrappedStart,
				wrappedSize))

			return
		}

		if err == io.EOF {
			panic("Unexpected EOF.")
		}
	}()

	// Read from the wrapped reader, translating the offset. We rely on the
	// wrapped reader to properly implement ReadAt, not returning a short read.
	wrappedOff := off - wrappedStart
	n, err = wrapped.ReadAt(ctx, p, wrappedOff)

	// Sanity check: the wrapped read proxy is supposed to return err == nil only
	// if the entire read was satisfied.
	if err == nil && n != len(p) {
		err = fmt.Errorf(
			"Wrapped proxy %d returned only %d bytes for a %d-byte read "+
				"starting at wrapped offset %d",
			index,
			n,
			len(p),
			wrappedOff)

		return
	}

	// Don't return io.EOF, as guaranteed.
	if err == io.EOF {
		// Sanity check: if we hit EOF, that should mean that we read up to the end
		// of the wrapped range.
		if int64(n) != wrappedSize-wrappedOff {
			err = fmt.Errorf(
				"Wrapped proxy %d returned unexpected EOF. n: %d, wrapped size: %d, "+
					"wrapped offset: %d",
				index,
				n,
				wrappedSize,
				wrappedOff)

			return
		}

		err = nil
	}

	return
}

// Upgrade the read proxy and copy its contents into the supplied read/write
// lease, then destroy it.
func (mrp *multiReadProxy) upgradeOne(
	ctx context.Context,
	dst ReadWriteLease,
	rp ReadProxy) (err error) {
	// Upgrade.
	src, err := rp.Upgrade(ctx)
	if err != nil {
		err = fmt.Errorf("Upgrade: %v", err)
		return
	}

	defer func() {
		src.Downgrade().Revoke()
	}()

	// Seek to the start and copy.
	_, err = src.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	_, err = io.Copy(dst, src)
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	return
}
<|endoftext|>


<commit_before>//go:build linux
// +build linux

package ipvlan

import (
	"fmt"

	"github.com/docker/docker/libnetwork/driverapi"
	"github.com/docker/docker/libnetwork/netlabel"
	"github.com/docker/docker/libnetwork/ns"
	"github.com/docker/docker/libnetwork/options"
	"github.com/docker/docker/libnetwork/osl"
	"github.com/docker/docker/libnetwork/types"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/stringid"
	"github.com/sirupsen/logrus"
)

// CreateNetwork the network for the specified driver type
func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
	defer osl.InitOSContext()()
	kv, err := kernel.GetKernelVersion()
	if err != nil {
		return fmt.Errorf("Failed to check kernel version for %s driver support: %v", ipvlanType, err)
	}
	// ensure Kernel version is >= v4.2 for ipvlan support
	if kv.Kernel < ipvlanKernelVer || (kv.Kernel == ipvlanKernelVer && kv.Major < ipvlanMajorVer) {
		return fmt.Errorf("kernel version failed to meet the minimum ipvlan kernel requirement of %d.%d, found %d.%d.%d",
			ipvlanKernelVer, ipvlanMajorVer, kv.Kernel, kv.Major, kv.Minor)
	}
	// reject a null v4 network
	if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" {
		return fmt.Errorf("ipv4 pool is empty")
	}
	// parse and validate the config and bind to networkConfiguration
	config, err := parseNetworkOptions(nid, option)
	if err != nil {
		return err
	}
	config.processIPAM(ipV4Data, ipV6Data)

	// verify the ipvlan mode from -o ipvlan_mode option
	switch config.IpvlanMode {
	case "", modeL2:
		// default to ipvlan L2 mode if -o ipvlan_mode is empty
		config.IpvlanMode = modeL2
	case modeL3:
		config.IpvlanMode = modeL3
	case modeL3S:
		config.IpvlanMode = modeL3S
	default:
		return fmt.Errorf("requested ipvlan mode '%s' is not valid, 'l2' mode is the ipvlan driver default", config.IpvlanMode)
	}
	// verify the ipvlan flag from -o ipvlan_flag option
	switch config.IpvlanFlag {
	case "", flagBridge:
		// default to bridge if -o ipvlan_flag is empty
		config.IpvlanFlag = flagBridge
	case flagPrivate:
		config.IpvlanFlag = flagPrivate
	case flagVepa:
		config.IpvlanFlag = flagVepa
	default:
		return fmt.Errorf("requested ipvlan flag '%s' is not valid, 'bridge' is the ipvlan driver default", config.IpvlanFlag)
	}
	// loopback is not a valid parent link
	if config.Parent == "lo" {
		return fmt.Errorf("loopback interface is not a valid %s parent link", ipvlanType)
	}
	// if parent interface not specified, create a dummy type link to use named dummy+net_id
	if config.Parent == "" {
		config.Parent = getDummyName(stringid.TruncateID(config.ID))
	}
	foundExisting, err := d.createNetwork(config)
	if err != nil {
		return err
	}

	if foundExisting {
		return types.InternalMaskableErrorf("restoring existing network %s", config.ID)
	}
	// update persistent db, rollback on fail
	err = d.storeUpdate(config)
	if err != nil {
		d.deleteNetwork(config.ID)
		logrus.Debugf("encountered an error rolling back a network create for %s : %v", config.ID, err)
		return err
	}

	return nil
}

// createNetwork is used by new network callbacks and persistent network cache
func (d *driver) createNetwork(config *configuration) (bool, error) {
	foundExisting := false
	networkList := d.getNetworks()
	for _, nw := range networkList {
		if config.Parent == nw.config.Parent {
			if config.ID != nw.config.ID {
				return false, fmt.Errorf("network %s is already using parent interface %s",
					getDummyName(stringid.TruncateID(nw.config.ID)), config.Parent)
			}
			logrus.Debugf("Create Network for the same ID %s\n", config.ID)
			foundExisting = true
			break
		}
	}
	if !parentExists(config.Parent) {
		// Create a dummy link if a dummy name is set for parent
		if dummyName := getDummyName(stringid.TruncateID(config.ID)); dummyName == config.Parent {
			err := createDummyLink(config.Parent, dummyName)
			if err != nil {
				return false, err
			}
			config.CreatedSlaveLink = true

			// notify the user in logs they have limited communications
			logrus.Debugf("Empty -o parent= flags limit communications to other containers inside of network: %s",
				config.Parent)
		} else {
			// if the subinterface parent_iface.vlan_id checks do not pass, return err.
			//  a valid example is 'eth0.10' for a parent iface 'eth0' with a vlan id '10'
			err := createVlanLink(config.Parent)
			if err != nil {
				return false, err
			}
			// if driver created the networks slave link, record it for future deletion
			config.CreatedSlaveLink = true
		}
	}
	if !foundExisting {
		n := &network{
			id:        config.ID,
			driver:    d,
			endpoints: endpointTable{},
			config:    config,
		}
		// add the network
		d.addNetwork(n)
	}

	return foundExisting, nil
}

// DeleteNetwork the network for the specified driver type
func (d *driver) DeleteNetwork(nid string) error {
	defer osl.InitOSContext()()
	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("network id %s not found", nid)
	}
	// if the driver created the slave interface, delete it, otherwise leave it
	if ok := n.config.CreatedSlaveLink; ok {
		// if the interface exists, only delete if it matches iface.vlan or dummy.net_id naming
		if ok := parentExists(n.config.Parent); ok {
			// only delete the link if it is named the net_id
			if n.config.Parent == getDummyName(stringid.TruncateID(nid)) {
				err := delDummyLink(n.config.Parent)
				if err != nil {
					logrus.Debugf("link %s was not deleted, continuing the delete network operation: %v",
						n.config.Parent, err)
				}
			} else {
				// only delete the link if it matches iface.vlan naming
				err := delVlanLink(n.config.Parent)
				if err != nil {
					logrus.Debugf("link %s was not deleted, continuing the delete network operation: %v",
						n.config.Parent, err)
				}
			}
		}
	}
	for _, ep := range n.endpoints {
		if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
			if err := ns.NlHandle().LinkDel(link); err != nil {
				logrus.WithError(err).Warnf("Failed to delete interface (%s)'s link on endpoint (%s) delete", ep.srcName, ep.id)
			}
		}

		if err := d.storeDelete(ep); err != nil {
			logrus.Warnf("Failed to remove ipvlan endpoint %.7s from store: %v", ep.id, err)
		}
	}
	// delete the *network
	d.deleteNetwork(nid)
	// delete the network record from persistent cache
	err := d.storeDelete(n.config)
	if err != nil {
		return fmt.Errorf("error deleting deleting id %s from datastore: %v", nid, err)
	}
	return nil
}

// parseNetworkOptions parse docker network options
func parseNetworkOptions(id string, option options.Generic) (*configuration, error) {
	var (
		err    error
		config = &configuration{}
	)
	// parse generic labels first
	if genData, ok := option[netlabel.GenericData]; ok && genData != nil {
		if config, err = parseNetworkGenericOptions(genData); err != nil {
			return nil, err
		}
	}
	if val, ok := option[netlabel.Internal]; ok {
		if internal, ok := val.(bool); ok && internal {
			config.Internal = true
		}
	}
	config.ID = id
	return config, nil
}

// parseNetworkGenericOptions parse generic driver docker network options
func parseNetworkGenericOptions(data interface{}) (*configuration, error) {
	var (
		err    error
		config *configuration
	)
	switch opt := data.(type) {
	case *configuration:
		config = opt
	case map[string]string:
		config = &configuration{}
		err = config.fromOptions(opt)
	case options.Generic:
		var opaqueConfig interface{}
		if opaqueConfig, err = options.GenerateFromModel(opt, config); err == nil {
			config = opaqueConfig.(*configuration)
		}
	default:
		err = types.BadRequestErrorf("unrecognized network configuration format: %v", opt)
	}
	return config, err
}

// fromOptions binds the generic options to networkConfiguration to cache
func (config *configuration) fromOptions(labels map[string]string) error {
	for label, value := range labels {
		switch label {
		case parentOpt:
			// parse driver option '-o parent'
			config.Parent = value
		case driverModeOpt:
			// parse driver option '-o ipvlan_mode'
			config.IpvlanMode = value
		case driverFlagOpt:
			// parse driver option '-o ipvlan_flag'
			config.IpvlanFlag = value
		}
	}
	return nil
}

// processIPAM parses v4 and v6 IP information and binds it to the network configuration
func (config *configuration) processIPAM(ipamV4Data, ipamV6Data []driverapi.IPAMData) {
	if len(ipamV4Data) > 0 {
		for _, ipd := range ipamV4Data {
			s := &ipSubnet{
				SubnetIP: ipd.Pool.String(),
				GwIP:     ipd.Gateway.String(),
			}
			config.Ipv4Subnets = append(config.Ipv4Subnets, s)
		}
	}
	if len(ipamV6Data) > 0 {
		for _, ipd := range ipamV6Data {
			s := &ipSubnet{
				SubnetIP: ipd.Pool.String(),
				GwIP:     ipd.Gateway.String(),
			}
			config.Ipv6Subnets = append(config.Ipv6Subnets, s)
		}
	}
}
<commit_msg>libnetwork: ipvlan: processIPAM(): simplify<commit_after>//go:build linux
// +build linux

package ipvlan

import (
	"fmt"

	"github.com/docker/docker/libnetwork/driverapi"
	"github.com/docker/docker/libnetwork/netlabel"
	"github.com/docker/docker/libnetwork/ns"
	"github.com/docker/docker/libnetwork/options"
	"github.com/docker/docker/libnetwork/osl"
	"github.com/docker/docker/libnetwork/types"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/stringid"
	"github.com/sirupsen/logrus"
)

// CreateNetwork the network for the specified driver type
func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
	defer osl.InitOSContext()()
	kv, err := kernel.GetKernelVersion()
	if err != nil {
		return fmt.Errorf("Failed to check kernel version for %s driver support: %v", ipvlanType, err)
	}
	// ensure Kernel version is >= v4.2 for ipvlan support
	if kv.Kernel < ipvlanKernelVer || (kv.Kernel == ipvlanKernelVer && kv.Major < ipvlanMajorVer) {
		return fmt.Errorf("kernel version failed to meet the minimum ipvlan kernel requirement of %d.%d, found %d.%d.%d",
			ipvlanKernelVer, ipvlanMajorVer, kv.Kernel, kv.Major, kv.Minor)
	}
	// reject a null v4 network
	if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" {
		return fmt.Errorf("ipv4 pool is empty")
	}
	// parse and validate the config and bind to networkConfiguration
	config, err := parseNetworkOptions(nid, option)
	if err != nil {
		return err
	}
	config.processIPAM(ipV4Data, ipV6Data)

	// verify the ipvlan mode from -o ipvlan_mode option
	switch config.IpvlanMode {
	case "", modeL2:
		// default to ipvlan L2 mode if -o ipvlan_mode is empty
		config.IpvlanMode = modeL2
	case modeL3:
		config.IpvlanMode = modeL3
	case modeL3S:
		config.IpvlanMode = modeL3S
	default:
		return fmt.Errorf("requested ipvlan mode '%s' is not valid, 'l2' mode is the ipvlan driver default", config.IpvlanMode)
	}
	// verify the ipvlan flag from -o ipvlan_flag option
	switch config.IpvlanFlag {
	case "", flagBridge:
		// default to bridge if -o ipvlan_flag is empty
		config.IpvlanFlag = flagBridge
	case flagPrivate:
		config.IpvlanFlag = flagPrivate
	case flagVepa:
		config.IpvlanFlag = flagVepa
	default:
		return fmt.Errorf("requested ipvlan flag '%s' is not valid, 'bridge' is the ipvlan driver default", config.IpvlanFlag)
	}
	// loopback is not a valid parent link
	if config.Parent == "lo" {
		return fmt.Errorf("loopback interface is not a valid %s parent link", ipvlanType)
	}
	// if parent interface not specified, create a dummy type link to use named dummy+net_id
	if config.Parent == "" {
		config.Parent = getDummyName(stringid.TruncateID(config.ID))
	}
	foundExisting, err := d.createNetwork(config)
	if err != nil {
		return err
	}

	if foundExisting {
		return types.InternalMaskableErrorf("restoring existing network %s", config.ID)
	}
	// update persistent db, rollback on fail
	err = d.storeUpdate(config)
	if err != nil {
		d.deleteNetwork(config.ID)
		logrus.Debugf("encountered an error rolling back a network create for %s : %v", config.ID, err)
		return err
	}

	return nil
}

// createNetwork is used by new network callbacks and persistent network cache
func (d *driver) createNetwork(config *configuration) (bool, error) {
	foundExisting := false
	networkList := d.getNetworks()
	for _, nw := range networkList {
		if config.Parent == nw.config.Parent {
			if config.ID != nw.config.ID {
				return false, fmt.Errorf("network %s is already using parent interface %s",
					getDummyName(stringid.TruncateID(nw.config.ID)), config.Parent)
			}
			logrus.Debugf("Create Network for the same ID %s\n", config.ID)
			foundExisting = true
			break
		}
	}
	if !parentExists(config.Parent) {
		// Create a dummy link if a dummy name is set for parent
		if dummyName := getDummyName(stringid.TruncateID(config.ID)); dummyName == config.Parent {
			err := createDummyLink(config.Parent, dummyName)
			if err != nil {
				return false, err
			}
			config.CreatedSlaveLink = true

			// notify the user in logs they have limited communications
			logrus.Debugf("Empty -o parent= flags limit communications to other containers inside of network: %s",
				config.Parent)
		} else {
			// if the subinterface parent_iface.vlan_id checks do not pass, return err.
			//  a valid example is 'eth0.10' for a parent iface 'eth0' with a vlan id '10'
			err := createVlanLink(config.Parent)
			if err != nil {
				return false, err
			}
			// if driver created the networks slave link, record it for future deletion
			config.CreatedSlaveLink = true
		}
	}
	if !foundExisting {
		n := &network{
			id:        config.ID,
			driver:    d,
			endpoints: endpointTable{},
			config:    config,
		}
		// add the network
		d.addNetwork(n)
	}

	return foundExisting, nil
}

// DeleteNetwork the network for the specified driver type
func (d *driver) DeleteNetwork(nid string) error {
	defer osl.InitOSContext()()
	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("network id %s not found", nid)
	}
	// if the driver created the slave interface, delete it, otherwise leave it
	if ok := n.config.CreatedSlaveLink; ok {
		// if the interface exists, only delete if it matches iface.vlan or dummy.net_id naming
		if ok := parentExists(n.config.Parent); ok {
			// only delete the link if it is named the net_id
			if n.config.Parent == getDummyName(stringid.TruncateID(nid)) {
				err := delDummyLink(n.config.Parent)
				if err != nil {
					logrus.Debugf("link %s was not deleted, continuing the delete network operation: %v",
						n.config.Parent, err)
				}
			} else {
				// only delete the link if it matches iface.vlan naming
				err := delVlanLink(n.config.Parent)
				if err != nil {
					logrus.Debugf("link %s was not deleted, continuing the delete network operation: %v",
						n.config.Parent, err)
				}
			}
		}
	}
	for _, ep := range n.endpoints {
		if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
			if err := ns.NlHandle().LinkDel(link); err != nil {
				logrus.WithError(err).Warnf("Failed to delete interface (%s)'s link on endpoint (%s) delete", ep.srcName, ep.id)
			}
		}

		if err := d.storeDelete(ep); err != nil {
			logrus.Warnf("Failed to remove ipvlan endpoint %.7s from store: %v", ep.id, err)
		}
	}
	// delete the *network
	d.deleteNetwork(nid)
	// delete the network record from persistent cache
	err := d.storeDelete(n.config)
	if err != nil {
		return fmt.Errorf("error deleting deleting id %s from datastore: %v", nid, err)
	}
	return nil
}

// parseNetworkOptions parse docker network options
func parseNetworkOptions(id string, option options.Generic) (*configuration, error) {
	var (
		err    error
		config = &configuration{}
	)
	// parse generic labels first
	if genData, ok := option[netlabel.GenericData]; ok && genData != nil {
		if config, err = parseNetworkGenericOptions(genData); err != nil {
			return nil, err
		}
	}
	if val, ok := option[netlabel.Internal]; ok {
		if internal, ok := val.(bool); ok && internal {
			config.Internal = true
		}
	}
	config.ID = id
	return config, nil
}

// parseNetworkGenericOptions parse generic driver docker network options
func parseNetworkGenericOptions(data interface{}) (*configuration, error) {
	var (
		err    error
		config *configuration
	)
	switch opt := data.(type) {
	case *configuration:
		config = opt
	case map[string]string:
		config = &configuration{}
		err = config.fromOptions(opt)
	case options.Generic:
		var opaqueConfig interface{}
		if opaqueConfig, err = options.GenerateFromModel(opt, config); err == nil {
			config = opaqueConfig.(*configuration)
		}
	default:
		err = types.BadRequestErrorf("unrecognized network configuration format: %v", opt)
	}
	return config, err
}

// fromOptions binds the generic options to networkConfiguration to cache
func (config *configuration) fromOptions(labels map[string]string) error {
	for label, value := range labels {
		switch label {
		case parentOpt:
			// parse driver option '-o parent'
			config.Parent = value
		case driverModeOpt:
			// parse driver option '-o ipvlan_mode'
			config.IpvlanMode = value
		case driverFlagOpt:
			// parse driver option '-o ipvlan_flag'
			config.IpvlanFlag = value
		}
	}
	return nil
}

// processIPAM parses v4 and v6 IP information and binds it to the network configuration
func (config *configuration) processIPAM(ipamV4Data, ipamV6Data []driverapi.IPAMData) {
	for _, ipd := range ipamV4Data {
		config.Ipv4Subnets = append(config.Ipv4Subnets, &ipSubnet{
			SubnetIP: ipd.Pool.String(),
			GwIP:     ipd.Gateway.String(),
		})
	}
	for _, ipd := range ipamV6Data {
		config.Ipv6Subnets = append(config.Ipv6Subnets, &ipSubnet{
			SubnetIP: ipd.Pool.String(),
			GwIP:     ipd.Gateway.String(),
		})
	}
}
<|endoftext|>


<commit_before>package server

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"context"
	"fmt"
	"github.com/mholt/archiver/v3"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
)

// Look through a given archive and determine if decompressing it would put the server over
// its allocated disk space limit.
func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (bool, error) {
	// Don't waste time trying to determine this if we know the server will have the space for
	// it since there is no limit.
	if fs.Server.Build.DiskSpace <= 0 {
		return true, nil
	}

	source, err := fs.SafePath(filepath.Join(dir, file))
	if err != nil {
		return false, err
	}

	wg := new(sync.WaitGroup)

	var dirSize int64
	var cErr error
	// Get the cached size in a parallel process so that if it is not cached we are not
	// waiting an unnecessary amount of time on this call.
	go func() {
		wg.Add(1)
		defer wg.Done()

		dirSize, cErr = fs.getCachedDiskUsage()
	}()

	var size int64
	// In a seperate thread, walk over the archive and figure out just how large the final
	// output would be from dearchiving it.
	go func() {
		wg.Add(1)
		defer wg.Done()

		// Walk all of the files and calculate the total decompressed size of this archive.
		archiver.Walk(source, func(f archiver.File) error {
			atomic.AddInt64(&size, f.Size())

			return nil
		})
	}()

	wg.Wait()

	return ((dirSize + size) / 1000.0 / 1000.0) <= fs.Server.Build.DiskSpace, cErr
}

// Decompress a file in a given directory by using the archiver tool to infer the file
// type and go from there. This will walk over all of the files within the given archive
// and ensure that there is not a zip-slip attack being attempted by validating that the
// final path is within the server data directory.
func (fs *Filesystem) DecompressFile(dir string, file string) error {
	source, err := fs.SafePath(filepath.Join(dir, file))
	if err != nil {
		return errors.WithStack(err)
	}

	g, ctx := errgroup.WithContext(context.Background())

	// Walk over all of the files spinning up an additional go-routine for each file we've encountered
	// and then extract that file from the archive and write it to the disk. If any part of this process
	// encounters an error the entire process will be stopped.
	archiver.Walk(source, func(f archiver.File) error {
		// Don't waste time with directories, we don't need to create them if they have no contents, and
		// we will ensure the directory exists when opening the file for writing anyways.
		if f.IsDir() {
			return nil
		}

		fi := f
		g.Go(func() error {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
				return fs.extractFileFromArchive(fi)
			}
		})

		return nil
	})

	// Wait until everything is done, and then return.
	if err := g.Wait(); err != nil {
		return errors.WithStack(err)
	}

	return nil
}

// Extracts a single file from the archive and writes it to the disk after verifying that it will end
// up in the server data directory.
func (fs *Filesystem) extractFileFromArchive(f archiver.File) error {
	var name string

	switch s := f.Sys().(type) {
	case *tar.Header:
		name = s.Name
	case *gzip.Header:
		name = s.Name
	case *zip.FileHeader:
		name = s.Name
	default:
		return errors.New(fmt.Sprintf("could not parse underlying data source with type %s", reflect.TypeOf(s).String()))
	}

	// Guard against a zip-slip attack and prevent writing a file to a destination outside of
	// the server root directory.
	p, err := fs.SafePath(name)
	if err != nil {
		return err
	}

	// Ensure the directory structure for this file exists before trying to write the file
	// to the disk, otherwise we'll have some unexpected fun.
	if err := os.MkdirAll(strings.TrimSuffix(p, filepath.Base(p)), 0755); err != nil {
		return err
	}

	// Open the file and truncate it if it already exists.
	o, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}

	defer o.Close()

	_, cerr := io.Copy(o, f)

	return cerr
}
<commit_msg>Don't cause a race condition<commit_after>package server

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"fmt"
	"github.com/mholt/archiver/v3"
	"github.com/pkg/errors"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
)

// Look through a given archive and determine if decompressing it would put the server over
// its allocated disk space limit.
func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (bool, error) {
	// Don't waste time trying to determine this if we know the server will have the space for
	// it since there is no limit.
	if fs.Server.Build.DiskSpace <= 0 {
		return true, nil
	}

	source, err := fs.SafePath(filepath.Join(dir, file))
	if err != nil {
		return false, err
	}

	wg := new(sync.WaitGroup)

	var dirSize int64
	var cErr error
	// Get the cached size in a parallel process so that if it is not cached we are not
	// waiting an unnecessary amount of time on this call.
	go func() {
		wg.Add(1)
		defer wg.Done()

		dirSize, cErr = fs.getCachedDiskUsage()
	}()

	var size int64
	// In a seperate thread, walk over the archive and figure out just how large the final
	// output would be from dearchiving it.
	go func() {
		wg.Add(1)
		defer wg.Done()

		// Walk all of the files and calculate the total decompressed size of this archive.
		archiver.Walk(source, func(f archiver.File) error {
			atomic.AddInt64(&size, f.Size())

			return nil
		})
	}()

	wg.Wait()

	return ((dirSize + size) / 1000.0 / 1000.0) <= fs.Server.Build.DiskSpace, cErr
}

// Decompress a file in a given directory by using the archiver tool to infer the file
// type and go from there. This will walk over all of the files within the given archive
// and ensure that there is not a zip-slip attack being attempted by validating that the
// final path is within the server data directory.
func (fs *Filesystem) DecompressFile(dir string, file string) error {
	source, err := fs.SafePath(filepath.Join(dir, file))
	if err != nil {
		return errors.WithStack(err)
	}

	// Walk over all of the files spinning up an additional go-routine for each file we've encountered
	// and then extract that file from the archive and write it to the disk. If any part of this process
	// encounters an error the entire process will be stopped.
	return archiver.Walk(source, func(f archiver.File) error {
		// Don't waste time with directories, we don't need to create them if they have no contents, and
		// we will ensure the directory exists when opening the file for writing anyways.
		if f.IsDir() {
			return nil
		}

		return fs.extractFileFromArchive(f)
	})
}

// Extracts a single file from the archive and writes it to the disk after verifying that it will end
// up in the server data directory.
func (fs *Filesystem) extractFileFromArchive(f archiver.File) error {
	var name string

	switch s := f.Sys().(type) {
	case *tar.Header:
		name = s.Name
	case *gzip.Header:
		name = s.Name
	case *zip.FileHeader:
		name = s.Name
	default:
		return errors.New(fmt.Sprintf("could not parse underlying data source with type %s", reflect.TypeOf(s).String()))
	}

	// Guard against a zip-slip attack and prevent writing a file to a destination outside of
	// the server root directory.
	p, err := fs.SafePath(name)
	if err != nil {
		return err
	}

	// Ensure the directory structure for this file exists before trying to write the file
	// to the disk, otherwise we'll have some unexpected fun.
	if err := os.MkdirAll(strings.TrimSuffix(p, filepath.Base(p)), 0755); err != nil {
		return err
	}

	// Open the file and truncate it if it already exists.
	o, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}

	defer o.Close()

	_, cerr := io.Copy(o, f)

	return cerr
}
<|endoftext|>


<commit_before>package tasks

import (
	"database/sql"
	"time"

	"github.com/go-errors/errors"
	_ "github.com/jackc/pgx/stdlib"
	"github.com/privacybydesign/irmago/server/keyshare"
)

type taskHandler struct {
	conf *Configuration
	db   keyshare.DB
}

func newHandler(conf *Configuration) (*taskHandler, error) {
	err := processConfiguration(conf)
	if err != nil {
		return nil, err
	}
	db, err := sql.Open("pgx", conf.DBConnStr)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, errors.Errorf("failed to connect to database: %v", err)
	}

	task := &taskHandler{db: keyshare.DB{DB: db}, conf: conf}
	return task, nil
}

func Do(conf *Configuration) error {
	task, err := newHandler(conf)
	if err != nil {
		return err
	}

	task.cleanupEmails()
	task.cleanupTokens()
	task.cleanupAccounts()
	task.expireAccounts()

	return nil
}

// Remove email addresses marked for deletion long enough ago
func (t *taskHandler) cleanupEmails() {
	_, err := t.db.Exec("DELETE FROM irma.emails WHERE delete_on < $1", time.Now().Unix())
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove email addresses marked for deletion")
	}
}

// Remove old login and email verifciation tokens
func (t *taskHandler) cleanupTokens() {
	_, err := t.db.Exec("DELETE FROM irma.email_login_tokens WHERE expiry < $1", time.Now().Unix())
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove email login tokens that have expired")
		return
	}
	_, err = t.db.Exec("DELETE FROM irma.email_verification_tokens WHERE expiry < $1", time.Now().Unix())
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove email verification tokens that have expired")
	}
}

// Cleanup accounts disabled long enough ago.
func (t *taskHandler) cleanupAccounts() {
	_, err := t.db.Exec("DELETE FROM irma.users WHERE delete_on < $1 AND (coredata IS NULL OR last_seen < delete_on - $2)",
		time.Now().Unix(),
		t.conf.DeleteDelay*24*60*60)
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove accounts scheduled for deletion")
	}
}

func (t *taskHandler) sendExpiryEmails(id int64, username, lang string) error {
	// Fetch user's email addresses
	err := t.db.QueryIterate("SELECT email FROM irma.emails WHERE user_id = $1",
		func(emailRes *sql.Rows) error {
			var email string
			err := emailRes.Scan(&email)
			if err != nil {
				return err
			}

			// And send
			err = t.conf.SendEmail(
				t.conf.deleteExpiredAccountTemplate,
				t.conf.DeleteExpiredAccountSubjects,
				map[string]string{"Username": username, "Email": email},
				[]string{email},
				lang,
			)
			return err
		},
		id,
	)
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not retrieve user's email addresses")
		return err
	}
	return nil
}

// Mark old unused accounts for deletion, and inform their owners.
func (t *taskHandler) expireAccounts() {
	// Disable this task when email server is not given
	if t.conf.EmailServer == "" {
		t.conf.Logger.Warning("Expiring accounts is disabled, as no email server is configured")
		return
	}

	// Iterate over users we havent seen in ExpiryDelay days, and which have a registered email.
	// We ignore (and thus keep alive) accounts without email addresses, as we can't inform their owners.
	// (Note that for such accounts we store no email addresses, i.e. no personal data whatsoever.)
	err := t.db.QueryIterate(`
		SELECT id, username, language
		FROM irma.users
		WHERE last_seen < $1 AND (
			SELECT count(*)
			FROM irma.emails
			WHERE irma.users.id = irma.emails.user_id
		) > 0
		LIMIT 10`,
		func(res *sql.Rows) error {
			var id int64
			var username string
			var lang string
			err := res.Scan(&id, &username, &lang)
			if err != nil {
				return err
			}

			// Send emails
			err = t.sendExpiryEmails(id, username, lang)
			if err != nil {
				return err // already logged, just abort
			}

			// Finally, do marking for deletion
			err = t.db.ExecUser("UPDATE irma.users SET delete_on = $2 WHERE id = $1", id,
				time.Now().Add(time.Duration(24*t.conf.DeleteDelay)*time.Hour).Unix())
			if err != nil {
				return err
			}
			return nil
		},
		time.Now().Add(time.Duration(-24*t.conf.ExpiryDelay)*time.Hour).Unix(),
	)
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not query for accounts that have expired")
		return
	}
}
<commit_msg>docs: fix spelling mistake in comment<commit_after>package tasks

import (
	"database/sql"
	"time"

	"github.com/go-errors/errors"
	_ "github.com/jackc/pgx/stdlib"
	"github.com/privacybydesign/irmago/server/keyshare"
)

type taskHandler struct {
	conf *Configuration
	db   keyshare.DB
}

func newHandler(conf *Configuration) (*taskHandler, error) {
	err := processConfiguration(conf)
	if err != nil {
		return nil, err
	}
	db, err := sql.Open("pgx", conf.DBConnStr)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, errors.Errorf("failed to connect to database: %v", err)
	}

	task := &taskHandler{db: keyshare.DB{DB: db}, conf: conf}
	return task, nil
}

func Do(conf *Configuration) error {
	task, err := newHandler(conf)
	if err != nil {
		return err
	}

	task.cleanupEmails()
	task.cleanupTokens()
	task.cleanupAccounts()
	task.expireAccounts()

	return nil
}

// Remove email addresses marked for deletion long enough ago
func (t *taskHandler) cleanupEmails() {
	_, err := t.db.Exec("DELETE FROM irma.emails WHERE delete_on < $1", time.Now().Unix())
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove email addresses marked for deletion")
	}
}

// Remove old login and email verification tokens
func (t *taskHandler) cleanupTokens() {
	_, err := t.db.Exec("DELETE FROM irma.email_login_tokens WHERE expiry < $1", time.Now().Unix())
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove email login tokens that have expired")
		return
	}
	_, err = t.db.Exec("DELETE FROM irma.email_verification_tokens WHERE expiry < $1", time.Now().Unix())
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove email verification tokens that have expired")
	}
}

// Cleanup accounts disabled long enough ago.
func (t *taskHandler) cleanupAccounts() {
	_, err := t.db.Exec("DELETE FROM irma.users WHERE delete_on < $1 AND (coredata IS NULL OR last_seen < delete_on - $2)",
		time.Now().Unix(),
		t.conf.DeleteDelay*24*60*60)
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not remove accounts scheduled for deletion")
	}
}

func (t *taskHandler) sendExpiryEmails(id int64, username, lang string) error {
	// Fetch user's email addresses
	err := t.db.QueryIterate("SELECT email FROM irma.emails WHERE user_id = $1",
		func(emailRes *sql.Rows) error {
			var email string
			err := emailRes.Scan(&email)
			if err != nil {
				return err
			}

			// And send
			err = t.conf.SendEmail(
				t.conf.deleteExpiredAccountTemplate,
				t.conf.DeleteExpiredAccountSubjects,
				map[string]string{"Username": username, "Email": email},
				[]string{email},
				lang,
			)
			return err
		},
		id,
	)
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not retrieve user's email addresses")
		return err
	}
	return nil
}

// Mark old unused accounts for deletion, and inform their owners.
func (t *taskHandler) expireAccounts() {
	// Disable this task when email server is not given
	if t.conf.EmailServer == "" {
		t.conf.Logger.Warning("Expiring accounts is disabled, as no email server is configured")
		return
	}

	// Iterate over users we havent seen in ExpiryDelay days, and which have a registered email.
	// We ignore (and thus keep alive) accounts without email addresses, as we can't inform their owners.
	// (Note that for such accounts we store no email addresses, i.e. no personal data whatsoever.)
	err := t.db.QueryIterate(`
		SELECT id, username, language
		FROM irma.users
		WHERE last_seen < $1 AND (
			SELECT count(*)
			FROM irma.emails
			WHERE irma.users.id = irma.emails.user_id
		) > 0
		LIMIT 10`,
		func(res *sql.Rows) error {
			var id int64
			var username string
			var lang string
			err := res.Scan(&id, &username, &lang)
			if err != nil {
				return err
			}

			// Send emails
			err = t.sendExpiryEmails(id, username, lang)
			if err != nil {
				return err // already logged, just abort
			}

			// Finally, do marking for deletion
			err = t.db.ExecUser("UPDATE irma.users SET delete_on = $2 WHERE id = $1", id,
				time.Now().Add(time.Duration(24*t.conf.DeleteDelay)*time.Hour).Unix())
			if err != nil {
				return err
			}
			return nil
		},
		time.Now().Add(time.Duration(-24*t.conf.ExpiryDelay)*time.Hour).Unix(),
	)
	if err != nil {
		t.conf.Logger.WithField("error", err).Error("Could not query for accounts that have expired")
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package maphash

import (
	"hash"
	"testing"
)

func TestUnseededHash(t *testing.T) {
	m := map[uint64]struct{}{}
	for i := 0; i < 1000; i++ {
		h := new(Hash)
		m[h.Sum64()] = struct{}{}
	}
	if len(m) < 900 {
		t.Errorf("empty hash not sufficiently random: got %d, want 1000", len(m))
	}
}

func TestSeededHash(t *testing.T) {
	s := MakeSeed()
	m := map[uint64]struct{}{}
	for i := 0; i < 1000; i++ {
		h := new(Hash)
		h.SetSeed(s)
		m[h.Sum64()] = struct{}{}
	}
	if len(m) != 1 {
		t.Errorf("seeded hash is random: got %d, want 1", len(m))
	}
}

func TestHashGrouping(t *testing.T) {
	b := []byte("foo")
	h1 := new(Hash)
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h1.Write(b)
	for _, x := range b {
		err := h2.WriteByte(x)
		if err != nil {
			t.Fatalf("WriteByte: %v", err)
		}
	}
	if h1.Sum64() != h2.Sum64() {
		t.Errorf("hash of \"foo\" and \"f\",\"o\",\"o\" not identical")
	}
}

func TestHashBytesVsString(t *testing.T) {
	s := "foo"
	b := []byte(s)
	h1 := new(Hash)
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	n1, err1 := h1.WriteString(s)
	if n1 != len(s) || err1 != nil {
		t.Fatalf("WriteString(s) = %d, %v, want %d, nil", n1, err1, len(s))
	}
	n2, err2 := h2.Write(b)
	if n2 != len(b) || err2 != nil {
		t.Fatalf("Write(b) = %d, %v, want %d, nil", n2, err2, len(b))
	}
	if h1.Sum64() != h2.Sum64() {
		t.Errorf("hash of string and bytes not identical")
	}
}

func TestHashHighBytes(t *testing.T) {
	// See issue 34925.
	const N = 10
	m := map[uint64]struct{}{}
	for i := 0; i < N; i++ {
		h := new(Hash)
		h.WriteString("foo")
		m[h.Sum64()>>32] = struct{}{}
	}
	if len(m) < N/2 {
		t.Errorf("from %d seeds, wanted at least %d different hashes; got %d", N, N/2, len(m))
	}
}

func TestRepeat(t *testing.T) {
	h1 := new(Hash)
	h1.WriteString("testing")
	sum1 := h1.Sum64()

	h1.Reset()
	h1.WriteString("testing")
	sum2 := h1.Sum64()

	if sum1 != sum2 {
		t.Errorf("different sum after reseting: %#x != %#x", sum1, sum2)
	}

	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h2.WriteString("testing")
	sum3 := h2.Sum64()

	if sum1 != sum3 {
		t.Errorf("different sum on the same seed: %#x != %#x", sum1, sum3)
	}
}

// Make sure a Hash implements the hash.Hash and hash.Hash64 interfaces.
var _ hash.Hash = &Hash{}
var _ hash.Hash64 = &Hash{}
<commit_msg>hash/maphash: add more tests for seed generation<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package maphash

import (
	"hash"
	"testing"
)

func TestUnseededHash(t *testing.T) {
	m := map[uint64]struct{}{}
	for i := 0; i < 1000; i++ {
		h := new(Hash)
		m[h.Sum64()] = struct{}{}
	}
	if len(m) < 900 {
		t.Errorf("empty hash not sufficiently random: got %d, want 1000", len(m))
	}
}

func TestSeededHash(t *testing.T) {
	s := MakeSeed()
	m := map[uint64]struct{}{}
	for i := 0; i < 1000; i++ {
		h := new(Hash)
		h.SetSeed(s)
		m[h.Sum64()] = struct{}{}
	}
	if len(m) != 1 {
		t.Errorf("seeded hash is random: got %d, want 1", len(m))
	}
}

func TestHashGrouping(t *testing.T) {
	b := []byte("foo")
	h1 := new(Hash)
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h1.Write(b)
	for _, x := range b {
		err := h2.WriteByte(x)
		if err != nil {
			t.Fatalf("WriteByte: %v", err)
		}
	}
	if h1.Sum64() != h2.Sum64() {
		t.Errorf("hash of \"foo\" and \"f\",\"o\",\"o\" not identical")
	}
}

func TestHashBytesVsString(t *testing.T) {
	s := "foo"
	b := []byte(s)
	h1 := new(Hash)
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	n1, err1 := h1.WriteString(s)
	if n1 != len(s) || err1 != nil {
		t.Fatalf("WriteString(s) = %d, %v, want %d, nil", n1, err1, len(s))
	}
	n2, err2 := h2.Write(b)
	if n2 != len(b) || err2 != nil {
		t.Fatalf("Write(b) = %d, %v, want %d, nil", n2, err2, len(b))
	}
	if h1.Sum64() != h2.Sum64() {
		t.Errorf("hash of string and bytes not identical")
	}
}

func TestHashHighBytes(t *testing.T) {
	// See issue 34925.
	const N = 10
	m := map[uint64]struct{}{}
	for i := 0; i < N; i++ {
		h := new(Hash)
		h.WriteString("foo")
		m[h.Sum64()>>32] = struct{}{}
	}
	if len(m) < N/2 {
		t.Errorf("from %d seeds, wanted at least %d different hashes; got %d", N, N/2, len(m))
	}
}

func TestRepeat(t *testing.T) {
	h1 := new(Hash)
	h1.WriteString("testing")
	sum1 := h1.Sum64()

	h1.Reset()
	h1.WriteString("testing")
	sum2 := h1.Sum64()

	if sum1 != sum2 {
		t.Errorf("different sum after reseting: %#x != %#x", sum1, sum2)
	}

	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h2.WriteString("testing")
	sum3 := h2.Sum64()

	if sum1 != sum3 {
		t.Errorf("different sum on the same seed: %#x != %#x", sum1, sum3)
	}
}

func TestSeedFromSum64(t *testing.T) {
	h1 := new(Hash)
	h1.WriteString("foo")
	x := h1.Sum64() // seed generated here
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h2.WriteString("foo")
	y := h2.Sum64()
	if x != y {
		t.Errorf("hashes don't match: want %x, got %x", x, y)
	}
}

func TestSeedFromSeed(t *testing.T) {
	h1 := new(Hash)
	h1.WriteString("foo")
	_ = h1.Seed() // seed generated here
	x := h1.Sum64()
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h2.WriteString("foo")
	y := h2.Sum64()
	if x != y {
		t.Errorf("hashes don't match: want %x, got %x", x, y)
	}
}

func TestSeedFromFlush(t *testing.T) {
	b := make([]byte, 65)
	h1 := new(Hash)
	h1.Write(b) // seed generated here
	x := h1.Sum64()
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h2.Write(b)
	y := h2.Sum64()
	if x != y {
		t.Errorf("hashes don't match: want %x, got %x", x, y)
	}
}

func TestSeedFromReset(t *testing.T) {
	h1 := new(Hash)
	h1.WriteString("foo")
	h1.Reset() // seed generated here
	h1.WriteString("foo")
	x := h1.Sum64()
	h2 := new(Hash)
	h2.SetSeed(h1.Seed())
	h2.WriteString("foo")
	y := h2.Sum64()
	if x != y {
		t.Errorf("hashes don't match: want %x, got %x", x, y)
	}
}

// Make sure a Hash implements the hash.Hash and hash.Hash64 interfaces.
var _ hash.Hash = &Hash{}
var _ hash.Hash64 = &Hash{}
<|endoftext|>


<commit_before>package blenc

import (
	"crypto/aes"
	"crypto/cipher"
	"errors"
	"io"

	"github.com/Yawning/chacha20"
	base58 "github.com/jbenet/go-base58"
)

// ErrInvalidKey is used as an error when given string representation if the
// key can not be interpreted
var ErrInvalidKey = errors.New("Invalid encryption key")

const (
	keyTypeAES      = 0x01
	keyTypeChaCha20 = 0x02

	keyTypeDefault = keyTypeChaCha20
)

const (
	maxKeyDataSizeUsed = 32
	keyTypeAESKeySize  = 24
)

type streamReader struct {
	cipher.StreamReader
}

func (s *streamReader) Close() error {
	if closer, ok := s.R.(io.Closer); ok {
		return closer.Close()
	}
	return nil
}

func streamCipherReaderForKey(key string, r io.ReadCloser) (io.ReadCloser, error) {

	keyData := base58.Decode(key)
	if len(keyData) == 0 {
		return nil, ErrInvalidKey
	}

	return streamCipherReaderForKeyData(keyData[0], keyData[1:], r)
}

func keyFromKeyData(keyType byte, keyData []byte) string {

	keyBytes := make([]byte, 1, maxKeyDataSizeUsed+1)
	keyBytes[0] = keyType

	switch keyType {
	case keyTypeAES:
		keyBytes = append(keyBytes, keyData[:keyTypeAESKeySize]...)

	case keyTypeChaCha20:
		keyBytes = append(keyBytes, keyData[:chacha20.KeySize]...)

	}

	return base58.Encode(keyBytes)
}

func streamCipherReaderForKeyData(keyType byte, keyData []byte, r io.ReadCloser) (io.ReadCloser, error) {

	switch keyType {
	case keyTypeAES:
		block, err := aes.NewCipher(keyData[:keyTypeAESKeySize])
		if err != nil {
			return nil, err
		}
		var iv [aes.BlockSize]byte
		stream := cipher.NewCTR(block, iv[:])
		return &streamReader{cipher.StreamReader{S: stream, R: r}}, nil

	case keyTypeChaCha20:
		var nonce [chacha20.NonceSize]byte
		stream, err := chacha20.NewCipher(keyData[:chacha20.KeySize], nonce[:])
		if err != nil {
			return nil, err
		}
		return &streamReader{cipher.StreamReader{S: stream, R: r}}, nil

	default:
		return nil, ErrInvalidKey
	}
}
<commit_msg>Fix import name<commit_after>package blenc

import (
	"crypto/aes"
	"crypto/cipher"
	"errors"
	"io"

	base58 "github.com/jbenet/go-base58"
	"github.com/yawning/chacha20"
)

// ErrInvalidKey is used as an error when given string representation if the
// key can not be interpreted
var ErrInvalidKey = errors.New("Invalid encryption key")

const (
	keyTypeAES      = 0x01
	keyTypeChaCha20 = 0x02

	keyTypeDefault = keyTypeChaCha20
)

const (
	maxKeyDataSizeUsed = 32
	keyTypeAESKeySize  = 24
)

type streamReader struct {
	cipher.StreamReader
}

func (s *streamReader) Close() error {
	if closer, ok := s.R.(io.Closer); ok {
		return closer.Close()
	}
	return nil
}

func streamCipherReaderForKey(key string, r io.ReadCloser) (io.ReadCloser, error) {

	keyData := base58.Decode(key)
	if len(keyData) == 0 {
		return nil, ErrInvalidKey
	}

	return streamCipherReaderForKeyData(keyData[0], keyData[1:], r)
}

func keyFromKeyData(keyType byte, keyData []byte) string {

	keyBytes := make([]byte, 1, maxKeyDataSizeUsed+1)
	keyBytes[0] = keyType

	switch keyType {
	case keyTypeAES:
		keyBytes = append(keyBytes, keyData[:keyTypeAESKeySize]...)

	case keyTypeChaCha20:
		keyBytes = append(keyBytes, keyData[:chacha20.KeySize]...)

	}

	return base58.Encode(keyBytes)
}

func streamCipherReaderForKeyData(keyType byte, keyData []byte, r io.ReadCloser) (io.ReadCloser, error) {

	switch keyType {
	case keyTypeAES:
		block, err := aes.NewCipher(keyData[:keyTypeAESKeySize])
		if err != nil {
			return nil, err
		}
		var iv [aes.BlockSize]byte
		stream := cipher.NewCTR(block, iv[:])
		return &streamReader{cipher.StreamReader{S: stream, R: r}}, nil

	case keyTypeChaCha20:
		var nonce [chacha20.NonceSize]byte
		stream, err := chacha20.NewCipher(keyData[:chacha20.KeySize], nonce[:])
		if err != nil {
			return nil, err
		}
		return &streamReader{cipher.StreamReader{S: stream, R: r}}, nil

	default:
		return nil, ErrInvalidKey
	}
}
<|endoftext|>