<commit_before>package stager

import (
	"errors"
	"fmt"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry-incubator/runtime-schema/router"
	"github.com/cloudfoundry/gunk/urljoiner"
	"strings"
	"time"
)

type Stager interface {
	Stage(models.StagingRequestFromCC, string) error
}

type stager struct {
	stagerBBS bbs.StagerBBS
	compilers map[string]string
}

func NewStager(stagerBBS bbs.StagerBBS, compilers map[string]string) Stager {
	return &stager{
		stagerBBS: stagerBBS,
		compilers: compilers,
	}
}

func (stager *stager) Stage(request models.StagingRequestFromCC, replyTo string) error {
	fileServerURL, err := stager.stagerBBS.GetAvailableFileServer()
	if err != nil {
		return errors.New("No available file server present")
	}

	compilerURL, err := stager.compilerDownloadURL(request, fileServerURL)
	if err != nil {
		return err
	}

	buildpacksOrder := []string{}
	for _, buildpack := range request.AdminBuildpacks {
		buildpacksOrder = append(buildpacksOrder, buildpack.Key)
	}

	smeltingConfig := models.NewLinuxSmeltingConfig(buildpacksOrder)

	actions := []models.ExecutorAction{}

	actions = append(actions, models.ExecutorAction{
		models.DownloadAction{
			From:    compilerURL,
			To:      smeltingConfig.CompilerPath(),
			Extract: true,
		},
	})

	actions = append(actions, models.ExecutorAction{
		models.DownloadAction{
			From:    request.DownloadUri,
			To:      smeltingConfig.AppPath(),
			Extract: true,
		},
	})

	for _, buildpack := range request.AdminBuildpacks {
		actions = append(actions, models.ExecutorAction{
			models.DownloadAction{
				From:    buildpack.Url,
				To:      smeltingConfig.BuildpackPath(buildpack.Key),
				Extract: true,
			},
		})
	}

	actions = append(actions, models.ExecutorAction{
		models.RunAction{
			Script:  smeltingConfig.Script(),
			Env:     request.Environment,
			Timeout: 15 * time.Minute,
		},
	})

	uploadURL, err := stager.dropletUploadURL(request, fileServerURL)
	if err != nil {
		return err
	}

	actions = append(actions, models.ExecutorAction{
		models.UploadAction{
			From: smeltingConfig.DropletArchivePath(),
			To:   uploadURL,
		},
	})

	actions = append(actions, models.ExecutorAction{
		models.FetchResultAction{
			File: smeltingConfig.ResultJsonPath(),
		},
	})

	err = stager.stagerBBS.DesireRunOnce(&models.RunOnce{
		Guid:            strings.Join([]string{request.AppId, request.TaskId}, "-"),
		Stack:           request.Stack,
		ReplyTo:         replyTo,
		FileDescriptors: request.FileDescriptors,
		MemoryMB:        request.MemoryMB,
		DiskMB:          request.DiskMB,
		Actions:         actions,
		Log: models.LogConfig{
			Guid:       request.AppId,
			SourceName: "STG",
		},
	})

	return err
}

func (stager *stager) compilerDownloadURL(request models.StagingRequestFromCC, fileServerURL string) (string, error) {
	compilerPath, ok := stager.compilers[request.Stack]
	if !ok {
		return "", errors.New("No compiler defined for requested stack")
	}

	staticRoute, ok := router.NewFileServerRoutes().RouteForHandler(router.FS_STATIC)
	if !ok {
		return "", errors.New("Couldn't generate the compiler download path")
	}

	return urljoiner.Join(fileServerURL, staticRoute.Path, compilerPath), nil
}

func (stager *stager) dropletUploadURL(request models.StagingRequestFromCC, fileServerURL string) (string, error) {
	staticRoute, ok := router.NewFileServerRoutes().RouteForHandler(router.FS_UPLOAD_DROPLET)
	if !ok {
		return "", errors.New("Couldn't generate the compiler download path")
	}

	path, err := staticRoute.PathWithParams(map[string]string{
		"guid": request.AppId,
	})

	if err != nil {
		return "", fmt.Errorf("Failed to build droplet upload URL: %s", err)
	}

	return urljoiner.Join(fileServerURL, path), nil
}
<commit_msg>AppPath -> AppDir<commit_after>package stager

import (
	"errors"
	"fmt"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry-incubator/runtime-schema/router"
	"github.com/cloudfoundry/gunk/urljoiner"
	"strings"
	"time"
)

type Stager interface {
	Stage(models.StagingRequestFromCC, string) error
}

type stager struct {
	stagerBBS bbs.StagerBBS
	compilers map[string]string
}

func NewStager(stagerBBS bbs.StagerBBS, compilers map[string]string) Stager {
	return &stager{
		stagerBBS: stagerBBS,
		compilers: compilers,
	}
}

func (stager *stager) Stage(request models.StagingRequestFromCC, replyTo string) error {
	fileServerURL, err := stager.stagerBBS.GetAvailableFileServer()
	if err != nil {
		return errors.New("No available file server present")
	}

	compilerURL, err := stager.compilerDownloadURL(request, fileServerURL)
	if err != nil {
		return err
	}

	buildpacksOrder := []string{}
	for _, buildpack := range request.AdminBuildpacks {
		buildpacksOrder = append(buildpacksOrder, buildpack.Key)
	}

	smeltingConfig := models.NewLinuxSmeltingConfig(buildpacksOrder)

	actions := []models.ExecutorAction{}

	actions = append(actions, models.ExecutorAction{
		models.DownloadAction{
			From:    compilerURL,
			To:      smeltingConfig.CompilerPath(),
			Extract: true,
		},
	})

	actions = append(actions, models.ExecutorAction{
		models.DownloadAction{
			From:    request.DownloadUri,
			To:      smeltingConfig.AppDir(),
			Extract: true,
		},
	})

	for _, buildpack := range request.AdminBuildpacks {
		actions = append(actions, models.ExecutorAction{
			models.DownloadAction{
				From:    buildpack.Url,
				To:      smeltingConfig.BuildpackPath(buildpack.Key),
				Extract: true,
			},
		})
	}

	actions = append(actions, models.ExecutorAction{
		models.RunAction{
			Script:  smeltingConfig.Script(),
			Env:     request.Environment,
			Timeout: 15 * time.Minute,
		},
	})

	uploadURL, err := stager.dropletUploadURL(request, fileServerURL)
	if err != nil {
		return err
	}

	actions = append(actions, models.ExecutorAction{
		models.UploadAction{
			From: smeltingConfig.DropletArchivePath(),
			To:   uploadURL,
		},
	})

	actions = append(actions, models.ExecutorAction{
		models.FetchResultAction{
			File: smeltingConfig.ResultJsonPath(),
		},
	})

	err = stager.stagerBBS.DesireRunOnce(&models.RunOnce{
		Guid:            strings.Join([]string{request.AppId, request.TaskId}, "-"),
		Stack:           request.Stack,
		ReplyTo:         replyTo,
		FileDescriptors: request.FileDescriptors,
		MemoryMB:        request.MemoryMB,
		DiskMB:          request.DiskMB,
		Actions:         actions,
		Log: models.LogConfig{
			Guid:       request.AppId,
			SourceName: "STG",
		},
	})

	return err
}

func (stager *stager) compilerDownloadURL(request models.StagingRequestFromCC, fileServerURL string) (string, error) {
	compilerPath, ok := stager.compilers[request.Stack]
	if !ok {
		return "", errors.New("No compiler defined for requested stack")
	}

	staticRoute, ok := router.NewFileServerRoutes().RouteForHandler(router.FS_STATIC)
	if !ok {
		return "", errors.New("Couldn't generate the compiler download path")
	}

	return urljoiner.Join(fileServerURL, staticRoute.Path, compilerPath), nil
}

func (stager *stager) dropletUploadURL(request models.StagingRequestFromCC, fileServerURL string) (string, error) {
	staticRoute, ok := router.NewFileServerRoutes().RouteForHandler(router.FS_UPLOAD_DROPLET)
	if !ok {
		return "", errors.New("Couldn't generate the compiler download path")
	}

	path, err := staticRoute.PathWithParams(map[string]string{
		"guid": request.AppId,
	})

	if err != nil {
		return "", fmt.Errorf("Failed to build droplet upload URL: %s", err)
	}

	return urljoiner.Join(fileServerURL, path), nil
}
<|endoftext|>


<commit_before>package service

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"

	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/log"
)

type BackupService interface {
	GetHost(host string) (dto.Host, error)
	ListHosts() ([]dto.Host, error)
	ListBackups(host dto.Host) ([]dto.Backup, error)
	ListOldBackups(host dto.Host) ([]dto.Backup, error)
	ListKeepBackups(host dto.Host) ([]dto.Backup, error)
	GetLatestBackup(host dto.Host) (dto.Backup, error)
	Cleanup(host dto.Host) error
	Resume(host dto.Host) error
}

type backupService struct {
	rootdir string
}

var logger = log.DefaultLogger

func NewBackupService(rootdir string) *backupService {
	s := new(backupService)
	s.rootdir = rootdir
	return s
}

func (s *backupService) Resume(host dto.Host) error {
	return nil
}

func (s *backupService) ListHosts() ([]dto.Host, error) {
	file, err := os.Open(s.rootdir)
	if err != nil {
		logger.Debugf("open rootdir %s failed: %v", s.rootdir, err)
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		logger.Debugf("file stat failed: %v", err)
		return nil, err
	}
	if !fileinfo.IsDir() {
		msg := fmt.Sprintf("rootdir %s is not a directory", s.rootdir)
		logger.Debug(msg)
		return nil, errors.New(msg)
	}
	names, err := file.Readdirnames(0)
	if err != nil {
		logger.Debugf("read dir names failed: %v", err)
		return nil, err
	}
	return s.createHosts(names)
}

func (s *backupService) createHosts(hosts []string) ([]dto.Host, error) {
	result := []dto.Host{}
	for _, host := range hosts {
		dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
		isDir, err := isDir(dir)
		if err != nil {
			logger.Debugf("is dir failed: %v", err)
			return nil, err
		}
		if isDir {
			result = append(result, createHost(host))
		} else {
			logger.Debugf("createHost for %s failed, is not a directory", host)
		}
	}
	return result, nil
}

func isDir(dir string) (bool, error) {
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir %s failed: %v", dir, err)
		return false, nil
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return fileinfo.IsDir(), nil
}

func createHost(host string) dto.Host {
	h := dto.NewHost()
	h.SetName(host)
	return h
}

func (s *backupService) ListBackups(host dto.Host) ([]dto.Backup, error) {
	if host == nil {
		return nil, errors.New("parameter host missing")
	}
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host.GetName())
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir failed: %v", err)
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		logger.Debugf("file stat failed: %v", err)
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	dirnames, err := file.Readdirnames(0)
	if err != nil {
		logger.Debugf("read dir names failed: %v", err)
		return nil, err
	}
	var names []string
	for _, name := range dirnames {
		if validBackupName(name) {
			names = append(names, name)
		}
	}
	backups := createBackups(names)
	return backups, nil
}

func validBackupName(name string) bool {
	re := regexp.MustCompile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
	return re.MatchString(name)
}

func createBackups(backups []string) []dto.Backup {
	result := make([]dto.Backup, len(backups))
	for i, backup := range backups {
		result[i] = createBackup(backup)
	}
	return result
}

func createBackup(backup string) dto.Backup {
	h := dto.NewBackup()
	h.SetName(backup)
	return h
}

func (s *backupService) GetHost(host string) (dto.Host, error) {
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	h := dto.NewHost()
	h.SetName(host)
	return h, nil
}

func (s *backupService) GetLatestBackup(host dto.Host) (dto.Backup, error) {
	list, err := s.ListBackups(host)
	if err != nil {
		logger.Debugf("list backups failed: %v", err)
		return nil, err
	}
	if len(list) == 0 {
		return nil, nil
	}
	var names []string
	backups := make(map[string]dto.Backup, 0)
	for _, backup := range list {
		backups[backup.GetName()] = backup
		names = append(names, backup.GetName())
	}
	sort.Strings(names)
	return backups[names[len(names)-1]], nil
}

func (s *backupService) ListOldBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	keep, err := getKeepBackups(backups)
	if err != nil {
		return nil, err
	}
	keepMap := make(map[string]bool)
	for _, b := range keep {
		keepMap[b.GetName()] = true
	}
	var result []dto.Backup
	for _, backup := range backups {
		if !keepMap[backup.GetName()] {
			result = append(result, backup)
		}
	}
	return result, nil
}

func getKeepBackups(backups []dto.Backup) ([]dto.Backup, error) {
	keep := make(map[string]dto.Backup, 0)
	now := time.Now()
	// keep all backups from today
	{
		b, err := getKeepToday(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per day if age <= 7 days
	{
		b, err := getKeepDay(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per week if age <= 28 days
	{
		b, err := getKeepWeek(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per month
	{
		b, err := getKeepMonth(backups)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep latest backup / current
	{
		backup := latestBackup(backups)
		if backup != nil {
			keep[backup.GetName()] = backup
		}
	}

	var result []dto.Backup
	for _, backup := range keep {
		result = append(result, backup)
	}
	return result, nil
}

func getKeepToday(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	var result []dto.Backup
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if isToday(t, now) {
			result = append(result, backup)
		}
	}
	return result, nil
}

func isToday(t time.Time, now time.Time) bool {
	return t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day()
}

func getKeepDay(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup

	var lastYear int = -1
	var lastMonth time.Month = -1
	var lastDay int = -1

	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if ageLessThanDays(t, now, 7) && (t.Year() != lastYear || t.Month() != lastMonth || t.Day() != lastDay) {
			result = append(result, backup)
			lastYear = t.Year()
			lastMonth = t.Month()
			lastDay = t.Day()
		}
	}
	return result, nil
}

func ageLessThanDays(t time.Time, now time.Time, days int64) bool {
	diff := now.Unix() - t.Unix()
	return diff <= 24*60*60*days
}

func getKeepMonth(backups []dto.Backup) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var lastYear int64 = -1
	var lastMonth int64 = -1
	var result []dto.Backup
	for _, backup := range backups {
		name := backup.GetName()
		year, err := strconv.ParseInt(name[0:4], 10, 64)
		if err != nil {
			return nil, err
		}
		month, err := strconv.ParseInt(name[5:7], 10, 64)
		if err != nil {
			return nil, err
		}
		logger.Tracef("year %d month %d", year, month)

		if year != lastYear || month != lastMonth {
			result = append(result, backup)
		}
		lastYear = year
		lastMonth = month
	}
	return result, nil
}

func (s *backupService) Cleanup(host dto.Host) error {
	if host == nil {
		return errors.New("parameter host missing")
	}
	backups, err := s.ListOldBackups(host)
	if err != nil {
		return err
	}
	logger.Debugf("found %d backup to delete for host %s", len(backups), host.GetName())
	for _, backup := range backups {
		dir := fmt.Sprintf("%s%c%s%c%s", s.rootdir, os.PathSeparator, host.GetName(), os.PathSeparator, backup.GetName())
		logger.Infof("delete %s started", dir)
		os.RemoveAll(dir)
		logger.Infof("delete %s finished", dir)
	}
	return nil
}

func (s *backupService) ListKeepBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	return getKeepBackups(backups)
}

func getTimeByName(backupName string) (time.Time, error) {
	return time.Parse("2006-01-02T15:04:05", backupName)
}

func getKeepWeek(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup
	var lastWeek int = -1
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		_, week := t.ISOWeek()
		if ageLessThanDays(t, now, 40) && week != lastWeek {
			result = append(result, backup)
			lastWeek = week
		}
	}
	return result, nil
}

func latestBackup(backups []dto.Backup) dto.Backup {
	if backups != nil && len(backups) > 0 {
		sort.Sort(util.BackupByDate(backups))
		return backups[len(backups)-1]
	}
	return nil
}
<commit_msg>refactor<commit_after>package service

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"

	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/log"
)

type BackupService interface {
	GetHost(host string) (dto.Host, error)
	ListHosts() ([]dto.Host, error)
	ListBackups(host dto.Host) ([]dto.Backup, error)
	ListOldBackups(host dto.Host) ([]dto.Backup, error)
	ListKeepBackups(host dto.Host) ([]dto.Backup, error)
	GetLatestBackup(host dto.Host) (dto.Backup, error)
	Cleanup(host dto.Host) error
	Resume(host dto.Host) error
}

type backupService struct {
	rootdir rootdir
}

type rootdir string

var logger = log.DefaultLogger

func NewBackupService(rootdirectory string) *backupService {
	s := new(backupService)
	s.rootdir = rootdir(rootdirectory)
	return s
}

func (s *backupService) Resume(host dto.Host) error {
	return nil
}

func (r *rootdir) names() ([]string, error ) {
	file, err := os.Open(string(*r))
	if err != nil {
		logger.Debugf("open rootdir %s failed: %v", r, err)
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		logger.Debugf("file stat failed: %v", err)
		return nil, err
	}
	if !fileinfo.IsDir() {
		msg := fmt.Sprintf("rootdir %s is not a directory", r)
		logger.Debug(msg)
		return nil, errors.New(msg)
	}
	names, err := file.Readdirnames(0)
	if err != nil {
		logger.Debugf("read dir names failed: %v", err)
		return nil, err
	}
	return names, nil
}

func (s *backupService) ListHosts() ([]dto.Host, error) {
	names, err := s.rootdir.names()
	if err != nil{
		return nil,err
	}
	return s.createHosts(names)
}

func (s *backupService) createHosts(hosts []string) ([]dto.Host, error) {
	result := []dto.Host{}
	for _, host := range hosts {
		dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
		isDir, err := isDir(dir)
		if err != nil {
			logger.Debugf("is dir failed: %v", err)
			return nil, err
		}
		if isDir {
			result = append(result, createHost(host))
		} else {
			logger.Debugf("createHost for %s failed, is not a directory", host)
		}
	}
	return result, nil
}

func isDir(dir string) (bool, error) {
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir %s failed: %v", dir, err)
		return false, nil
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return fileinfo.IsDir(), nil
}

func createHost(host string) dto.Host {
	h := dto.NewHost()
	h.SetName(host)
	return h
}

func (s *backupService) ListBackups(host dto.Host) ([]dto.Backup, error) {
	if host == nil {
		return nil, errors.New("parameter host missing")
	}
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host.GetName())
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir failed: %v", err)
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		logger.Debugf("file stat failed: %v", err)
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	dirnames, err := file.Readdirnames(0)
	if err != nil {
		logger.Debugf("read dir names failed: %v", err)
		return nil, err
	}
	var names []string
	for _, name := range dirnames {
		if validBackupName(name) {
			names = append(names, name)
		}
	}
	backups := createBackups(names)
	return backups, nil
}

func validBackupName(name string) bool {
	re := regexp.MustCompile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
	return re.MatchString(name)
}

func createBackups(backups []string) []dto.Backup {
	result := make([]dto.Backup, len(backups))
	for i, backup := range backups {
		result[i] = createBackup(backup)
	}
	return result
}

func createBackup(backup string) dto.Backup {
	h := dto.NewBackup()
	h.SetName(backup)
	return h
}

func (s *backupService) GetHost(host string) (dto.Host, error) {
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	h := dto.NewHost()
	h.SetName(host)
	return h, nil
}

func (s *backupService) GetLatestBackup(host dto.Host) (dto.Backup, error) {
	list, err := s.ListBackups(host)
	if err != nil {
		logger.Debugf("list backups failed: %v", err)
		return nil, err
	}
	if len(list) == 0 {
		return nil, nil
	}
	var names []string
	backups := make(map[string]dto.Backup, 0)
	for _, backup := range list {
		backups[backup.GetName()] = backup
		names = append(names, backup.GetName())
	}
	sort.Strings(names)
	return backups[names[len(names) - 1]], nil
}

func (s *backupService) ListOldBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	keep, err := getKeepBackups(backups)
	if err != nil {
		return nil, err
	}
	keepMap := make(map[string]bool)
	for _, b := range keep {
		keepMap[b.GetName()] = true
	}
	var result []dto.Backup
	for _, backup := range backups {
		if !keepMap[backup.GetName()] {
			result = append(result, backup)
		}
	}
	return result, nil
}

func getKeepBackups(backups []dto.Backup) ([]dto.Backup, error) {
	keep := make(map[string]dto.Backup, 0)
	now := time.Now()
	// keep all backups from today
	{
		b, err := getKeepToday(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per day if age <= 7 days
	{
		b, err := getKeepDay(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per week if age <= 28 days
	{
		b, err := getKeepWeek(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per month
	{
		b, err := getKeepMonth(backups)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep latest backup / current
	{
		backup := latestBackup(backups)
		if backup != nil {
			keep[backup.GetName()] = backup
		}
	}

	var result []dto.Backup
	for _, backup := range keep {
		result = append(result, backup)
	}
	return result, nil
}

func getKeepToday(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	var result []dto.Backup
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if isToday(t, now) {
			result = append(result, backup)
		}
	}
	return result, nil
}

func isToday(t time.Time, now time.Time) bool {
	return t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day()
}

func getKeepDay(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup

	var lastYear int = -1
	var lastMonth time.Month = -1
	var lastDay int = -1

	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if ageLessThanDays(t, now, 7) && (t.Year() != lastYear || t.Month() != lastMonth || t.Day() != lastDay) {
			result = append(result, backup)
			lastYear = t.Year()
			lastMonth = t.Month()
			lastDay = t.Day()
		}
	}
	return result, nil
}

func ageLessThanDays(t time.Time, now time.Time, days int64) bool {
	diff := now.Unix() - t.Unix()
	return diff <= 24*60*60*days
}

func getKeepMonth(backups []dto.Backup) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var lastYear int64 = -1
	var lastMonth int64 = -1
	var result []dto.Backup
	for _, backup := range backups {
		name := backup.GetName()
		year, err := strconv.ParseInt(name[0:4], 10, 64)
		if err != nil {
			return nil, err
		}
		month, err := strconv.ParseInt(name[5:7], 10, 64)
		if err != nil {
			return nil, err
		}
		logger.Tracef("year %d month %d", year, month)

		if year != lastYear || month != lastMonth {
			result = append(result, backup)
		}
		lastYear = year
		lastMonth = month
	}
	return result, nil
}

func (s *backupService) Cleanup(host dto.Host) error {
	if host == nil {
		return errors.New("parameter host missing")
	}
	backups, err := s.ListOldBackups(host)
	if err != nil {
		return err
	}
	logger.Debugf("found %d backup to delete for host %s", len(backups), host.GetName())
	for _, backup := range backups {
		dir := fmt.Sprintf("%s%c%s%c%s", s.rootdir, os.PathSeparator, host.GetName(), os.PathSeparator, backup.GetName())
		logger.Infof("delete %s started", dir)
		os.RemoveAll(dir)
		logger.Infof("delete %s finished", dir)
	}
	return nil
}

func (s *backupService) ListKeepBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	return getKeepBackups(backups)
}

func getTimeByName(backupName string) (time.Time, error) {
	return time.Parse("2006-01-02T15:04:05", backupName)
}

func getKeepWeek(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup
	var lastWeek int = -1
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		_, week := t.ISOWeek()
		if ageLessThanDays(t, now, 40) && week != lastWeek {
			result = append(result, backup)
			lastWeek = week
		}
	}
	return result, nil
}

func latestBackup(backups []dto.Backup) dto.Backup {
	if backups != nil && len(backups) > 0 {
		sort.Sort(util.BackupByDate(backups))
		return backups[len(backups) - 1]
	}
	return nil
}
<|endoftext|>


<commit_before>package urlfetch

import "testing"
import "regexp"

// Local helper functions
func parse(output string) bool {
    test, err := regexp.MatchString("", output)
    if  err != nil {
        return false
    }
    return test
}
func test_getdata4urls(urls []string) bool {
    ch := make(chan []byte)
    for _, url := range urls {
        go Getdata(url, ch)
    }
    for i := 0; i<len(urls); i++ {
        res := string(<-ch)
        if  ! parse(res) {
            return false
        }
    }
    return true
}
func test_getdata(url string) bool {
    ch := make(chan []byte)
    go Getdata(url, ch)
    res := string(<-ch)
    return parse(res)
}

// Test function
func TestGetdata(t *testing.T) {
    url1 := "http://www.google.com"
    url2 := "http://www.golang.org"
    urls := []string{url1, url2}
    var test bool
    test = test_getdata(url1)
    if  ! test {
        t.Log("test getdata call", url1)
        t.Fail()
    }
    test = test_getdata4urls(urls)
    if  ! test {
        t.Log("test getdata call with multiple urls", urls)
        t.Fail()
    }
}
<commit_msg>Replace Getdata with Fetch<commit_after>package urlfetch

import "testing"
import "regexp"

// Local helper functions
func parse(output string) bool {
    test, err := regexp.MatchString("", output)
    if  err != nil {
        return false
    }
    return test
}
func test_getdata4urls(urls []string) bool {
    ch := make(chan []byte)
    for _, url := range urls {
        go Fetch(url, ch)
    }
    for i := 0; i<len(urls); i++ {
        res := string(<-ch)
        if  ! parse(res) {
            return false
        }
    }
    return true
}
func test_getdata(url string) bool {
    ch := make(chan []byte)
    go Fetch(url, ch)
    res := string(<-ch)
    return parse(res)
}

// Test function
func TestFetch(t *testing.T) {
    url1 := "http://www.google.com"
    url2 := "http://www.golang.org"
    urls := []string{url1, url2}
    var test bool
    test = test_getdata(url1)
    if  ! test {
        t.Log("test getdata call", url1)
        t.Fail()
    }
    test = test_getdata4urls(urls)
    if  ! test {
        t.Log("test getdata call with multiple urls", urls)
        t.Fail()
    }
}
<|endoftext|>


<commit_before>package httpclient

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"time"

	"github.com/pkg/errors"
	"github.com/quan-xie/tuba/util/retry"
	"github.com/quan-xie/tuba/util/xtime"
)

const defaultRetryCount int = 0

type Config struct {
	Dial       xtime.Duration
	Timeout    xtime.Duration
	KeepAlive  xtime.Duration
	retryCount int
}

type HttpClient struct {
	client     *http.Client
	dialer     *net.Dialer
	transport  *http.Transport
	retryCount int
	retrier    retry.Retriable
}

// NewHTTPClientWithContext returns a new instance of httpClientWithContext
func NewHTTPClient(c *Config) *HttpClient {
	dialer := &net.Dialer{
		Timeout:   time.Duration(c.Dial),
		KeepAlive: time.Duration(c.KeepAlive),
	}
	transport := &http.Transport{
		DialContext:     dialer.DialContext,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	return &HttpClient{
		client: &http.Client{
			Transport: transport,
		},
		retryCount: defaultRetryCount,
		retrier:    retry.NewNoRetrier(),
	}
}

// SetRetryCount sets the retry count for the httpClient
func (c *HttpClient) SetRetryCount(count int) {
	c.retryCount = count
}

// SetRetryCount sets the retry count for the httpClient
func (c *HttpClient) SetRetrier(retrier retry.Retriable) {
	c.retrier = retrier
}

// Get makes a HTTP GET request to provided URL with context passed in
func (c *HttpClient) Get(ctx context.Context, url string, headers http.Header) (*http.Response, error) {
	var response *http.Response
	request, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return response, errors.Wrap(err, "GET - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request)
}

// Post makes a HTTP POST request to provided URL with context passed in
func (c *HttpClient) Post(ctx context.Context, url string, body io.Reader, headers http.Header) (*http.Response, error) {
	var response *http.Response
	request, err := http.NewRequest(http.MethodPost, url, body)
	if err != nil {
		return response, errors.Wrap(err, "POST - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request)
}

// Put makes a HTTP PUT request to provided URL with context passed in
func (c *HttpClient) Put(ctx context.Context, url string, body io.Reader, headers http.Header) (*http.Response, error) {
	var response *http.Response
	request, err := http.NewRequest(http.MethodPut, url, body)
	if err != nil {
		return response, errors.Wrap(err, "PUT - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request)
}

// Patch makes a HTTP PATCH request to provided URL with context passed in
func (c *HttpClient) Patch(ctx context.Context, url string, body io.Reader, headers http.Header) (*http.Response, error) {
	var response *http.Response
	request, err := http.NewRequest(http.MethodPatch, url, body)
	if err != nil {
		return response, errors.Wrap(err, "PATCH - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request)
}

// Delete makes a HTTP DELETE request to provided URL with context passed in
func (c *HttpClient) Delete(ctx context.Context, url string, headers http.Header) (*http.Response, error) {
	var response *http.Response
	request, err := http.NewRequest(http.MethodDelete, url, nil)
	if err != nil {
		return response, errors.Wrap(err, "DELETE - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request)
}

// Do makes an HTTP request with the native `http.Do` interface and context passed in
func (c *HttpClient) Do(ctx context.Context, req *http.Request) (response *http.Response, err error) {

	for i := 0; i <= c.retryCount; i++ {
		contextCancelled := false
		var err error
		response, err = c.client.Do(req.WithContext(ctx))
		if err != nil {
			select {
			case <-ctx.Done():
				err = ctx.Err()
				contextCancelled = true
			}

			if contextCancelled {
				break
			}
			backoffTime := c.retrier.NextInterval(i)
			time.Sleep(backoffTime)
			continue
		}

		if response.StatusCode >= http.StatusInternalServerError {

			backoffTime := c.retrier.NextInterval(i)
			time.Sleep(backoffTime)
			fmt.Println("R: ", response.StatusCode)
			continue
		}
		// Clear errors if any iteration succeeds
		break
	}

	return response, err
}
<commit_msg>modify httpclient<commit_after>package httpclient

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/json"
	"io"
	"net"
	"net/http"
	"time"

	"github.com/pkg/errors"
	"github.com/quan-xie/tuba/util/retry"
	"github.com/quan-xie/tuba/util/xtime"
)

const (
	_minRead              = 16 * 1024 // 16kb
	defaultRetryCount int = 0
)

type Config struct {
	Dial       xtime.Duration
	Timeout    xtime.Duration
	KeepAlive  xtime.Duration
	retryCount int
}

type HttpClient struct {
	client     *http.Client
	dialer     *net.Dialer
	transport  *http.Transport
	retryCount int
	retrier    retry.Retriable
}

// NewHTTPClient returns a new instance of httpClient
func NewHTTPClient(c *Config) *HttpClient {
	dialer := &net.Dialer{
		Timeout:   time.Duration(c.Dial),
		KeepAlive: time.Duration(c.KeepAlive),
	}
	transport := &http.Transport{
		DialContext:     dialer.DialContext,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	return &HttpClient{
		client: &http.Client{
			Transport: transport,
		},
		retryCount: defaultRetryCount,
		retrier:    retry.NewNoRetrier(),
	}
}

// SetRetryCount sets the retry count for the httpClient
func (c *HttpClient) SetRetryCount(count int) {
	c.retryCount = count
}

// SetRetryCount sets the retry count for the httpClient
func (c *HttpClient) SetRetrier(retrier retry.Retriable) {
	c.retrier = retrier
}

// Get makes a HTTP GET request to provided URL with context passed in
func (c *HttpClient) Get(ctx context.Context, url string, headers http.Header, res interface{}) (err error) {
	request, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return errors.Wrap(err, "GET - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request, res)
}

// Post makes a HTTP POST request to provided URL with context passed in
func (c *HttpClient) Post(ctx context.Context, url string, body io.Reader, headers http.Header, res interface{}) (err error) {
	request, err := http.NewRequest(http.MethodPost, url, body)
	if err != nil {
		return errors.Wrap(err, "POST - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request, res)
}

// Put makes a HTTP PUT request to provided URL with context passed in
func (c *HttpClient) Put(ctx context.Context, url string, body io.Reader, headers http.Header, res interface{}) (err error) {
	request, err := http.NewRequest(http.MethodPut, url, body)
	if err != nil {
		return errors.Wrap(err, "PUT - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request, res)
}

// Patch makes a HTTP PATCH request to provided URL with context passed in
func (c *HttpClient) Patch(ctx context.Context, url string, body io.Reader, headers http.Header, res interface{}) (err error) {
	request, err := http.NewRequest(http.MethodPatch, url, body)
	if err != nil {
		return errors.Wrap(err, "PATCH - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request, res)
}

// Delete makes a HTTP DELETE request to provided URL with context passed in
func (c *HttpClient) Delete(ctx context.Context, url string, headers http.Header, res interface{}) (err error) {
	request, err := http.NewRequest(http.MethodDelete, url, nil)
	if err != nil {
		return errors.Wrap(err, "DELETE - request creation failed")
	}

	request.Header = headers

	return c.Do(ctx, request, res)
}

// Do makes an HTTP request with the native `http.Do` interface and context passed in
func (c *HttpClient) Do(ctx context.Context, req *http.Request, res interface{}) (err error) {
	var (
		response *http.Response
		bs       []byte
	)
	for i := 0; i <= c.retryCount; i++ {
		contextCancelled := false
		var err error
		response, err = c.client.Do(req.WithContext(ctx))
		if err != nil {
			select {
			case <-ctx.Done():
				err = ctx.Err()
				contextCancelled = true
			}

			if contextCancelled {
				break
			}
			backoffTime := c.retrier.NextInterval(i)
			time.Sleep(backoffTime)
			continue
		}
		defer response.Body.Close()
		if response.StatusCode >= http.StatusInternalServerError {

			backoffTime := c.retrier.NextInterval(i)
			time.Sleep(backoffTime)
			continue
		}
		bs, err = readAll(response.Body, _minRead)
		if err != nil {
			err = errors.Wrap(err, "readAll - readAll failed")
			return err
		}
		if res != nil {
			if err = json.Unmarshal(bs, res); err != nil {
				err = errors.Wrap(err, "Unmarshal failed")
				return err
			}
		}
		// Clear errors if any iteration succeeds
		break
	}
	return
}

func readAll(r io.Reader, capacity int64) (b []byte, err error) {
	buf := bytes.NewBuffer(make([]byte, 0, capacity))
	// If the buffer overflows, we will get bytes.ErrTooLarge.
	// Return that as an error. Any other panic remains.
	defer func() {
		e := recover()
		if e == nil {
			return
		}
		if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge {
			err = panicErr
		} else {
			panic(e)
		}
	}()
	_, err = buf.ReadFrom(r)
	return buf.Bytes(), err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
	"strings"
)

const (
	defQueryType  = "ANY"
	defQueryClass = "IN"
	defProtocol   = "UDP"
)

var (
	eDns0 = false
)

// checkQueryParam checks against possible list of parameters
func checkQueryParam(arg string, list map[string]bool) bool {
	_, ok := list[strings.ToUpper(arg)]
	return ok
}

// init injects our "dns" related commands/options.
func init() {
	// Fill-in the various commands
	cliCommands = append(cliCommands, cli.Command{
		Name:        "dns",
		Usage:       "send dns queries",
		Description: "send DNS queries about an host/IP/domain\n   use: <Q> [<TYPE> [<CLASS>]]",
		Aliases: []string{
			"dig",
			"drill",
		},
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:        "6, ipv6",
				Usage:       "displays only IPv6",
				Destination: &fWant6,
			},
			cli.BoolFlag{
				Name:        "4, ipv4",
				Usage:       "displays only IPv4",
				Destination: &fWant4,
			},
			cli.BoolFlag{
				Name:        "E, edns0",
				Usage:       "use EDNS0",
				Destination: &eDns0,
			},
			/*			cli.StringFlag{
							Name:        "t, qtype",
							Usage:       "Select the query type",
							Destination: &defQueryType,
						},
						cli.StringFlag{
							Name:        "c, qclass",
							Usage:       "Select the query class",
							Destination: &defQueryClass,
						},*/
			cli.StringFlag{
				Name:        "p, protocol",
				Usage:       "Select UDP or TCP",
				Destination: &fProtocol,
			},
		},
		Action: cmdDNS,
	})
}

func cmdDNS(c *cli.Context) error {
	var addr, qtype, qclass, proto string

	// By default we want both
	if !fWant4 && !fWant6 {
		fWant6, fWant4 = true, true
	}
	args := c.Args()
	if args == nil || len(args) == 0 {
		log.Fatal("Error: you must specify at least a name")
	}

	qtype = defQueryType
	qclass = defQueryClass
	proto = defProtocol

	if len(args) == 1 {
		addr = args[0]
	} else if len(args) == 2 {
		addr = args[0]
		qtype = args[1]
	} else if len(args) == 3 {
		addr = args[0]
		qtype = args[1]
		qclass = args[2]
	}

	if fProtocol != "" {
		log.Printf("Use %s", fProtocol)
		proto = fProtocol
	}

	var defs []atlas.Definition

	if fWant4 {
		def := atlas.Definition{
			AF:            4,
			Description:   fmt.Sprintf("DNS v4 - %s %s %s", addr, qtype, qclass),
			Type:          "dns",
			Protocol:      proto,
			QueryArgument: addr,
			QueryClass:    qclass,
			QueryType:     qtype,
		}
		if eDns0 {
			def.UDPPayloadSize = 4096
			def.Protocol = "UDP"
		}
		defs = append(defs, def)
	}

	if fWant6 {
		def := atlas.Definition{
			AF:            6,
			Description:   fmt.Sprintf("DNS v6 - %s %s %s", addr, qtype, qclass),
			Type:          "dns",
			Protocol:      proto,
			QueryArgument: addr,
			QueryClass:    qclass,
			QueryType:     qtype,
		}
		if eDns0 {
			def.UDPPayloadSize = 4096
			def.Protocol = "UDP"
		}
		defs = append(defs, def)
	}

	req := atlas.MeasurementRequest{
		Definitions: defs,
		IsOneoff:    true,
	}
	// Default set of probes
	probes := atlas.ProbeSet{
		{
			Requested: mycnf.PoolSize,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}

	req.Probes = probes
	log.Printf("req=%#v", req)
	m, err := atlas.DNS(req)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}

	//str := res.Result.Display()
	fmt.Printf("m: %v\n", m)

	return nil
}
<commit_msg>Add two dns related parameters for CD/DO bits.<commit_after>package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
	"strings"
)

const (
	defQueryType  = "ANY"
	defQueryClass = "IN"
	defProtocol   = "UDP"
)

var (
	eDns0 = false
)

// checkQueryParam checks against possible list of parameters
func checkQueryParam(arg string, list map[string]bool) bool {
	_, ok := list[strings.ToUpper(arg)]
	return ok
}

// init injects our "dns" related commands/options.
func init() {
	// Fill-in the various commands
	cliCommands = append(cliCommands, cli.Command{
		Name:        "dns",
		Usage:       "send dns queries",
		Description: "send DNS queries about an host/IP/domain\n   use: <Q> [<TYPE> [<CLASS>]]",
		Aliases: []string{
			"dig",
			"drill",
		},
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:        "6, ipv6",
				Usage:       "displays only IPv6",
				Destination: &fWant6,
			},
			cli.BoolFlag{
				Name:        "4, ipv4",
				Usage:       "displays only IPv4",
				Destination: &fWant4,
			},
			cli.BoolFlag{
				Name:        "E, edns0",
				Usage:       "use EDNS0",
				Destination: &eDns0,
			},
			cli.BoolFlag{
				Name:        "D, disable-dnssec",
				Usage:       "Do not try to validate DNSSEC RR",
				Destination: &fDisableDNSSEC,
			},
			cli.BoolFlag{
				Name:        "C, disable-dnssec-checks",
				Usage:       "Do not try to validate DNSSEC Check by probes",
				Destination: &fBitCD,
			},
			/*			cli.StringFlag{
							Name:        "t, qtype",
							Usage:       "Select the query type",
							Destination: &defQueryType,
						},
						cli.StringFlag{
							Name:        "c, qclass",
							Usage:       "Select the query class",
							Destination: &defQueryClass,
						},*/
			cli.StringFlag{
				Name:        "p, protocol",
				Usage:       "Select UDP or TCP",
				Destination: &fProtocol,
			},
		},
		Action: cmdDNS,
	})
}

func cmdDNS(c *cli.Context) error {
	var (
		bitDO  = true
		bitCD  = false
		qtype  = defQueryType
		qclass = defQueryClass
		proto  = defProtocol

		addr string
	)

	// By default we want both
	if !fWant4 && !fWant6 {
		fWant6, fWant4 = true, true
	}
	args := c.Args()
	if args == nil || len(args) == 0 {
		log.Fatal("Error: you must specify at least a name")
	}

	qtype = defQueryType
	qclass = defQueryClass
	proto = defProtocol

	if len(args) == 1 {
		addr = args[0]
	} else if len(args) == 2 {
		addr = args[0]
		qtype = args[1]
	} else if len(args) == 3 {
		addr = args[0]
		qtype = args[1]
		qclass = args[2]
	}

	if fProtocol != "" {
		log.Printf("Use %s", fProtocol)
		proto = fProtocol
	}

	if fDisableDNSSEC {
		bitDO = false
	}

	if fBitCD {
		bitCD = true
	}

	var defs []atlas.Definition

	if fWant4 {
		def := atlas.Definition{
			AF:            4,
			Description:   fmt.Sprintf("DNS v4 - %s %s %s", addr, qtype, qclass),
			Type:          "dns",
			Protocol:      proto,
			QueryArgument: addr,
			QueryClass:    qclass,
			QueryType:     qtype,
			SetDOBit:      bitDO,
			SetCDBit:      bitCD,
		}
		if eDns0 {
			def.UDPPayloadSize = 4096
			def.Protocol = "UDP"
		}
		defs = append(defs, def)
	}

	if fWant6 {
		def := atlas.Definition{
			AF:            6,
			Description:   fmt.Sprintf("DNS v6 - %s %s %s", addr, qtype, qclass),
			Type:          "dns",
			Protocol:      proto,
			QueryArgument: addr,
			QueryClass:    qclass,
			QueryType:     qtype,
			SetDOBit:      bitDO,
			SetCDBit:      bitCD,
		}
		if eDns0 {
			def.UDPPayloadSize = 4096
			def.Protocol = "UDP"
		}
		defs = append(defs, def)
	}

	req := atlas.MeasurementRequest{
		Definitions: defs,
		IsOneoff:    true,
	}
	// Default set of probes
	probes := atlas.ProbeSet{
		{
			Requested: mycnf.PoolSize,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}

	req.Probes = probes
	log.Printf("req=%#v", req)
	m, err := atlas.DNS(req)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}

	//str := res.Result.Display()
	fmt.Printf("m: %v\n", m)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package ssh

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"os/user"
	"strings"

	"code.google.com/p/go.crypto/ssh"

	"launchpad.net/juju-core/utils"
)

const sshDefaultPort = 22

// GoCryptoClient is an implementation of Client that
// uses the embedded go.crypto/ssh SSH client.
//
// GoCryptoClient is intentionally limited in the
// functionality that it enables, as it is currently
// intended to be used only for non-interactive command
// execution.
type GoCryptoClient struct {
	signers []ssh.Signer
}

// NewGoCryptoClient creates a new GoCryptoClient.
//
// If no signers are specified, NewGoCryptoClient will
// use the private key generated by LoadClientKeys.
func NewGoCryptoClient(signers ...ssh.Signer) (*GoCryptoClient, error) {
	return &GoCryptoClient{signers: signers}, nil
}

// Command implements Client.Command.
func (c *GoCryptoClient) Command(host string, command []string, options *Options) *Cmd {
	shellCommand := utils.CommandString(command...)
	signers := c.signers
	if len(signers) == 0 {
		signers = privateKeys()
	}
	user, host := splitUserHost(host)
	port := sshDefaultPort
	var proxyCommand []string
	if options != nil {
		if options.port != 0 {
			port = options.port
		}
		proxyCommand = options.proxyCommand
	}
	logger.Debugf(`running (equivalent of): ssh "%s@%s" -p %d '%s'`, user, host, port, shellCommand)
	return &Cmd{impl: &goCryptoCommand{
		signers:      signers,
		user:         user,
		addr:         fmt.Sprintf("%s:%d", host, port),
		command:      shellCommand,
		proxyCommand: proxyCommand,
	}}
}

// Copy implements Client.Copy.
//
// Copy is currently unimplemented, and will always return an error.
func (c *GoCryptoClient) Copy(targets, extraArgs []string, options *Options) error {
	return fmt.Errorf("scp command is not implemented (OpenSSH scp not available in PATH)")
}

type goCryptoCommand struct {
	signers      []ssh.Signer
	user         string
	addr         string
	command      string
	proxyCommand []string
	stdin        io.Reader
	stdout       io.Writer
	stderr       io.Writer
	conn         *ssh.ClientConn
	sess         *ssh.Session
}

var sshDial = ssh.Dial

var sshDialWithProxy = func(addr string, proxyCommand []string, config *ssh.ClientConfig) (*ssh.ClientConn, error) {
	if len(proxyCommand) == 0 {
		return sshDial("tcp", addr, config)
	}
	// User has specified a proxy. Create a pipe and
	// redirect the proxy command's stdin/stdout to it.
	host, port, err := net.SplitHostPort(addr)
	if err != nil {
		host = addr
	}
	for i, arg := range proxyCommand {
		arg = strings.Replace(arg, "%h", host, -1)
		if port != "" {
			arg = strings.Replace(arg, "%p", port, -1)
		}
		arg = strings.Replace(arg, "%r", config.User, -1)
		proxyCommand[i] = arg
	}
	client, server := net.Pipe()
	logger.Debugf(`executing proxy command %q`, proxyCommand)
	cmd := exec.Command(proxyCommand[0], proxyCommand[1:]...)
	cmd.Stdin = server
	cmd.Stdout = server
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return ssh.Client(client, config)
}

func (c *goCryptoCommand) ensureSession() (*ssh.Session, error) {
	if c.sess != nil {
		return c.sess, nil
	}
	if len(c.signers) == 0 {
		return nil, fmt.Errorf("no private keys available")
	}
	if c.user == "" {
		currentUser, err := user.Current()
		if err != nil {
			return nil, fmt.Errorf("getting current user: %v", err)
		}
		c.user = currentUser.Username
	}
	config := &ssh.ClientConfig{
		User: c.user,
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthKeyring(keyring{c.signers}),
		},
	}
	conn, err := sshDialWithProxy(c.addr, c.proxyCommand, config)
	if err != nil {
		return nil, err
	}
	sess, err := conn.NewSession()
	if err != nil {
		conn.Close()
		return nil, err
	}
	c.conn = conn
	c.sess = sess
	c.sess.Stdin = c.stdin
	c.sess.Stdout = c.stdout
	c.sess.Stderr = c.stderr
	return sess, nil
}

func (c *goCryptoCommand) Start() error {
	sess, err := c.ensureSession()
	if err != nil {
		return err
	}
	if c.command == "" {
		return sess.Shell()
	}
	return sess.Start(c.command)
}

func (c *goCryptoCommand) Close() error {
	if c.sess == nil {
		return nil
	}
	err0 := c.sess.Close()
	err1 := c.conn.Close()
	if err0 == nil {
		err0 = err1
	}
	c.sess = nil
	c.conn = nil
	return err0
}

func (c *goCryptoCommand) Wait() error {
	if c.sess == nil {
		return fmt.Errorf("Command has not been started")
	}
	err := c.sess.Wait()
	c.Close()
	return err
}

func (c *goCryptoCommand) Kill() error {
	if c.sess == nil {
		return fmt.Errorf("Command has not been started")
	}
	return c.sess.Signal(ssh.SIGKILL)
}

func (c *goCryptoCommand) SetStdio(stdin io.Reader, stdout, stderr io.Writer) {
	c.stdin = stdin
	c.stdout = stdout
	c.stderr = stderr
}

func (c *goCryptoCommand) StdinPipe() (io.WriteCloser, io.Reader, error) {
	sess, err := c.ensureSession()
	if err != nil {
		return nil, nil, err
	}
	wc, err := sess.StdinPipe()
	return wc, sess.Stdin, err
}

func (c *goCryptoCommand) StdoutPipe() (io.ReadCloser, io.Writer, error) {
	sess, err := c.ensureSession()
	if err != nil {
		return nil, nil, err
	}
	wc, err := sess.StdoutPipe()
	return ioutil.NopCloser(wc), sess.Stdout, err
}

func (c *goCryptoCommand) StderrPipe() (io.ReadCloser, io.Writer, error) {
	sess, err := c.ensureSession()
	if err != nil {
		return nil, nil, err
	}
	wc, err := sess.StderrPipe()
	return ioutil.NopCloser(wc), sess.Stderr, err
}

// keyring implements ssh.ClientKeyring
type keyring struct {
	signers []ssh.Signer
}

func (k keyring) Key(i int) (ssh.PublicKey, error) {
	if i < 0 || i >= len(k.signers) {
		// nil key marks the end of the keyring; must not return an error.
		return nil, nil
	}
	return k.signers[i].PublicKey(), nil
}

func (k keyring) Sign(i int, rand io.Reader, data []byte) ([]byte, error) {
	if i < 0 || i >= len(k.signers) {
		return nil, fmt.Errorf("no key at position %d", i)
	}
	return k.signers[i].Sign(rand, data)
}

func splitUserHost(s string) (user, host string) {
	userHost := strings.SplitN(s, "@", 2)
	if len(userHost) == 2 {
		return userHost[0], userHost[1]
	}
	return "", userHost[0]
}
<commit_msg>cmd/jujud: start peergrouper; enable ensure-availability command<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package ssh

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"os/user"
	"strings"

	"code.google.com/p/go.crypto/ssh"

	"launchpad.net/juju-core/utils"
)

const sshDefaultPort = 22

// GoCryptoClient is an implementation of Client that
// uses the embedded go.crypto/ssh SSH client.
//
// GoCryptoClient is intentionally limited in the
// functionality that it enables, as it is currently
// intended to be used only for non-interactive command
// execution.
type GoCryptoClient struct {
	signers []ssh.Signer
}

// NewGoCryptoClient creates a new GoCryptoClient.
//
// If no signers are specified, NewGoCryptoClient will
// use the private key generated by LoadClientKeys.
func NewGoCryptoClient(signers ...ssh.Signer) (*GoCryptoClient, error) {
	return &GoCryptoClient{signers: signers}, nil
}

// Command implements Client.Command.
func (c *GoCryptoClient) Command(host string, command []string, options *Options) *Cmd {
	shellCommand := utils.CommandString(command...)
	signers := c.signers
	if len(signers) == 0 {
		signers = privateKeys()
	}
	user, host := splitUserHost(host)
	port := sshDefaultPort
	var proxyCommand []string
	if options != nil {
		if options.port != 0 {
			port = options.port
		}
		proxyCommand = options.proxyCommand
	}
	logger.Debugf(`running (equivalent of): ssh "%s@%s" -p %d '%s'`, user, host, port, shellCommand)
	return &Cmd{impl: &goCryptoCommand{
		signers:      signers,
		user:         user,
		addr:         fmt.Sprintf("%s:%d", host, port),
		command:      shellCommand,
		proxyCommand: proxyCommand,
	}}
}

// Copy implements Client.Copy.
//
// Copy is currently unimplemented, and will always return an error.
func (c *GoCryptoClient) Copy(targets, extraArgs []string, options *Options) error {
	return fmt.Errorf("scp command is not implemented (OpenSSH scp not available in PATH)")
}

type goCryptoCommand struct {
	signers      []ssh.Signer
	user         string
	addr         string
	command      string
	proxyCommand []string
	stdin        io.Reader
	stdout       io.Writer
	stderr       io.Writer
	conn         *ssh.ClientConn
	sess         *ssh.Session
}

var sshDial = ssh.Dial

var sshDialWithProxy = func(addr string, proxyCommand []string, config *ssh.ClientConfig) (*ssh.ClientConn, error) {
	if len(proxyCommand) == 0 {
		return sshDial("tcp", addr, config)
	}
	// User has specified a proxy. Create a pipe and
	// redirect the proxy command's stdin/stdout to it.
	host, port, err := net.SplitHostPort(addr)
	if err != nil {
		host = addr
	}
	for i, arg := range proxyCommand {
		arg = strings.Replace(arg, "%h", host, -1)
		if port != "" {
			arg = strings.Replace(arg, "%p", port, -1)
		}
		arg = strings.Replace(arg, "%r", config.User, -1)
		proxyCommand[i] = arg
	}
	client, server := net.Pipe()
	logger.Debugf(`executing proxy command %q`, proxyCommand)
	cmd := exec.Command(proxyCommand[0], proxyCommand[1:]...)
	cmd.Stdin = server
	cmd.Stdout = server
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return ssh.Client(client, config)
}

func (c *goCryptoCommand) ensureSession() (*ssh.Session, error) {
	if c.sess != nil {
		return c.sess, nil
	}
	if len(c.signers) == 0 {
		return nil, fmt.Errorf("no private keys available")
	}
	if c.user == "" {
		currentUser, err := user.Current()
		if err != nil {
			return nil, fmt.Errorf("getting current user: %v", err)
		}
		c.user = currentUser.Username
	}
	config := &ssh.ClientConfig{
		User: c.user,
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthKeyring(keyring{c.signers}),
		},
	}
	logger.Debugf("dialing with proxyCommand %q", c.proxyCommand)
	conn, err := sshDialWithProxy(c.addr, c.proxyCommand, config)
	if err != nil {
		return nil, err
	}
	sess, err := conn.NewSession()
	if err != nil {
		conn.Close()
		return nil, err
	}
	c.conn = conn
	c.sess = sess
	c.sess.Stdin = c.stdin
	c.sess.Stdout = c.stdout
	c.sess.Stderr = c.stderr
	return sess, nil
}

func (c *goCryptoCommand) Start() error {
	sess, err := c.ensureSession()
	if err != nil {
		return err
	}
	if c.command == "" {
		return sess.Shell()
	}
	return sess.Start(c.command)
}

func (c *goCryptoCommand) Close() error {
	if c.sess == nil {
		return nil
	}
	err0 := c.sess.Close()
	err1 := c.conn.Close()
	if err0 == nil {
		err0 = err1
	}
	c.sess = nil
	c.conn = nil
	return err0
}

func (c *goCryptoCommand) Wait() error {
	if c.sess == nil {
		return fmt.Errorf("Command has not been started")
	}
	err := c.sess.Wait()
	c.Close()
	return err
}

func (c *goCryptoCommand) Kill() error {
	if c.sess == nil {
		return fmt.Errorf("Command has not been started")
	}
	return c.sess.Signal(ssh.SIGKILL)
}

func (c *goCryptoCommand) SetStdio(stdin io.Reader, stdout, stderr io.Writer) {
	c.stdin = stdin
	c.stdout = stdout
	c.stderr = stderr
}

func (c *goCryptoCommand) StdinPipe() (io.WriteCloser, io.Reader, error) {
	sess, err := c.ensureSession()
	if err != nil {
		return nil, nil, err
	}
	wc, err := sess.StdinPipe()
	return wc, sess.Stdin, err
}

func (c *goCryptoCommand) StdoutPipe() (io.ReadCloser, io.Writer, error) {
	sess, err := c.ensureSession()
	if err != nil {
		return nil, nil, err
	}
	wc, err := sess.StdoutPipe()
	return ioutil.NopCloser(wc), sess.Stdout, err
}

func (c *goCryptoCommand) StderrPipe() (io.ReadCloser, io.Writer, error) {
	sess, err := c.ensureSession()
	if err != nil {
		return nil, nil, err
	}
	wc, err := sess.StderrPipe()
	return ioutil.NopCloser(wc), sess.Stderr, err
}

// keyring implements ssh.ClientKeyring
type keyring struct {
	signers []ssh.Signer
}

func (k keyring) Key(i int) (ssh.PublicKey, error) {
	if i < 0 || i >= len(k.signers) {
		// nil key marks the end of the keyring; must not return an error.
		return nil, nil
	}
	return k.signers[i].PublicKey(), nil
}

func (k keyring) Sign(i int, rand io.Reader, data []byte) ([]byte, error) {
	if i < 0 || i >= len(k.signers) {
		return nil, fmt.Errorf("no key at position %d", i)
	}
	return k.signers[i].Sign(rand, data)
}

func splitUserHost(s string) (user, host string) {
	userHost := strings.SplitN(s, "@", 2)
	if len(userHost) == 2 {
		return userHost[0], userHost[1]
	}
	return "", userHost[0]
}
<|endoftext|>


<commit_before>package service

import (
	"errors"
	"fmt"
	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"
)

type BackupService interface {
	GetHost(host string) (dto.Host, error)
	ListHosts() ([]dto.Host, error)
	ListBackups(host dto.Host) ([]dto.Backup, error)
	ListOldBackups(host dto.Host) ([]dto.Backup, error)
	ListKeepBackups(host dto.Host) ([]dto.Backup, error)
	GetLatestBackup(host dto.Host) (dto.Backup, error)
	Cleanup(host dto.Host) error
}

type backupService struct {
	rootdir string
}

var logger = log.DefaultLogger

func NewBackupService(rootdir string) *backupService {
	s := new(backupService)
	s.rootdir = rootdir
	return s
}

func (s *backupService) ListHosts() ([]dto.Host, error) {
	file, err := os.Open(s.rootdir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("rootdir %s is not a directory", s.rootdir)
	}
	names, err := file.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	return s.createHosts(names)
}

func (s *backupService) createHosts(hosts []string) ([]dto.Host, error) {
	var result []dto.Host
	for _, host := range hosts {
		dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
		isDir, err := isDir(dir)
		if err != nil {
			return nil, err
		}
		if isDir {
			result = append(result, createHost(host))
		} else {
			logger.Warnf("createHost for %s failed, is not a directory", host)
		}
	}
	return result, nil
}

func isDir(dir string) (bool, error) {
	file, err := os.Open(dir)
	if err != nil {
		return false, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return fileinfo.IsDir(), nil
}

func createHost(host string) dto.Host {
	h := dto.NewHost()
	h.SetName(host)
	return h
}

func (s *backupService) ListBackups(host dto.Host) ([]dto.Backup, error) {
	if host == nil {
		return nil, errors.New("parameter host missing")
	}
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host.GetName())
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	dirnames, err := file.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	var names []string
	for _, name := range dirnames {
		if validBackupName(name) {
			names = append(names, name)
		}
	}
	backups := createBackups(names)
	return backups, nil
}

func validBackupName(name string) bool {
	re := regexp.MustCompile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
	return re.MatchString(name)
}

func createBackups(backups []string) []dto.Backup {
	result := make([]dto.Backup, len(backups))
	for i, backup := range backups {
		result[i] = createBackup(backup)
	}
	return result
}

func createBackup(backup string) dto.Backup {
	h := dto.NewBackup()
	h.SetName(backup)
	return h
}

func (s *backupService) GetHost(host string) (dto.Host, error) {
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	h := dto.NewHost()
	h.SetName(host)
	return h, nil
}

func (s *backupService) GetLatestBackup(host dto.Host) (dto.Backup, error) {
	list, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	if len(list) == 0 {
		return nil, nil
	}
	var names []string
	backups := make(map[string]dto.Backup, 0)
	for _, backup := range list {
		backups[backup.GetName()] = backup
		names = append(names, backup.GetName())
	}
	sort.Strings(names)
	return backups[names[len(names)-1]], nil
}

func (s *backupService) ListOldBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	keep, err := getKeepBackups(backups)
	if err != nil {
		return nil, err
	}
	keepMap := make(map[string]bool)
	for _, b := range keep {
		keepMap[b.GetName()] = true
	}
	var result []dto.Backup
	for _, backup := range backups {
		if !keepMap[backup.GetName()] {
			result = append(result, backup)
		}
	}
	return result, nil
}

func getKeepBackups(backups []dto.Backup) ([]dto.Backup, error) {
	keep := make(map[string]dto.Backup, 0)
	now := time.Now()
	// keep all backups from today
	{
		b, err := getKeepToday(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per day if age <= 7 days
	{
		b, err := getKeepDay(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per week if age <= 28 days
	{
		b, err := getKeepWeek(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per month
	{
		b, err := getKeepMonth(backups)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}

	var result []dto.Backup
	for _, backup := range keep {
		result = append(result, backup)
	}
	return result, nil
}

func getKeepToday(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	var result []dto.Backup
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if isToday(t, now) {
			result = append(result, backup)
		}
	}
	return result, nil
}

func isToday(t time.Time, now time.Time) bool {
	return t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day()
}

func getKeepDay(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup

	var lastYear int = -1
	var lastMonth time.Month = -1
	var lastDay int = -1

	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if ageLessThanDays(t, now, 7) && (t.Year() != lastYear || t.Month() != lastMonth || t.Day() != lastDay) {
			result = append(result, backup)
			lastYear = t.Year()
			lastMonth = t.Month()
			lastDay = t.Day()
		}
	}
	return result, nil
}

func ageLessThanDays(t time.Time, now time.Time, days int64) bool {
	diff := now.Unix() - t.Unix()
	return diff <= 24*60*60*days
}

func getKeepMonth(backups []dto.Backup) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var lastYear int64 = -1
	var lastMonth int64 = -1
	var result []dto.Backup
	for _, backup := range backups {
		name := backup.GetName()
		year, err := strconv.ParseInt(name[0:4], 10, 64)
		if err != nil {
			return nil, err
		}
		month, err := strconv.ParseInt(name[5:7], 10, 64)
		if err != nil {
			return nil, err
		}
		logger.Debugf("year %d month %d", year, month)

		if year != lastYear || month != lastMonth {
			result = append(result, backup)
		}
		lastYear = year
		lastMonth = month
	}
	return result, nil
}

func (s *backupService) Cleanup(host dto.Host) error {
	if host == nil {
		return errors.New("parameter host missing")
	}
	backups, err := s.ListOldBackups(host)
	if err != nil {
		return err
	}
	logger.Debugf("found %d backup to delete for host %s", len(backups), host.GetName())
	for _, backup := range backups {
		dir := fmt.Sprintf("%s%c%s%c%s", s.rootdir, os.PathSeparator, host.GetName(), os.PathSeparator, backup.GetName())
		logger.Infof("delete %s started", dir)
		os.RemoveAll(dir)
		logger.Infof("delete %s finished", dir)
	}
	return nil
}

func (s *backupService) ListKeepBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	return getKeepBackups(backups)
}

func getTimeByName(backupName string) (time.Time, error) {
	return time.Parse("2006-01-02T15:04:05", backupName)
}

func getKeepWeek(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup
	var lastWeek int = -1
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		_, week := t.ISOWeek()
		if ageLessThanDays(t, now, 40) && week != lastWeek {
			result = append(result, backup)
			lastWeek = week
		}
	}
	return result, nil
}
<commit_msg>change loglevel<commit_after>package service

import (
	"errors"
	"fmt"
	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"
)

type BackupService interface {
	GetHost(host string) (dto.Host, error)
	ListHosts() ([]dto.Host, error)
	ListBackups(host dto.Host) ([]dto.Backup, error)
	ListOldBackups(host dto.Host) ([]dto.Backup, error)
	ListKeepBackups(host dto.Host) ([]dto.Backup, error)
	GetLatestBackup(host dto.Host) (dto.Backup, error)
	Cleanup(host dto.Host) error
}

type backupService struct {
	rootdir string
}

var logger = log.DefaultLogger

func NewBackupService(rootdir string) *backupService {
	s := new(backupService)
	s.rootdir = rootdir
	return s
}

func (s *backupService) ListHosts() ([]dto.Host, error) {
	file, err := os.Open(s.rootdir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("rootdir %s is not a directory", s.rootdir)
	}
	names, err := file.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	return s.createHosts(names)
}

func (s *backupService) createHosts(hosts []string) ([]dto.Host, error) {
	var result []dto.Host
	for _, host := range hosts {
		dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
		isDir, err := isDir(dir)
		if err != nil {
			return nil, err
		}
		if isDir {
			result = append(result, createHost(host))
		} else {
			logger.Warnf("createHost for %s failed, is not a directory", host)
		}
	}
	return result, nil
}

func isDir(dir string) (bool, error) {
	file, err := os.Open(dir)
	if err != nil {
		return false, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return fileinfo.IsDir(), nil
}

func createHost(host string) dto.Host {
	h := dto.NewHost()
	h.SetName(host)
	return h
}

func (s *backupService) ListBackups(host dto.Host) ([]dto.Backup, error) {
	if host == nil {
		return nil, errors.New("parameter host missing")
	}
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host.GetName())
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	dirnames, err := file.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	var names []string
	for _, name := range dirnames {
		if validBackupName(name) {
			names = append(names, name)
		}
	}
	backups := createBackups(names)
	return backups, nil
}

func validBackupName(name string) bool {
	re := regexp.MustCompile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
	return re.MatchString(name)
}

func createBackups(backups []string) []dto.Backup {
	result := make([]dto.Backup, len(backups))
	for i, backup := range backups {
		result[i] = createBackup(backup)
	}
	return result
}

func createBackup(backup string) dto.Backup {
	h := dto.NewBackup()
	h.SetName(backup)
	return h
}

func (s *backupService) GetHost(host string) (dto.Host, error) {
	dir := fmt.Sprintf("%s%c%s", s.rootdir, os.PathSeparator, host)
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	h := dto.NewHost()
	h.SetName(host)
	return h, nil
}

func (s *backupService) GetLatestBackup(host dto.Host) (dto.Backup, error) {
	list, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	if len(list) == 0 {
		return nil, nil
	}
	var names []string
	backups := make(map[string]dto.Backup, 0)
	for _, backup := range list {
		backups[backup.GetName()] = backup
		names = append(names, backup.GetName())
	}
	sort.Strings(names)
	return backups[names[len(names)-1]], nil
}

func (s *backupService) ListOldBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	keep, err := getKeepBackups(backups)
	if err != nil {
		return nil, err
	}
	keepMap := make(map[string]bool)
	for _, b := range keep {
		keepMap[b.GetName()] = true
	}
	var result []dto.Backup
	for _, backup := range backups {
		if !keepMap[backup.GetName()] {
			result = append(result, backup)
		}
	}
	return result, nil
}

func getKeepBackups(backups []dto.Backup) ([]dto.Backup, error) {
	keep := make(map[string]dto.Backup, 0)
	now := time.Now()
	// keep all backups from today
	{
		b, err := getKeepToday(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per day if age <= 7 days
	{
		b, err := getKeepDay(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per week if age <= 28 days
	{
		b, err := getKeepWeek(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per month
	{
		b, err := getKeepMonth(backups)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}

	var result []dto.Backup
	for _, backup := range keep {
		result = append(result, backup)
	}
	return result, nil
}

func getKeepToday(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	var result []dto.Backup
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if isToday(t, now) {
			result = append(result, backup)
		}
	}
	return result, nil
}

func isToday(t time.Time, now time.Time) bool {
	return t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day()
}

func getKeepDay(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup

	var lastYear int = -1
	var lastMonth time.Month = -1
	var lastDay int = -1

	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if ageLessThanDays(t, now, 7) && (t.Year() != lastYear || t.Month() != lastMonth || t.Day() != lastDay) {
			result = append(result, backup)
			lastYear = t.Year()
			lastMonth = t.Month()
			lastDay = t.Day()
		}
	}
	return result, nil
}

func ageLessThanDays(t time.Time, now time.Time, days int64) bool {
	diff := now.Unix() - t.Unix()
	return diff <= 24*60*60*days
}

func getKeepMonth(backups []dto.Backup) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var lastYear int64 = -1
	var lastMonth int64 = -1
	var result []dto.Backup
	for _, backup := range backups {
		name := backup.GetName()
		year, err := strconv.ParseInt(name[0:4], 10, 64)
		if err != nil {
			return nil, err
		}
		month, err := strconv.ParseInt(name[5:7], 10, 64)
		if err != nil {
			return nil, err
		}
		logger.Tracef("year %d month %d", year, month)

		if year != lastYear || month != lastMonth {
			result = append(result, backup)
		}
		lastYear = year
		lastMonth = month
	}
	return result, nil
}

func (s *backupService) Cleanup(host dto.Host) error {
	if host == nil {
		return errors.New("parameter host missing")
	}
	backups, err := s.ListOldBackups(host)
	if err != nil {
		return err
	}
	logger.Debugf("found %d backup to delete for host %s", len(backups), host.GetName())
	for _, backup := range backups {
		dir := fmt.Sprintf("%s%c%s%c%s", s.rootdir, os.PathSeparator, host.GetName(), os.PathSeparator, backup.GetName())
		logger.Infof("delete %s started", dir)
		os.RemoveAll(dir)
		logger.Infof("delete %s finished", dir)
	}
	return nil
}

func (s *backupService) ListKeepBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	return getKeepBackups(backups)
}

func getTimeByName(backupName string) (time.Time, error) {
	return time.Parse("2006-01-02T15:04:05", backupName)
}

func getKeepWeek(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup
	var lastWeek int = -1
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		_, week := t.ISOWeek()
		if ageLessThanDays(t, now, 40) && week != lastWeek {
			result = append(result, backup)
			lastWeek = week
		}
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"github.com/go-gl/gl"
)

type Sprite struct {
	Element
	system    *System
	texture   *Texture
	frame     int
	texture1  float64
	texture2  float64
	VelocityX float64
	VelocityY float64
	Type      int
	Collide   bool
	FlipX     bool
}

func (s *System) NewSprite(name string, x, y, w, h float64, t int) *Sprite {
	sprite := &Sprite{
		frame:   -1,
		system:  s,
		texture: s.Textures[name],
		Type:    t,
		FlipX:   false,
		Collide: true,
	}
	sprite.SetBounds(Rect(x, y, x+w, y+h))
	sprite.SetFrame(0)
	return sprite
}

func (s *Sprite) SetVelocity(pt Point) {
	s.VelocityX = pt.X
	s.VelocityY = pt.Y
}

func (s *Sprite) Velocity() Point {
	return Pt(s.VelocityX, s.VelocityY)
}

func (s *Sprite) TestMove(dx float64, dy float64, r *Sprite) bool {
	var (
		pad = float64(0.01)
		sb  = s.GlobalBounds()
		rb  = r.GlobalBounds()
		p   = Pt(dx, dy)
	)
	sb.Min.X += pad
	sb.Min.Y += pad
	sb.Max.X -= pad
	sb.Max.Y -= pad
	if sb.Add(p).Overlaps(rb) {
		return false
	}
	return true
}

func (s *Sprite) CollidesWith(sprite *Sprite) bool {
	return !s.TestMove(0, 0, sprite)
}

func (s *Sprite) SetFrame(frame int) {
	if frame == s.frame {
		return
	}
	s.frame = frame % len(s.texture.Frames)
	var (
		tex = s.texture.Frames[s.frame]
	)
	s.texture1 = float64(tex[0]) / float64(s.texture.Width)
	s.texture2 = float64(tex[1]) / float64(s.texture.Width)
}

func (s *Sprite) Draw() {
	var (
		b    = s.GlobalBounds()
		z    = s.Z()
		minx = b.Min.X
		maxx = b.Max.X
	)
	if s.FlipX {
		minx = b.Max.X
		maxx = b.Min.X
	}
	s.texture.Bind()
	gl.MatrixMode(gl.TEXTURE)
	gl.Begin(gl.QUADS)
	gl.TexCoord2d(s.texture1, 0)
	gl.Vertex3d(minx, b.Min.Y, z)
	gl.TexCoord2d(s.texture2, 0)
	gl.Vertex3d(maxx, b.Min.Y, z)
	gl.TexCoord2d(s.texture2, 1)
	gl.Vertex3d(maxx, b.Max.Y, z)
	gl.TexCoord2d(s.texture1, 1)
	gl.Vertex3d(minx, b.Max.Y, z)
	gl.End()
	gl.MatrixMode(gl.MODELVIEW)
	s.texture.Unbind()
	s.Element.Draw()
}

func (s *Sprite) Update() {
	s.MoveTo(Pt(s.X()+s.VelocityX, s.Y()+s.VelocityY))
}
<commit_msg>Allow for sprites which don't use their entire texture's vertical height.<commit_after>// Copyright 2013 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"github.com/go-gl/gl"
)

type Sprite struct {
	Element
	system    *System
	texture   *Texture
	frame     int
	texture1  float64
	texture2  float64
	textureB  float64
	VelocityX float64
	VelocityY float64
	Type      int
	Collide   bool
	FlipX     bool
}

func (s *System) NewSprite(name string, x, y, w, h float64, t int) *Sprite {
	sprite := &Sprite{
		frame:   -1,
		system:  s,
		texture: s.Textures[name],
		textureB: 0.0,
		Type:    t,
		FlipX:   false,
		Collide: true,
	}
	sprite.SetBounds(Rect(x, y, x+w, y+h))
	sprite.SetFrame(0)
	return sprite
}

func (s *Sprite) SetTextureHeight(h float64) {
	s.textureB = 1 - (h / float64(s.texture.Height))
}

func (s *Sprite) SetVelocity(pt Point) {
	s.VelocityX = pt.X
	s.VelocityY = pt.Y
}

func (s *Sprite) Velocity() Point {
	return Pt(s.VelocityX, s.VelocityY)
}

func (s *Sprite) TestMove(dx float64, dy float64, r *Sprite) bool {
	var (
		pad = float64(0.01)
		sb  = s.GlobalBounds()
		rb  = r.GlobalBounds()
		p   = Pt(dx, dy)
	)
	sb.Min.X += pad
	sb.Min.Y += pad
	sb.Max.X -= pad
	sb.Max.Y -= pad
	if sb.Add(p).Overlaps(rb) {
		return false
	}
	return true
}

func (s *Sprite) CollidesWith(sprite *Sprite) bool {
	return !s.TestMove(0, 0, sprite)
}

func (s *Sprite) SetFrame(frame int) {
	if frame == s.frame {
		return
	}
	s.frame = frame % len(s.texture.Frames)
	var (
		tex = s.texture.Frames[s.frame]
	)
	s.texture1 = float64(tex[0]) / float64(s.texture.Width)
	s.texture2 = float64(tex[1]) / float64(s.texture.Width)
}

func (s *Sprite) Draw() {
	var (
		b    = s.GlobalBounds()
		z    = s.Z()
		minx = b.Min.X
		maxx = b.Max.X
	)
	if s.FlipX {
		minx = b.Max.X
		maxx = b.Min.X
	}
	s.texture.Bind()
	gl.MatrixMode(gl.TEXTURE)
	gl.Begin(gl.QUADS)
	gl.TexCoord2d(s.texture1, s.textureB)
	gl.Vertex3d(minx, b.Min.Y, z)
	gl.TexCoord2d(s.texture2, s.textureB)
	gl.Vertex3d(maxx, b.Min.Y, z)
	gl.TexCoord2d(s.texture2, 1)
	gl.Vertex3d(maxx, b.Max.Y, z)
	gl.TexCoord2d(s.texture1, 1)
	gl.Vertex3d(minx, b.Max.Y, z)
	gl.End()
	gl.MatrixMode(gl.MODELVIEW)
	s.texture.Unbind()
	s.Element.Draw()
}

func (s *Sprite) Update() {
	s.MoveTo(Pt(s.X()+s.VelocityX, s.Y()+s.VelocityY))
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"net"
	"reflect"
	"strconv"

	"github.com/juju/errors"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/network"
)

// controllerAddresses returns the list of internal addresses of the state
// server machines.
func (st *State) controllerAddresses() ([]string, error) {
	ssState := st
	model, err := st.ControllerModel()
	if err != nil {
		return nil, errors.Trace(err)
	}
	if st.ModelTag() != model.ModelTag() {
		// We are not using the controller model, so get one.
		logger.Debugf("getting a controller state connection, current env: %s", st.ModelTag())
		ssState, err = st.ForModel(model.ModelTag())
		if err != nil {
			return nil, errors.Trace(err)
		}
		defer ssState.Close()
		logger.Debugf("ssState env: %s", ssState.ModelTag())
	}

	type addressMachine struct {
		Addresses []address
	}
	var allAddresses []addressMachine
	// TODO(rog) 2013/10/14 index machines on jobs.
	machines, closer := ssState.getCollection(machinesC)
	defer closer()
	err = machines.Find(bson.D{{"jobs", JobManageModel}}).All(&allAddresses)
	if err != nil {
		return nil, err
	}
	if len(allAddresses) == 0 {
		return nil, errors.New("no controller machines found")
	}
	apiAddrs := make([]string, 0, len(allAddresses))
	for _, addrs := range allAddresses {
		naddrs := networkAddresses(addrs.Addresses)
		addr, ok := network.SelectControllerAddress(naddrs, false)
		if ok {
			apiAddrs = append(apiAddrs, addr.Value)
		}
	}
	if len(apiAddrs) == 0 {
		return nil, errors.New("no controller machines with addresses found")
	}
	return apiAddrs, nil
}

func appendPort(addrs []string, port int) []string {
	newAddrs := make([]string, len(addrs))
	for i, addr := range addrs {
		newAddrs[i] = net.JoinHostPort(addr, strconv.Itoa(port))
	}
	return newAddrs
}

// Addresses returns the list of cloud-internal addresses that
// can be used to connect to the state.
func (st *State) Addresses() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ControllerConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.StatePort()), nil
}

// APIAddressesFromMachines returns the list of cloud-internal addresses that
// can be used to connect to the state API server.
// This method will be deprecated when API addresses are
// stored independently in their own document.
func (st *State) APIAddressesFromMachines() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ControllerConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.APIPort()), nil
}

const apiHostPortsKey = "apiHostPorts"

type apiHostPortsDoc struct {
	APIHostPorts [][]hostPort `bson:"apihostports"`
}

// SetAPIHostPorts sets the addresses of the API server instances.
// Each server is represented by one element in the top level slice.
func (st *State) SetAPIHostPorts(netHostsPorts [][]network.HostPort) error {
	doc := apiHostPortsDoc{
		APIHostPorts: fromNetworkHostsPorts(netHostsPorts),
	}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		existing, err := st.APIHostPorts()
		if err != nil {
			return nil, err
		}
		op := txn.Op{
			C:  controllersC,
			Id: apiHostPortsKey,
			Assert: bson.D{{
				"apihostports", fromNetworkHostsPorts(existing),
			}},
		}
		if !hostsPortsEqual(netHostsPorts, existing) {
			op.Update = bson.D{{
				"$set", bson.D{{"apihostports", doc.APIHostPorts}},
			}}
		}
		return []txn.Op{op}, nil
	}
	if err := st.run(buildTxn); err != nil {
		return errors.Annotate(err, "cannot set API addresses")
	}
	logger.Debugf("setting API hostPorts: %v", netHostsPorts)
	return nil
}

// APIHostPorts returns the API addresses as set by SetAPIHostPorts.
func (st *State) APIHostPorts() ([][]network.HostPort, error) {
	var doc apiHostPortsDoc
	controllers, closer := st.getCollection(controllersC)
	defer closer()
	err := controllers.Find(bson.D{{"_id", apiHostPortsKey}}).One(&doc)
	if err != nil {
		return nil, err
	}
	return networkHostsPorts(doc.APIHostPorts), nil
}

// address represents the location of a machine, including metadata
// about what kind of location the address describes.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type address struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	Scope       string `bson:"networkscope,omitempty"`
	Origin      string `bson:"origin,omitempty"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// Origin specifies where an address comes from, whether it was reported by a
// provider or by a machine.
type Origin string

const (
	// Address origin unknown.
	OriginUnknown Origin = ""
	// Address comes from a provider.
	OriginProvider Origin = "provider"
	// Address comes from a machine.
	OriginMachine Origin = "machine"
)

// fromNetworkAddress is a convenience helper to create a state type
// out of the network type, here for Address with a given Origin.
func fromNetworkAddress(netAddr network.Address, origin Origin) address {
	return address{
		Value:       netAddr.Value,
		AddressType: string(netAddr.Type),
		Scope:       string(netAddr.Scope),
		Origin:      string(origin),
		SpaceName:   string(netAddr.SpaceName),
	}
}

// networkAddress is a convenience helper to return the state type
// as network type, here for Address.
func (addr *address) networkAddress() network.Address {
	return network.Address{
		Value:     addr.Value,
		Type:      network.AddressType(addr.AddressType),
		Scope:     network.Scope(addr.Scope),
		SpaceName: network.SpaceName(addr.SpaceName),
	}
}

// fromNetworkAddresses is a convenience helper to create a state type
// out of the network type, here for a slice of Address with a given origin.
func fromNetworkAddresses(netAddrs []network.Address, origin Origin) []address {
	addrs := make([]address, len(netAddrs))
	for i, netAddr := range netAddrs {
		addrs[i] = fromNetworkAddress(netAddr, origin)
	}
	return addrs
}

// networkAddresses is a convenience helper to return the state type
// as network type, here for a slice of Address.
func networkAddresses(addrs []address) []network.Address {
	netAddrs := make([]network.Address, len(addrs))
	for i, addr := range addrs {
		netAddrs[i] = addr.networkAddress()
	}
	return netAddrs
}

// hostPort associates an address with a port. See also network.HostPort,
// from/to which this is transformed.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type hostPort struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	Scope       string `bson:"networkscope,omitempty"`
	Port        int    `bson:"port"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// fromNetworkHostPort is a convenience helper to create a state type
// out of the network type, here for HostPort.
func fromNetworkHostPort(netHostPort network.HostPort) hostPort {
	return hostPort{
		Value:       netHostPort.Value,
		AddressType: string(netHostPort.Type),
		Scope:       string(netHostPort.Scope),
		Port:        netHostPort.Port,
		SpaceName:   string(netHostPort.SpaceName),
	}
}

// networkHostPort is a convenience helper to return the state type
// as network type, here for HostPort.
func (hp *hostPort) networkHostPort() network.HostPort {
	return network.HostPort{
		Address: network.Address{
			Value:     hp.Value,
			Type:      network.AddressType(hp.AddressType),
			Scope:     network.Scope(hp.Scope),
			SpaceName: network.SpaceName(hp.SpaceName),
		},
		Port: hp.Port,
	}
}

// fromNetworkHostsPorts is a helper to create a state type
// out of the network type, here for a nested slice of HostPort.
func fromNetworkHostsPorts(netHostsPorts [][]network.HostPort) [][]hostPort {
	hsps := make([][]hostPort, len(netHostsPorts))
	for i, netHostPorts := range netHostsPorts {
		hsps[i] = make([]hostPort, len(netHostPorts))
		for j, netHostPort := range netHostPorts {
			hsps[i][j] = fromNetworkHostPort(netHostPort)
		}
	}
	return hsps
}

// networkHostsPorts is a convenience helper to return the state type
// as network type, here for a nested slice of HostPort.
func networkHostsPorts(hsps [][]hostPort) [][]network.HostPort {
	netHostsPorts := make([][]network.HostPort, len(hsps))
	for i, hps := range hsps {
		netHostsPorts[i] = make([]network.HostPort, len(hps))
		for j, hp := range hps {
			netHostsPorts[i][j] = hp.networkHostPort()
		}
	}
	return netHostsPorts
}

// addressEqual checks that two slices of network addresses are equal.
func addressesEqual(a, b []network.Address) bool {
	return reflect.DeepEqual(a, b)
}

// hostsPortsEqual checks that two arrays of network hostports are equal.
func hostsPortsEqual(a, b [][]network.HostPort) bool {
	return reflect.DeepEqual(a, b)
}
<commit_msg>Use txn-revno to assert that APIHostPorts are unchanged<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"net"
	"reflect"
	"strconv"

	"github.com/juju/errors"
	statetxn "github.com/juju/txn"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/network"
)

// controllerAddresses returns the list of internal addresses of the state
// server machines.
func (st *State) controllerAddresses() ([]string, error) {
	ssState := st
	model, err := st.ControllerModel()
	if err != nil {
		return nil, errors.Trace(err)
	}
	if st.ModelTag() != model.ModelTag() {
		// We are not using the controller model, so get one.
		logger.Debugf("getting a controller state connection, current env: %s", st.ModelTag())
		ssState, err = st.ForModel(model.ModelTag())
		if err != nil {
			return nil, errors.Trace(err)
		}
		defer ssState.Close()
		logger.Debugf("ssState env: %s", ssState.ModelTag())
	}

	type addressMachine struct {
		Addresses []address
	}
	var allAddresses []addressMachine
	// TODO(rog) 2013/10/14 index machines on jobs.
	machines, closer := ssState.getCollection(machinesC)
	defer closer()
	err = machines.Find(bson.D{{"jobs", JobManageModel}}).All(&allAddresses)
	if err != nil {
		return nil, err
	}
	if len(allAddresses) == 0 {
		return nil, errors.New("no controller machines found")
	}
	apiAddrs := make([]string, 0, len(allAddresses))
	for _, addrs := range allAddresses {
		naddrs := networkAddresses(addrs.Addresses)
		addr, ok := network.SelectControllerAddress(naddrs, false)
		if ok {
			apiAddrs = append(apiAddrs, addr.Value)
		}
	}
	if len(apiAddrs) == 0 {
		return nil, errors.New("no controller machines with addresses found")
	}
	return apiAddrs, nil
}

func appendPort(addrs []string, port int) []string {
	newAddrs := make([]string, len(addrs))
	for i, addr := range addrs {
		newAddrs[i] = net.JoinHostPort(addr, strconv.Itoa(port))
	}
	return newAddrs
}

// Addresses returns the list of cloud-internal addresses that
// can be used to connect to the state.
func (st *State) Addresses() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ControllerConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.StatePort()), nil
}

// APIAddressesFromMachines returns the list of cloud-internal addresses that
// can be used to connect to the state API server.
// This method will be deprecated when API addresses are
// stored independently in their own document.
func (st *State) APIAddressesFromMachines() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ControllerConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.APIPort()), nil
}

const apiHostPortsKey = "apiHostPorts"

type apiHostPortsDoc struct {
	APIHostPorts [][]hostPort `bson:"apihostports"`
	TxnRevno     int64        `bson:"txn-revno"`
}

// SetAPIHostPorts sets the addresses of the API server instances.
// Each server is represented by one element in the top level slice.
func (st *State) SetAPIHostPorts(netHostsPorts [][]network.HostPort) error {
	controllers, closer := st.getCollection(controllersC)
	defer closer()
	doc := apiHostPortsDoc{
		APIHostPorts: fromNetworkHostsPorts(netHostsPorts),
	}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		var existingDoc apiHostPortsDoc
		err := controllers.Find(bson.D{{"_id", apiHostPortsKey}}).One(&existingDoc)
		if err != nil {
			return nil, err
		}
		op := txn.Op{
			C:  controllersC,
			Id: apiHostPortsKey,
			Assert: bson.D{{
				"txn-revno", existingDoc.TxnRevno,
			}},
		}
		hostPorts := networkHostsPorts(existingDoc.APIHostPorts)
		if !hostsPortsEqual(netHostsPorts, hostPorts) {
			op.Update = bson.D{{
				"$set", bson.D{{"apihostports", doc.APIHostPorts}},
			}}
		} else {
			return nil, statetxn.ErrNoOperations
		}
		return []txn.Op{op}, nil
	}
	if err := st.run(buildTxn); err != nil {
		return errors.Annotate(err, "cannot set API addresses")
	}
	logger.Debugf("setting API hostPorts: %v", netHostsPorts)
	return nil
}

// APIHostPorts returns the API addresses as set by SetAPIHostPorts.
func (st *State) APIHostPorts() ([][]network.HostPort, error) {
	var doc apiHostPortsDoc
	controllers, closer := st.getCollection(controllersC)
	defer closer()
	err := controllers.Find(bson.D{{"_id", apiHostPortsKey}}).One(&doc)
	if err != nil {
		return nil, err
	}
	return networkHostsPorts(doc.APIHostPorts), nil
}

// address represents the location of a machine, including metadata
// about what kind of location the address describes.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type address struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	Scope       string `bson:"networkscope,omitempty"`
	Origin      string `bson:"origin,omitempty"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// Origin specifies where an address comes from, whether it was reported by a
// provider or by a machine.
type Origin string

const (
	// Address origin unknown.
	OriginUnknown Origin = ""
	// Address comes from a provider.
	OriginProvider Origin = "provider"
	// Address comes from a machine.
	OriginMachine Origin = "machine"
)

// fromNetworkAddress is a convenience helper to create a state type
// out of the network type, here for Address with a given Origin.
func fromNetworkAddress(netAddr network.Address, origin Origin) address {
	return address{
		Value:       netAddr.Value,
		AddressType: string(netAddr.Type),
		Scope:       string(netAddr.Scope),
		Origin:      string(origin),
		SpaceName:   string(netAddr.SpaceName),
	}
}

// networkAddress is a convenience helper to return the state type
// as network type, here for Address.
func (addr *address) networkAddress() network.Address {
	return network.Address{
		Value:     addr.Value,
		Type:      network.AddressType(addr.AddressType),
		Scope:     network.Scope(addr.Scope),
		SpaceName: network.SpaceName(addr.SpaceName),
	}
}

// fromNetworkAddresses is a convenience helper to create a state type
// out of the network type, here for a slice of Address with a given origin.
func fromNetworkAddresses(netAddrs []network.Address, origin Origin) []address {
	addrs := make([]address, len(netAddrs))
	for i, netAddr := range netAddrs {
		addrs[i] = fromNetworkAddress(netAddr, origin)
	}
	return addrs
}

// networkAddresses is a convenience helper to return the state type
// as network type, here for a slice of Address.
func networkAddresses(addrs []address) []network.Address {
	netAddrs := make([]network.Address, len(addrs))
	for i, addr := range addrs {
		netAddrs[i] = addr.networkAddress()
	}
	return netAddrs
}

// hostPort associates an address with a port. See also network.HostPort,
// from/to which this is transformed.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type hostPort struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	Scope       string `bson:"networkscope,omitempty"`
	Port        int    `bson:"port"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// fromNetworkHostPort is a convenience helper to create a state type
// out of the network type, here for HostPort.
func fromNetworkHostPort(netHostPort network.HostPort) hostPort {
	return hostPort{
		Value:       netHostPort.Value,
		AddressType: string(netHostPort.Type),
		Scope:       string(netHostPort.Scope),
		Port:        netHostPort.Port,
		SpaceName:   string(netHostPort.SpaceName),
	}
}

// networkHostPort is a convenience helper to return the state type
// as network type, here for HostPort.
func (hp *hostPort) networkHostPort() network.HostPort {
	return network.HostPort{
		Address: network.Address{
			Value:     hp.Value,
			Type:      network.AddressType(hp.AddressType),
			Scope:     network.Scope(hp.Scope),
			SpaceName: network.SpaceName(hp.SpaceName),
		},
		Port: hp.Port,
	}
}

// fromNetworkHostsPorts is a helper to create a state type
// out of the network type, here for a nested slice of HostPort.
func fromNetworkHostsPorts(netHostsPorts [][]network.HostPort) [][]hostPort {
	hsps := make([][]hostPort, len(netHostsPorts))
	for i, netHostPorts := range netHostsPorts {
		hsps[i] = make([]hostPort, len(netHostPorts))
		for j, netHostPort := range netHostPorts {
			hsps[i][j] = fromNetworkHostPort(netHostPort)
		}
	}
	return hsps
}

// networkHostsPorts is a convenience helper to return the state type
// as network type, here for a nested slice of HostPort.
func networkHostsPorts(hsps [][]hostPort) [][]network.HostPort {
	netHostsPorts := make([][]network.HostPort, len(hsps))
	for i, hps := range hsps {
		netHostsPorts[i] = make([]network.HostPort, len(hps))
		for j, hp := range hps {
			netHostsPorts[i][j] = hp.networkHostPort()
		}
	}
	return netHostsPorts
}

// addressEqual checks that two slices of network addresses are equal.
func addressesEqual(a, b []network.Address) bool {
	return reflect.DeepEqual(a, b)
}

// hostsPortsEqual checks that two arrays of network hostports are equal.
func hostsPortsEqual(a, b [][]network.HostPort) bool {
	return reflect.DeepEqual(a, b)
}
<|endoftext|>