<commit_before>package spider

import (
	"math"
	"sync"
	"time"

	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/app/scheduler"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/status"
)

const (
	KEYIN       = util.USE_KEYIN // 若使用Spider.Keyin，则须在规则中设置初始值为USE_KEYIN
	LIMIT       = math.MaxInt64  // 如希望在规则中自定义控制Limit，则Limit初始值必须为LIMIT
	ACTIVE_STOP = "——主动终止Spider——"
)

type (
	// 蜘蛛规则
	Spider struct {
		// 以下字段由用户定义
		Name            string                                                     // 用户界面显示的名称（应保证唯一性）
		Description     string                                                     // 用户界面显示的描述
		Pausetime       int64                                                      // 随机暂停区间(50%~200%)，若规则中直接定义，则不被界面传参覆盖
		Limit           int64                                                      // 默认限制请求数，0为不限；若规则中定义为LIMIT，则采用规则的自定义限制方案
		Keyin           string                                                     // 自定义输入的配置信息，使用前须在规则中设置初始值为KEYIN
		EnableCookie    bool                                                       // 所有请求是否使用cookie记录
		NotDefaultField bool                                                       // 是否禁止输出结果中的默认字段 Url/ParentUrl/DownloadTime
		Namespace       func(self *Spider) string                                  // 命名空间，用于输出文件、路径的命名
		SubNamespace    func(self *Spider, dataCell map[string]interface{}) string // 次级命名，用于输出文件、路径的命名，可依赖具体数据内容
		RuleTree        *RuleTree                                                  // 定义具体的采集规则树

		// 以下字段系统自动赋值
		id        int               // 自动分配的SpiderQueue中的索引
		subName   string            // 由Keyin转换为的二级标识名
		reqMatrix *scheduler.Matrix // 请求矩阵
		timer     *Timer            // 定时器
		status    int               // 执行状态
		lock      sync.RWMutex
		once      sync.Once
	}
	//采集规则树
	RuleTree struct {
		Root  func(*Context)   // 根节点(执行入口)
		Trunk map[string]*Rule // 节点散列表(执行采集过程)
	}
	// 采集规则节点
	Rule struct {
		ItemFields []string                                           // 结果字段列表(选填，写上可保证字段顺序)
		ParseFunc  func(*Context)                                     // 内容解析函数
		AidFunc    func(*Context, map[string]interface{}) interface{} // 通用辅助函数
	}
)

// 添加自身到蜘蛛菜单
func (self Spider) Register() *Spider {
	self.status = status.STOPPED
	return Species.Add(&self)
}

// 指定规则的获取结果的字段名列表
func (self *Spider) GetItemFields(rule *Rule) []string {
	return rule.ItemFields
}

// 返回结果字段名的值
// 不存在时返回空字符串
func (self *Spider) GetItemField(rule *Rule, index int) (field string) {
	if index > len(rule.ItemFields)-1 || index < 0 {
		return ""
	}
	return rule.ItemFields[index]
}

// 返回结果字段名的其索引
// 不存在时索引为-1
func (self *Spider) GetItemFieldIndex(rule *Rule, field string) (index int) {
	for idx, v := range rule.ItemFields {
		if v == field {
			return idx
		}
	}
	return -1
}

// 为指定Rule动态追加结果字段名，并返回索引位置
// 已存在时返回原来索引位置
func (self *Spider) UpsertItemField(rule *Rule, field string) (index int) {
	for i, v := range rule.ItemFields {
		if v == field {
			return i
		}
	}
	rule.ItemFields = append(rule.ItemFields, field)
	return len(rule.ItemFields) - 1
}

// 获取蜘蛛名称
func (self *Spider) GetName() string {
	return self.Name
}

// 获取蜘蛛二级标识名
func (self *Spider) GetSubName() string {
	self.once.Do(func() {
		self.subName = self.GetKeyin()
		if len([]rune(self.subName)) > 6 {
			self.subName = util.MakeHash(self.subName)
		}
	})
	return self.subName
}

// 安全返回指定规则
func (self *Spider) GetRule(ruleName string) (*Rule, bool) {
	rule, found := self.RuleTree.Trunk[ruleName]
	return rule, found
}

// 返回指定规则
func (self *Spider) MustGetRule(ruleName string) *Rule {
	return self.RuleTree.Trunk[ruleName]
}

// 返回规则树
func (self *Spider) GetRules() map[string]*Rule {
	return self.RuleTree.Trunk
}

// 获取蜘蛛描述
func (self *Spider) GetDescription() string {
	return self.Description
}

// 获取蜘蛛ID
func (self *Spider) GetId() int {
	return self.id
}

// 设置蜘蛛ID
func (self *Spider) SetId(id int) {
	self.id = id
}

// 获取自定义配置信息
func (self *Spider) GetKeyin() string {
	return self.Keyin
}

// 设置自定义配置信息
func (self *Spider) SetKeyin(keyword string) {
	self.Keyin = keyword
}

// 获取采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) GetLimit() int64 {
	return self.Limit
}

// 设置采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) SetLimit(max int64) {
	self.Limit = max
}

// 控制所有请求是否使用cookie
func (self *Spider) GetEnableCookie() bool {
	return self.EnableCookie
}

// 自定义暂停时间 pause[0]~(pause[0]+pause[1])，优先级高于外部传参
// 当且仅当runtime[0]为true时可覆盖现有值
func (self *Spider) SetPausetime(pause int64, runtime ...bool) {
	if self.Pausetime == 0 || len(runtime) > 0 && runtime[0] {
		self.Pausetime = pause
	}
}

// 设置定时器
// @id为定时器唯一标识
// @bell==nil时为倒计时器，此时@tol为睡眠时长
// @bell!=nil时为闹铃，此时@tol用于指定醒来时刻（从now起遇到的第tol个bell）
func (self *Spider) SetTimer(id string, tol time.Duration, bell *Bell) bool {
	if self.timer == nil {
		self.timer = newTimer()
	}
	return self.timer.set(id, tol, bell)
}

// 启动定时器，并返回定时器是否可以继续使用
func (self *Spider) RunTimer(id string) bool {
	if self.timer == nil {
		return false
	}
	return self.timer.sleep(id)
}

// 返回一个自身复制品
func (self *Spider) Copy() *Spider {
	ghost := &Spider{}
	ghost.Name = self.Name
	ghost.subName = self.subName

	ghost.RuleTree = &RuleTree{
		Root:  self.RuleTree.Root,
		Trunk: make(map[string]*Rule, len(self.RuleTree.Trunk)),
	}
	for k, v := range self.RuleTree.Trunk {
		ghost.RuleTree.Trunk[k] = new(Rule)

		ghost.RuleTree.Trunk[k].ItemFields = make([]string, len(v.ItemFields))
		copy(ghost.RuleTree.Trunk[k].ItemFields, v.ItemFields)

		ghost.RuleTree.Trunk[k].ParseFunc = v.ParseFunc
		ghost.RuleTree.Trunk[k].AidFunc = v.AidFunc
	}

	ghost.Description = self.Description
	ghost.Pausetime = self.Pausetime
	ghost.EnableCookie = self.EnableCookie
	ghost.Limit = self.Limit
	ghost.Keyin = self.Keyin

	ghost.NotDefaultField = self.NotDefaultField
	ghost.Namespace = self.Namespace
	ghost.SubNamespace = self.SubNamespace

	ghost.timer = self.timer
	ghost.status = self.status

	return ghost
}

func (self *Spider) ReqmatrixInit() *Spider {
	if self.Limit < 0 {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), self.Limit)
		self.SetLimit(0)
	} else {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), math.MinInt64)
	}
	return self
}

// 返回是否作为新的失败请求被添加至队列尾部
func (self *Spider) DoHistory(req *request.Request, ok bool) bool {
	return self.reqMatrix.DoHistory(req, ok)
}

func (self *Spider) RequestPush(req *request.Request) {
	self.reqMatrix.Push(req)
}

func (self *Spider) RequestPull() *request.Request {
	return self.reqMatrix.Pull()
}

func (self *Spider) RequestUse() {
	self.reqMatrix.Use()
}

func (self *Spider) RequestFree() {
	self.reqMatrix.Free()
}

func (self *Spider) RequestLen() int {
	return self.reqMatrix.Len()
}

func (self *Spider) TryFlushSuccess() {
	self.reqMatrix.TryFlushSuccess()
}

func (self *Spider) TryFlushFailure() {
	self.reqMatrix.TryFlushFailure()
}

// 开始执行蜘蛛
func (self *Spider) Start() {
	defer func() {
		if p := recover(); p != nil {
			logs.Log.Error(" *     Panic  [root]: %v\n", p)
		}
		self.lock.Lock()
		self.status = status.RUN
		self.lock.Unlock()
	}()
	self.RuleTree.Root(GetContext(self, nil))
}

// 主动崩溃爬虫运行协程
func (self *Spider) Stop() {
	self.lock.Lock()
	self.status = status.STOP
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	self.lock.Unlock()
}

func (self *Spider) CanStop() bool {
	return self.status != status.STOPPED && self.reqMatrix.CanStop()
}

// 若已主动终止任务，则崩溃爬虫协程
func (self *Spider) tryPanic() {
	self.lock.RLock()
	if self.status == status.STOP {
		self.lock.RUnlock()
		panic(ACTIVE_STOP)
	}
	self.lock.RUnlock()
}

// 退出任务前收尾工作
func (self *Spider) Defer() {
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	// 等待处理中的请求完成
	self.reqMatrix.Wait()
	// 更新失败记录
	self.reqMatrix.TryFlushFailure()
}

// 是否输出默认添加的字段 Url/ParentUrl/DownloadTime
func (self *Spider) OutDefaultField() bool {
	return !self.NotDefaultField
}
<commit_msg>调整文件名哈希临界长度为>8<commit_after>package spider

import (
	"math"
	"sync"
	"time"

	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/app/scheduler"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/status"
)

const (
	KEYIN       = util.USE_KEYIN // 若使用Spider.Keyin，则须在规则中设置初始值为USE_KEYIN
	LIMIT       = math.MaxInt64  // 如希望在规则中自定义控制Limit，则Limit初始值必须为LIMIT
	ACTIVE_STOP = "——主动终止Spider——"
)

type (
	// 蜘蛛规则
	Spider struct {
		// 以下字段由用户定义
		Name            string                                                     // 用户界面显示的名称（应保证唯一性）
		Description     string                                                     // 用户界面显示的描述
		Pausetime       int64                                                      // 随机暂停区间(50%~200%)，若规则中直接定义，则不被界面传参覆盖
		Limit           int64                                                      // 默认限制请求数，0为不限；若规则中定义为LIMIT，则采用规则的自定义限制方案
		Keyin           string                                                     // 自定义输入的配置信息，使用前须在规则中设置初始值为KEYIN
		EnableCookie    bool                                                       // 所有请求是否使用cookie记录
		NotDefaultField bool                                                       // 是否禁止输出结果中的默认字段 Url/ParentUrl/DownloadTime
		Namespace       func(self *Spider) string                                  // 命名空间，用于输出文件、路径的命名
		SubNamespace    func(self *Spider, dataCell map[string]interface{}) string // 次级命名，用于输出文件、路径的命名，可依赖具体数据内容
		RuleTree        *RuleTree                                                  // 定义具体的采集规则树

		// 以下字段系统自动赋值
		id        int               // 自动分配的SpiderQueue中的索引
		subName   string            // 由Keyin转换为的二级标识名
		reqMatrix *scheduler.Matrix // 请求矩阵
		timer     *Timer            // 定时器
		status    int               // 执行状态
		lock      sync.RWMutex
		once      sync.Once
	}
	//采集规则树
	RuleTree struct {
		Root  func(*Context)   // 根节点(执行入口)
		Trunk map[string]*Rule // 节点散列表(执行采集过程)
	}
	// 采集规则节点
	Rule struct {
		ItemFields []string                                           // 结果字段列表(选填，写上可保证字段顺序)
		ParseFunc  func(*Context)                                     // 内容解析函数
		AidFunc    func(*Context, map[string]interface{}) interface{} // 通用辅助函数
	}
)

// 添加自身到蜘蛛菜单
func (self Spider) Register() *Spider {
	self.status = status.STOPPED
	return Species.Add(&self)
}

// 指定规则的获取结果的字段名列表
func (self *Spider) GetItemFields(rule *Rule) []string {
	return rule.ItemFields
}

// 返回结果字段名的值
// 不存在时返回空字符串
func (self *Spider) GetItemField(rule *Rule, index int) (field string) {
	if index > len(rule.ItemFields)-1 || index < 0 {
		return ""
	}
	return rule.ItemFields[index]
}

// 返回结果字段名的其索引
// 不存在时索引为-1
func (self *Spider) GetItemFieldIndex(rule *Rule, field string) (index int) {
	for idx, v := range rule.ItemFields {
		if v == field {
			return idx
		}
	}
	return -1
}

// 为指定Rule动态追加结果字段名，并返回索引位置
// 已存在时返回原来索引位置
func (self *Spider) UpsertItemField(rule *Rule, field string) (index int) {
	for i, v := range rule.ItemFields {
		if v == field {
			return i
		}
	}
	rule.ItemFields = append(rule.ItemFields, field)
	return len(rule.ItemFields) - 1
}

// 获取蜘蛛名称
func (self *Spider) GetName() string {
	return self.Name
}

// 获取蜘蛛二级标识名
func (self *Spider) GetSubName() string {
	self.once.Do(func() {
		self.subName = self.GetKeyin()
		if len([]rune(self.subName)) > 8 {
			self.subName = util.MakeHash(self.subName)
		}
	})
	return self.subName
}

// 安全返回指定规则
func (self *Spider) GetRule(ruleName string) (*Rule, bool) {
	rule, found := self.RuleTree.Trunk[ruleName]
	return rule, found
}

// 返回指定规则
func (self *Spider) MustGetRule(ruleName string) *Rule {
	return self.RuleTree.Trunk[ruleName]
}

// 返回规则树
func (self *Spider) GetRules() map[string]*Rule {
	return self.RuleTree.Trunk
}

// 获取蜘蛛描述
func (self *Spider) GetDescription() string {
	return self.Description
}

// 获取蜘蛛ID
func (self *Spider) GetId() int {
	return self.id
}

// 设置蜘蛛ID
func (self *Spider) SetId(id int) {
	self.id = id
}

// 获取自定义配置信息
func (self *Spider) GetKeyin() string {
	return self.Keyin
}

// 设置自定义配置信息
func (self *Spider) SetKeyin(keyword string) {
	self.Keyin = keyword
}

// 获取采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) GetLimit() int64 {
	return self.Limit
}

// 设置采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) SetLimit(max int64) {
	self.Limit = max
}

// 控制所有请求是否使用cookie
func (self *Spider) GetEnableCookie() bool {
	return self.EnableCookie
}

// 自定义暂停时间 pause[0]~(pause[0]+pause[1])，优先级高于外部传参
// 当且仅当runtime[0]为true时可覆盖现有值
func (self *Spider) SetPausetime(pause int64, runtime ...bool) {
	if self.Pausetime == 0 || len(runtime) > 0 && runtime[0] {
		self.Pausetime = pause
	}
}

// 设置定时器
// @id为定时器唯一标识
// @bell==nil时为倒计时器，此时@tol为睡眠时长
// @bell!=nil时为闹铃，此时@tol用于指定醒来时刻（从now起遇到的第tol个bell）
func (self *Spider) SetTimer(id string, tol time.Duration, bell *Bell) bool {
	if self.timer == nil {
		self.timer = newTimer()
	}
	return self.timer.set(id, tol, bell)
}

// 启动定时器，并返回定时器是否可以继续使用
func (self *Spider) RunTimer(id string) bool {
	if self.timer == nil {
		return false
	}
	return self.timer.sleep(id)
}

// 返回一个自身复制品
func (self *Spider) Copy() *Spider {
	ghost := &Spider{}
	ghost.Name = self.Name
	ghost.subName = self.subName

	ghost.RuleTree = &RuleTree{
		Root:  self.RuleTree.Root,
		Trunk: make(map[string]*Rule, len(self.RuleTree.Trunk)),
	}
	for k, v := range self.RuleTree.Trunk {
		ghost.RuleTree.Trunk[k] = new(Rule)

		ghost.RuleTree.Trunk[k].ItemFields = make([]string, len(v.ItemFields))
		copy(ghost.RuleTree.Trunk[k].ItemFields, v.ItemFields)

		ghost.RuleTree.Trunk[k].ParseFunc = v.ParseFunc
		ghost.RuleTree.Trunk[k].AidFunc = v.AidFunc
	}

	ghost.Description = self.Description
	ghost.Pausetime = self.Pausetime
	ghost.EnableCookie = self.EnableCookie
	ghost.Limit = self.Limit
	ghost.Keyin = self.Keyin

	ghost.NotDefaultField = self.NotDefaultField
	ghost.Namespace = self.Namespace
	ghost.SubNamespace = self.SubNamespace

	ghost.timer = self.timer
	ghost.status = self.status

	return ghost
}

func (self *Spider) ReqmatrixInit() *Spider {
	if self.Limit < 0 {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), self.Limit)
		self.SetLimit(0)
	} else {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), math.MinInt64)
	}
	return self
}

// 返回是否作为新的失败请求被添加至队列尾部
func (self *Spider) DoHistory(req *request.Request, ok bool) bool {
	return self.reqMatrix.DoHistory(req, ok)
}

func (self *Spider) RequestPush(req *request.Request) {
	self.reqMatrix.Push(req)
}

func (self *Spider) RequestPull() *request.Request {
	return self.reqMatrix.Pull()
}

func (self *Spider) RequestUse() {
	self.reqMatrix.Use()
}

func (self *Spider) RequestFree() {
	self.reqMatrix.Free()
}

func (self *Spider) RequestLen() int {
	return self.reqMatrix.Len()
}

func (self *Spider) TryFlushSuccess() {
	self.reqMatrix.TryFlushSuccess()
}

func (self *Spider) TryFlushFailure() {
	self.reqMatrix.TryFlushFailure()
}

// 开始执行蜘蛛
func (self *Spider) Start() {
	defer func() {
		if p := recover(); p != nil {
			logs.Log.Error(" *     Panic  [root]: %v\n", p)
		}
		self.lock.Lock()
		self.status = status.RUN
		self.lock.Unlock()
	}()
	self.RuleTree.Root(GetContext(self, nil))
}

// 主动崩溃爬虫运行协程
func (self *Spider) Stop() {
	self.lock.Lock()
	self.status = status.STOP
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	self.lock.Unlock()
}

func (self *Spider) CanStop() bool {
	return self.status != status.STOPPED && self.reqMatrix.CanStop()
}

// 若已主动终止任务，则崩溃爬虫协程
func (self *Spider) tryPanic() {
	self.lock.RLock()
	if self.status == status.STOP {
		self.lock.RUnlock()
		panic(ACTIVE_STOP)
	}
	self.lock.RUnlock()
}

// 退出任务前收尾工作
func (self *Spider) Defer() {
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	// 等待处理中的请求完成
	self.reqMatrix.Wait()
	// 更新失败记录
	self.reqMatrix.TryFlushFailure()
}

// 是否输出默认添加的字段 Url/ParentUrl/DownloadTime
func (self *Spider) OutDefaultField() bool {
	return !self.NotDefaultField
}
<|endoftext|>


<commit_before>package drivers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"strings"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

var cephAllowedFilesystems = []string{"btrfs", "ext4", "xfs"}
var cephVersion string
var cephLoaded bool

type ceph struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *ceph) load() error {
	// Register the patches.
	d.patches = map[string]func() error{
		"storage_lvm_skipactivation":       nil,
		"storage_missing_snapshot_records": nil,
	}

	// Done if previously loaded.
	if cephLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"ceph", "rbd"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if cephVersion == "" {
		out, err := shared.RunCommand("rbd", "--version")
		if err != nil {
			return err
		}

		out = strings.TrimSpace(out)

		fields := strings.Split(out, " ")
		if strings.HasPrefix(out, "ceph version ") && len(fields) > 2 {
			cephVersion = fields[2]
		} else {
			cephVersion = out
		}
	}

	cephLoaded = true
	return nil
}

// isRemote returns true indicating this driver uses remote storage.
func (d *ceph) isRemote() bool {
	return true
}

// Info returns info about the driver and its environment.
func (d *ceph) Info() Info {
	return Info{
		Name:              "ceph",
		Version:           cephVersion,
		OptimizedImages:   true,
		PreservesInodes:   false,
		Remote:            d.isRemote(),
		VolumeTypes:       []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:      true,
		RunningCopyFreeze: true,
		DirectIO:          true,
		MountedRoot:       false,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *ceph) Create() error {
	revert := revert.New()
	defer revert.Fail()

	d.config["volatile.initial_source"] = d.config["source"]

	// Set default properties if missing.
	if d.config["ceph.cluster_name"] == "" {
		d.config["ceph.cluster_name"] = CephDefaultCluster
	}

	if d.config["ceph.user.name"] == "" {
		d.config["ceph.user.name"] = CephDefaultUser
	}

	if d.config["ceph.osd.pg_num"] == "" {
		d.config["ceph.osd.pg_num"] = "32"
	} else {
		// Validate.
		_, err := units.ParseByteSizeString(d.config["ceph.osd.pg_num"])
		if err != nil {
			return err
		}
	}

	// Quick check.
	if d.config["source"] != "" && d.config["ceph.osd.pool_name"] != "" && d.config["source"] != d.config["ceph.osd.pool_name"] {
		return fmt.Errorf(`The "source" and "ceph.osd.pool_name" property must not differ for Ceph OSD storage pools`)
	}

	// Use an existing OSD pool.
	if d.config["source"] != "" {
		d.config["ceph.osd.pool_name"] = d.config["source"]
	}

	if d.config["ceph.osd.pool_name"] == "" {
		d.config["ceph.osd.pool_name"] = d.name
		d.config["source"] = d.name
	}

	placeholderVol := NewVolume(d, d.name, VolumeType("lxd"), ContentTypeFS, d.config["ceph.osd.pool_name"], nil, nil)

	if !d.osdPoolExists() {
		// Create new osd pool.
		_, err := shared.TryRunCommand("ceph",
			"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
			"--cluster", d.config["ceph.cluster_name"],
			"osd",
			"pool",
			"create",
			d.config["ceph.osd.pool_name"],
			d.config["ceph.osd.pg_num"])
		if err != nil {
			return err
		}

		revert.Add(func() { _ = d.osdDeletePool() })

		// Initialize the pool. This is not necessary but allows the pool to be monitored.
		_, err = shared.TryRunCommand("rbd",
			"--id", d.config["ceph.user.name"],
			"--cluster", d.config["ceph.cluster_name"],
			"pool",
			"init",
			d.config["ceph.osd.pool_name"])
		if err != nil {
			d.logger.Warn("Failed to initialize pool", logger.Ctx{"pool": d.config["ceph.osd.pool_name"], "cluster": d.config["ceph.cluster_name"]})
		}

		// Create placeholder storage volume. Other LXD instances will use this to detect whether this osd
		// pool is already in use by another LXD instance.
		err = d.rbdCreateVolume(placeholderVol, "0")
		if err != nil {
			return err
		}

		d.config["volatile.pool.pristine"] = "true"
	} else {
		ok := d.HasVolume(placeholderVol)
		if ok {
			if shared.IsFalseOrEmpty(d.config["ceph.osd.force_reuse"]) {
				return fmt.Errorf("Pool '%s' in cluster '%s' seems to be in use by another LXD instance. Use 'ceph.osd.force_reuse=true' to force", d.config["ceph.osd.pool_name"], d.config["ceph.cluster_name"])
			}

			d.config["volatile.pool.pristine"] = "false"
		} else {
			// Create placeholder storage volume. Other LXD instances will use this to detect whether this osd
			// pool is already in use by another LXD instance.
			err := d.rbdCreateVolume(placeholderVol, "0")
			if err != nil {
				return err
			}

			d.config["volatile.pool.pristine"] = "true"
		}

		// Use existing OSD pool.
		msg, err := shared.RunCommand("ceph",
			"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
			"--cluster", d.config["ceph.cluster_name"],
			"osd",
			"pool",
			"get",
			d.config["ceph.osd.pool_name"],
			"pg_num")
		if err != nil {
			return err
		}

		idx := strings.Index(msg, "pg_num:")
		if idx == -1 {
			return fmt.Errorf("Failed to parse number of placement groups for pool: %s", msg)
		}

		msg = msg[(idx + len("pg_num:")):]
		msg = strings.TrimSpace(msg)

		// It is ok to update the pool configuration since storage pool
		// creation via API is implemented such that the storage pool is
		// checked for a changed config after this function returns and
		// if so the db for it is updated.
		d.config["ceph.osd.pg_num"] = msg
	}

	revert.Success()

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *ceph) Delete(op *operations.Operation) error {
	// Test if the pool exists.
	poolExists := d.osdPoolExists()
	if !poolExists {
		d.logger.Warn("Pool does not exist", logger.Ctx{"pool": d.config["ceph.osd.pool_name"], "cluster": d.config["ceph.cluster_name"]})
	}

	// Check whether we own the pool and only remove in this case.
	if shared.IsTrue(d.config["volatile.pool.pristine"]) {
		// Delete the osd pool.
		if poolExists {
			err := d.osdDeletePool()
			if err != nil {
				return err
			}
		}
	}

	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// On delete, wipe everything in the directory.
	err := wipeDirectory(GetPoolMountPath(d.name))
	if err != nil {
		return err
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *ceph) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"ceph.cluster_name":          validate.IsAny,
		"ceph.osd.force_reuse":       validate.Optional(validate.IsBool),
		"ceph.osd.pg_num":            validate.IsAny,
		"ceph.osd.pool_name":         validate.IsAny,
		"ceph.osd.data_pool_name":    validate.IsAny,
		"ceph.rbd.clone_copy":        validate.Optional(validate.IsBool),
		"ceph.rbd.du":                validate.Optional(validate.IsBool),
		"ceph.rbd.features":          validate.IsAny,
		"ceph.user.name":             validate.IsAny,
		"volatile.pool.pristine":     validate.IsAny,
		"volume.block.filesystem":    validate.Optional(validate.IsOneOf(cephAllowedFilesystems...)),
		"volume.block.mount_options": validate.IsAny,
	}

	return d.validatePool(config, rules)
}

// Update applies any driver changes required from a configuration change.
func (d *ceph) Update(changedConfig map[string]string) error {
	return nil
}

// Mount mounts the storage pool.
func (d *ceph) Mount() (bool, error) {
	// Nothing to do here.
	return true, nil
}

// Unmount unmounts the storage pool.
func (d *ceph) Unmount() (bool, error) {
	// Nothing to do here.
	return true, nil
}

// GetResources returns the pool resource usage information.
func (d *ceph) GetResources() (*api.ResourcesStoragePool, error) {
	var stdout bytes.Buffer

	err := shared.RunCommandWithFds(nil, &stdout,
		"ceph",
		"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
		"--cluster", d.config["ceph.cluster_name"],
		"df",
		"-f", "json")
	if err != nil {
		return nil, err
	}

	// Temporary structs for parsing.
	type cephDfPoolStats struct {
		BytesUsed      int64 `json:"bytes_used"`
		BytesAvailable int64 `json:"max_avail"`
	}

	type cephDfPool struct {
		Name  string          `json:"name"`
		Stats cephDfPoolStats `json:"stats"`
	}

	type cephDf struct {
		Pools []cephDfPool `json:"pools"`
	}

	// Parse the JSON output.
	df := cephDf{}
	err = json.NewDecoder(&stdout).Decode(&df)
	if err != nil {
		return nil, err
	}

	var pool *cephDfPool
	for _, entry := range df.Pools {
		if entry.Name == d.config["ceph.osd.pool_name"] {
			pool = &entry
			break
		}
	}

	if pool == nil {
		return nil, fmt.Errorf("OSD pool missing in df output")
	}

	spaceUsed := uint64(pool.Stats.BytesUsed)
	spaceAvailable := uint64(pool.Stats.BytesAvailable)

	res := api.ResourcesStoragePool{}
	res.Space.Total = spaceAvailable + spaceUsed
	res.Space.Used = spaceUsed

	return &res, nil
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *ceph) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	var rsyncFeatures []string

	// Do not pass compression argument to rsync if the associated
	// config key, that is rsync.compression, is set to false.
	if shared.IsFalse(d.Config()["rsync.compression"]) {
		rsyncFeatures = []string{"delete", "bidirectional"}
	} else {
		rsyncFeatures = []string{"delete", "compress", "bidirectional"}
	}

	if refresh {
		var transportType migration.MigrationFSType

		if contentType == ContentTypeBlock {
			transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
		} else {
			transportType = migration.MigrationFSType_RSYNC
		}

		return []migration.Type{
			{
				FSType:   transportType,
				Features: rsyncFeatures,
			},
		}
	}

	if contentType == ContentTypeBlock {
		return []migration.Type{
			{
				FSType: migration.MigrationFSType_RBD,
			},
			{
				FSType:   migration.MigrationFSType_BLOCK_AND_RSYNC,
				Features: rsyncFeatures,
			},
		}
	}

	return []migration.Type{
		{
			FSType: migration.MigrationFSType_RBD,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: rsyncFeatures,
		},
	}
}
<commit_msg>lxd/storage/drivers/driver/ceph: Mark ceph.osd.force_reuse as deprecated<commit_after>package drivers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"strings"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

var cephAllowedFilesystems = []string{"btrfs", "ext4", "xfs"}
var cephVersion string
var cephLoaded bool

type ceph struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *ceph) load() error {
	// Register the patches.
	d.patches = map[string]func() error{
		"storage_lvm_skipactivation":       nil,
		"storage_missing_snapshot_records": nil,
	}

	// Done if previously loaded.
	if cephLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"ceph", "rbd"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if cephVersion == "" {
		out, err := shared.RunCommand("rbd", "--version")
		if err != nil {
			return err
		}

		out = strings.TrimSpace(out)

		fields := strings.Split(out, " ")
		if strings.HasPrefix(out, "ceph version ") && len(fields) > 2 {
			cephVersion = fields[2]
		} else {
			cephVersion = out
		}
	}

	cephLoaded = true
	return nil
}

// isRemote returns true indicating this driver uses remote storage.
func (d *ceph) isRemote() bool {
	return true
}

// Info returns info about the driver and its environment.
func (d *ceph) Info() Info {
	return Info{
		Name:              "ceph",
		Version:           cephVersion,
		OptimizedImages:   true,
		PreservesInodes:   false,
		Remote:            d.isRemote(),
		VolumeTypes:       []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:      true,
		RunningCopyFreeze: true,
		DirectIO:          true,
		MountedRoot:       false,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *ceph) Create() error {
	revert := revert.New()
	defer revert.Fail()

	d.config["volatile.initial_source"] = d.config["source"]

	// Set default properties if missing.
	if d.config["ceph.cluster_name"] == "" {
		d.config["ceph.cluster_name"] = CephDefaultCluster
	}

	if d.config["ceph.user.name"] == "" {
		d.config["ceph.user.name"] = CephDefaultUser
	}

	if d.config["ceph.osd.pg_num"] == "" {
		d.config["ceph.osd.pg_num"] = "32"
	} else {
		// Validate.
		_, err := units.ParseByteSizeString(d.config["ceph.osd.pg_num"])
		if err != nil {
			return err
		}
	}

	// Quick check.
	if d.config["source"] != "" && d.config["ceph.osd.pool_name"] != "" && d.config["source"] != d.config["ceph.osd.pool_name"] {
		return fmt.Errorf(`The "source" and "ceph.osd.pool_name" property must not differ for Ceph OSD storage pools`)
	}

	// Use an existing OSD pool.
	if d.config["source"] != "" {
		d.config["ceph.osd.pool_name"] = d.config["source"]
	}

	if d.config["ceph.osd.pool_name"] == "" {
		d.config["ceph.osd.pool_name"] = d.name
		d.config["source"] = d.name
	}

	placeholderVol := NewVolume(d, d.name, VolumeType("lxd"), ContentTypeFS, d.config["ceph.osd.pool_name"], nil, nil)

	if !d.osdPoolExists() {
		// Create new osd pool.
		_, err := shared.TryRunCommand("ceph",
			"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
			"--cluster", d.config["ceph.cluster_name"],
			"osd",
			"pool",
			"create",
			d.config["ceph.osd.pool_name"],
			d.config["ceph.osd.pg_num"])
		if err != nil {
			return err
		}

		revert.Add(func() { _ = d.osdDeletePool() })

		// Initialize the pool. This is not necessary but allows the pool to be monitored.
		_, err = shared.TryRunCommand("rbd",
			"--id", d.config["ceph.user.name"],
			"--cluster", d.config["ceph.cluster_name"],
			"pool",
			"init",
			d.config["ceph.osd.pool_name"])
		if err != nil {
			d.logger.Warn("Failed to initialize pool", logger.Ctx{"pool": d.config["ceph.osd.pool_name"], "cluster": d.config["ceph.cluster_name"]})
		}

		// Create placeholder storage volume. Other LXD instances will use this to detect whether this osd
		// pool is already in use by another LXD instance.
		err = d.rbdCreateVolume(placeholderVol, "0")
		if err != nil {
			return err
		}

		d.config["volatile.pool.pristine"] = "true"
	} else {
		ok := d.HasVolume(placeholderVol)
		if ok {
			// ceph.osd.force_reuse is deprecated and should not be used. OSD pools are a logical
			// construct there is no good reason not to create one for dedicated use by LXD.
			if shared.IsFalseOrEmpty(d.config["ceph.osd.force_reuse"]) {
				return fmt.Errorf("Pool '%s' in cluster '%s' seems to be in use by another LXD instance. Use 'ceph.osd.force_reuse=true' to force", d.config["ceph.osd.pool_name"], d.config["ceph.cluster_name"])
			}

			d.config["volatile.pool.pristine"] = "false"
		} else {
			// Create placeholder storage volume. Other LXD instances will use this to detect whether this osd
			// pool is already in use by another LXD instance.
			err := d.rbdCreateVolume(placeholderVol, "0")
			if err != nil {
				return err
			}

			d.config["volatile.pool.pristine"] = "true"
		}

		// Use existing OSD pool.
		msg, err := shared.RunCommand("ceph",
			"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
			"--cluster", d.config["ceph.cluster_name"],
			"osd",
			"pool",
			"get",
			d.config["ceph.osd.pool_name"],
			"pg_num")
		if err != nil {
			return err
		}

		idx := strings.Index(msg, "pg_num:")
		if idx == -1 {
			return fmt.Errorf("Failed to parse number of placement groups for pool: %s", msg)
		}

		msg = msg[(idx + len("pg_num:")):]
		msg = strings.TrimSpace(msg)

		// It is ok to update the pool configuration since storage pool
		// creation via API is implemented such that the storage pool is
		// checked for a changed config after this function returns and
		// if so the db for it is updated.
		d.config["ceph.osd.pg_num"] = msg
	}

	revert.Success()

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *ceph) Delete(op *operations.Operation) error {
	// Test if the pool exists.
	poolExists := d.osdPoolExists()
	if !poolExists {
		d.logger.Warn("Pool does not exist", logger.Ctx{"pool": d.config["ceph.osd.pool_name"], "cluster": d.config["ceph.cluster_name"]})
	}

	// Check whether we own the pool and only remove in this case.
	if shared.IsTrue(d.config["volatile.pool.pristine"]) {
		// Delete the osd pool.
		if poolExists {
			err := d.osdDeletePool()
			if err != nil {
				return err
			}
		}
	}

	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// On delete, wipe everything in the directory.
	err := wipeDirectory(GetPoolMountPath(d.name))
	if err != nil {
		return err
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *ceph) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"ceph.cluster_name":          validate.IsAny,
		"ceph.osd.force_reuse":       validate.Optional(validate.IsBool), // Deprecated, should not be used.
		"ceph.osd.pg_num":            validate.IsAny,
		"ceph.osd.pool_name":         validate.IsAny,
		"ceph.osd.data_pool_name":    validate.IsAny,
		"ceph.rbd.clone_copy":        validate.Optional(validate.IsBool),
		"ceph.rbd.du":                validate.Optional(validate.IsBool),
		"ceph.rbd.features":          validate.IsAny,
		"ceph.user.name":             validate.IsAny,
		"volatile.pool.pristine":     validate.IsAny,
		"volume.block.filesystem":    validate.Optional(validate.IsOneOf(cephAllowedFilesystems...)),
		"volume.block.mount_options": validate.IsAny,
	}

	return d.validatePool(config, rules)
}

// Update applies any driver changes required from a configuration change.
func (d *ceph) Update(changedConfig map[string]string) error {
	return nil
}

// Mount mounts the storage pool.
func (d *ceph) Mount() (bool, error) {
	// Nothing to do here.
	return true, nil
}

// Unmount unmounts the storage pool.
func (d *ceph) Unmount() (bool, error) {
	// Nothing to do here.
	return true, nil
}

// GetResources returns the pool resource usage information.
func (d *ceph) GetResources() (*api.ResourcesStoragePool, error) {
	var stdout bytes.Buffer

	err := shared.RunCommandWithFds(nil, &stdout,
		"ceph",
		"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
		"--cluster", d.config["ceph.cluster_name"],
		"df",
		"-f", "json")
	if err != nil {
		return nil, err
	}

	// Temporary structs for parsing.
	type cephDfPoolStats struct {
		BytesUsed      int64 `json:"bytes_used"`
		BytesAvailable int64 `json:"max_avail"`
	}

	type cephDfPool struct {
		Name  string          `json:"name"`
		Stats cephDfPoolStats `json:"stats"`
	}

	type cephDf struct {
		Pools []cephDfPool `json:"pools"`
	}

	// Parse the JSON output.
	df := cephDf{}
	err = json.NewDecoder(&stdout).Decode(&df)
	if err != nil {
		return nil, err
	}

	var pool *cephDfPool
	for _, entry := range df.Pools {
		if entry.Name == d.config["ceph.osd.pool_name"] {
			pool = &entry
			break
		}
	}

	if pool == nil {
		return nil, fmt.Errorf("OSD pool missing in df output")
	}

	spaceUsed := uint64(pool.Stats.BytesUsed)
	spaceAvailable := uint64(pool.Stats.BytesAvailable)

	res := api.ResourcesStoragePool{}
	res.Space.Total = spaceAvailable + spaceUsed
	res.Space.Used = spaceUsed

	return &res, nil
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *ceph) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	var rsyncFeatures []string

	// Do not pass compression argument to rsync if the associated
	// config key, that is rsync.compression, is set to false.
	if shared.IsFalse(d.Config()["rsync.compression"]) {
		rsyncFeatures = []string{"delete", "bidirectional"}
	} else {
		rsyncFeatures = []string{"delete", "compress", "bidirectional"}
	}

	if refresh {
		var transportType migration.MigrationFSType

		if contentType == ContentTypeBlock {
			transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
		} else {
			transportType = migration.MigrationFSType_RSYNC
		}

		return []migration.Type{
			{
				FSType:   transportType,
				Features: rsyncFeatures,
			},
		}
	}

	if contentType == ContentTypeBlock {
		return []migration.Type{
			{
				FSType: migration.MigrationFSType_RBD,
			},
			{
				FSType:   migration.MigrationFSType_BLOCK_AND_RSYNC,
				Features: rsyncFeatures,
			},
		}
	}

	return []migration.Type{
		{
			FSType: migration.MigrationFSType_RBD,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: rsyncFeatures,
		},
	}
}
<|endoftext|>


<commit_before>package box

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
)

// TODO(ttacon): reconcile this with Folder for one common struct?
type File struct {
	ID                string          `json:"id,omitempty"`
	FolderUploadEmail *AccessEmail    `json:"folder_upload_email,omitempty"`
	Parent            *Item           `json:"parent,omitempty"`
	ItemStatus        string          `json:"item_status"`
	ItemCollection    *ItemCollection `json:"item_collection"`
	Type              string          `json:"type"` // TODO(ttacon): enum
	Description       string          `json:"description"`
	Size              int             `json:"size"`
	CreateBy          *Item           `json:"created_by"`
	ModifiedBy        *Item           `json:"modified_by"`
	TrashedAt         *string         `json:"trashed_at"`          // TODO(ttacon): change to time.Time
	ContentModifiedAt *string         `json:"content_modified_at"` // TODO(ttacon): change to time.Time
	PurgedAt          *string         `json:"purged_at"`           // TODO(ttacon): change to time.Time, this field isn't documented but I keep getting it back...
	SharedLinkg       *string         `json:"shared_link"`
	SequenceId        string          `json:"sequence_id"`
	ETag              *string         `json:"etag"`
	Name              string          `json:"name"`
	CreatedAt         *string         `json:"created_at"` // TODO(ttacon): change to time.Time
	OwnedBy           *Item           `json:"owned_by"`
	ModifiedAt        *string         `json:"modified_at"`        // TODO(ttacon): change to time.Time
	ContentCreatedAt  *string         `json:"content_created_at"` // TODO(ttacon): change to time.Time
	PathCollection    *PathCollection `json:"path_collection"`    // TODO(ttacon): make sure this is the correct kind of struct(ure)
	SharedLink        *Link           `json:"shared_link"`

	SHA1 string `json:"sha1"`
}

type FileCollection struct {
	TotalCount int     `json:"total_count"`
	Entries    []*File `json:"entries"`
}

// Documentation: https://developer.box.com/docs/#files-get
func (c *Client) GetFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation https://developer.box.com/docs/#files-upload-a-file
// TODO(ttacon): deal with handling SHA1 headers
func (c *Client) UploadFile(filePath, parentId string) (*http.Response, *FileCollection, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	fileContents, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, nil, err
	}

	fi, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	var (
		body   = &bytes.Buffer{}
		writer = multipart.NewWriter(body)
	)

	// write the file
	part, err := writer.CreateFormFile("file", fi.Name())
	if err != nil {
		return nil, nil, err
	}
	part.Write(fileContents)

	// write the other form fields we need
	err = writer.WriteField("filename", fi.Name())
	if err != nil {
		return nil, nil, err
	}

	err = writer.WriteField("parent_id", parentId)
	if err != nil {
		return nil, nil, err
	}

	// TODO(ttacon): add in content_created_at, content_modified_at

	err = writer.Close()
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("https://upload.box.com/api/2.0/files/content"),
		body,
	)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-delete-a-file
func (c *Client) DeleteFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"DELETE",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-copy-a-file
func (c *Client) CopyFile(fileId, parent, name string) (*http.Response, *File, error) {
	var bodyData = map[string]interface{}{
		"parent": map[string]string{
			"id": parent,
		},
		"name": name,
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("/files/%s/copy", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we return the http.Response as Box may return a 202 if there is not
// yet a download link, or a 302 with the link - this allows the user to
// decide what to do.
// Documentation: https://developers.box.com/docs/#files-download-a-file
func (c *Client) DownloadFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/content", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-view-versions-of-a-file
// TODO(ttacon): don't use file collection, make actual structs specific to file versions
func (c *Client) ViewVersionsOfFile(fileId string) (*http.Response, *FileCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/versions", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// NOTE: we only return the response as there are many possible responses that we
// feel the user should have control over
// Documentation: https://developers.box.com/docs/#files-get-a-thumbnail-for-a-file
func (c *Client) GetThumbnail(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/thumbnail.extension", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-create-a-shared-link-for-a-file
func (c *Client) CreateSharedLinkForFile(fileId, access, unsharedAt string, canDownload, canPreview bool) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(access) > 0 {
		dataMap["access"] = access
	}
	// TODO(ttacon): support unshared_at as time.Time
	// TODO(ttacon): validate access is open or company before add permissions
	if canDownload {
		dataMap["permissions"] = map[string]bool{
			"can_download": canDownload,
		}
	}
	if canPreview {
		if m, ok := dataMap["permissions"]; ok {
			mVal, _ := m.(map[string]bool)
			mVal["can_preview"] = canPreview
		} else {
			dataMap["permissions"] = map[string]bool{
				"can_preview": canPreview,
			}
		}
	}

	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"PUT",
		fmt.Sprintf("%s/files/%s", BASE_URL, fileId),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-a-trashed-file
func (c *Client) GetTrashedFile(fileId string) (*http.Response, *File, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-restore-a-trashed-item
func (c *Client) RestoreTrashedItem(name, parentId string) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(name) > 0 {
		dataMap["name"] = name
	}
	if len(parentId) > 0 {
		dataMap["parent"] = map[string]string{
			"id": parentId,
		}
	}
	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf(),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-permanently-delete-a-trashed-file
func (c *Client) PermanentlyDeleteTrashedFile(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"DELETE",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-view-the-comments-on-a-file
func (c *Client) ViewCommentsOnFile(fileId string) (*http.Response, *CommentCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/comments", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data CommentCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-the-tasks-for-a-file
func (c *Client) GetTasksForFile(fileId string) (*http.Response, *TaskCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/tasks", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data TaskCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}
<commit_msg>refactor view versions of file<commit_after>package box

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
)

// TODO(ttacon): reconcile this with Folder for one common struct?
type File struct {
	ID                string          `json:"id,omitempty"`
	FolderUploadEmail *AccessEmail    `json:"folder_upload_email,omitempty"`
	Parent            *Item           `json:"parent,omitempty"`
	ItemStatus        string          `json:"item_status"`
	ItemCollection    *ItemCollection `json:"item_collection"`
	Type              string          `json:"type"` // TODO(ttacon): enum
	Description       string          `json:"description"`
	Size              int             `json:"size"`
	CreateBy          *Item           `json:"created_by"`
	ModifiedBy        *Item           `json:"modified_by"`
	TrashedAt         *string         `json:"trashed_at"`          // TODO(ttacon): change to time.Time
	ContentModifiedAt *string         `json:"content_modified_at"` // TODO(ttacon): change to time.Time
	PurgedAt          *string         `json:"purged_at"`           // TODO(ttacon): change to time.Time, this field isn't documented but I keep getting it back...
	SharedLinkg       *string         `json:"shared_link"`
	SequenceId        string          `json:"sequence_id"`
	ETag              *string         `json:"etag"`
	Name              string          `json:"name"`
	CreatedAt         *string         `json:"created_at"` // TODO(ttacon): change to time.Time
	OwnedBy           *Item           `json:"owned_by"`
	ModifiedAt        *string         `json:"modified_at"`        // TODO(ttacon): change to time.Time
	ContentCreatedAt  *string         `json:"content_created_at"` // TODO(ttacon): change to time.Time
	PathCollection    *PathCollection `json:"path_collection"`    // TODO(ttacon): make sure this is the correct kind of struct(ure)
	SharedLink        *Link           `json:"shared_link"`

	SHA1 string `json:"sha1"`
}

type FileCollection struct {
	TotalCount int     `json:"total_count"`
	Entries    []*File `json:"entries"`
}

// Documentation: https://developer.box.com/docs/#files-get
func (c *Client) GetFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation https://developer.box.com/docs/#files-upload-a-file
// TODO(ttacon): deal with handling SHA1 headers
func (c *Client) UploadFile(filePath, parentId string) (*http.Response, *FileCollection, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	fileContents, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, nil, err
	}

	fi, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	var (
		body   = &bytes.Buffer{}
		writer = multipart.NewWriter(body)
	)

	// write the file
	part, err := writer.CreateFormFile("file", fi.Name())
	if err != nil {
		return nil, nil, err
	}
	part.Write(fileContents)

	// write the other form fields we need
	err = writer.WriteField("filename", fi.Name())
	if err != nil {
		return nil, nil, err
	}

	err = writer.WriteField("parent_id", parentId)
	if err != nil {
		return nil, nil, err
	}

	// TODO(ttacon): add in content_created_at, content_modified_at

	err = writer.Close()
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("https://upload.box.com/api/2.0/files/content"),
		body,
	)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-delete-a-file
func (c *Client) DeleteFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"DELETE",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-copy-a-file
func (c *Client) CopyFile(fileId, parent, name string) (*http.Response, *File, error) {
	var bodyData = map[string]interface{}{
		"parent": map[string]string{
			"id": parent,
		},
		"name": name,
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("/files/%s/copy", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we return the http.Response as Box may return a 202 if there is not
// yet a download link, or a 302 with the link - this allows the user to
// decide what to do.
// Documentation: https://developers.box.com/docs/#files-download-a-file
func (c *Client) DownloadFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/content", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-view-versions-of-a-file
// TODO(ttacon): don't use file collection, make actual structs specific to file versions
func (c *Client) ViewVersionsOfFile(fileId string) (*http.Response, *FileCollection, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/versions", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *FileCollection
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we only return the response as there are many possible responses that we
// feel the user should have control over
// Documentation: https://developers.box.com/docs/#files-get-a-thumbnail-for-a-file
func (c *Client) GetThumbnail(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/thumbnail.extension", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-create-a-shared-link-for-a-file
func (c *Client) CreateSharedLinkForFile(fileId, access, unsharedAt string, canDownload, canPreview bool) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(access) > 0 {
		dataMap["access"] = access
	}
	// TODO(ttacon): support unshared_at as time.Time
	// TODO(ttacon): validate access is open or company before add permissions
	if canDownload {
		dataMap["permissions"] = map[string]bool{
			"can_download": canDownload,
		}
	}
	if canPreview {
		if m, ok := dataMap["permissions"]; ok {
			mVal, _ := m.(map[string]bool)
			mVal["can_preview"] = canPreview
		} else {
			dataMap["permissions"] = map[string]bool{
				"can_preview": canPreview,
			}
		}
	}

	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"PUT",
		fmt.Sprintf("%s/files/%s", BASE_URL, fileId),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-a-trashed-file
func (c *Client) GetTrashedFile(fileId string) (*http.Response, *File, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-restore-a-trashed-item
func (c *Client) RestoreTrashedItem(name, parentId string) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(name) > 0 {
		dataMap["name"] = name
	}
	if len(parentId) > 0 {
		dataMap["parent"] = map[string]string{
			"id": parentId,
		}
	}
	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf(),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-permanently-delete-a-trashed-file
func (c *Client) PermanentlyDeleteTrashedFile(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"DELETE",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-view-the-comments-on-a-file
func (c *Client) ViewCommentsOnFile(fileId string) (*http.Response, *CommentCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/comments", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data CommentCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-the-tasks-for-a-file
func (c *Client) GetTasksForFile(fileId string) (*http.Response, *TaskCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/tasks", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data TaskCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}
<|endoftext|>


<commit_before>package scaniigo

// ProcessedFileResponse holds the returned value from a call to
// the previously processed file endpoint
type ProcessFileResponse struct {
	ID            string   `json:"id"`
	Checksum      string   `json:"checksum"`
	ContentLength int      `json:"content_length"`
	Findings      []string `json:"findings"`
	CreationDate  string   `json:"creation_date"`
	ContentType   string   `json:"content_type"`
}

// ProcessFileRequest holds; the options needed for the given API call
type ProcessFileParams struct {
	FileLocation string
	Callback     string
	Metadata     string
}

// RetrieveProcessedFile retrieves a previously processed file resource
func (c *Client) RetrieveProcessedFile(id string) (*ProcessFileResponse, error) {
	return nil, nil
}

// ProcessFileSync submits a file for processing synchronously
func (c *Client) ProcessFileSync(pfr *ProcessFileParams) (*ProcessFileResponse, error) {
	return nil, nil
}

// ProcessFileAsync submits a file for processing synchronously
func (c *Client) ProcessFileAsync(pfr *ProcessFileParams) (*ProcessFileResponse, error) {
	return nil, nil
}
<commit_msg>buld of files work<commit_after>package scaniigo

import (
	"encoding/json"
	"net/http"
)

// ProcessedFileResponse holds the returned value from a call to
// the previously processed file endpoint
type ProcessFileResponse struct {
	ID            string   `json:"id"`
	Checksum      string   `json:"checksum"`
	ContentLength int      `json:"content_length"`
	Findings      []string `json:"findings"`
	CreationDate  string   `json:"creation_date"`
	ContentType   string   `json:"content_type"`
}

// ProcessFileRequest holds; the options needed for the given API call
type ProcessFileParams struct {
	FileLocation string
	Callback     string
	Metadata     string
}

// RetrieveProcessedFile retrieves a previously processed file resource
func (c *Client) RetrieveProcessedFile(id string) (*ProcessFileResponse, error) {
	req, err := http.NewRequest("GET", c.Endpoint+FilePath+"/"+id, nil)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(c.APIAuth.Key, c.APIAuth.Secret)

	res, err := c.HTTPClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	var pfr ProcessFileResponse
	if err := json.NewDecoder(res.Body).Decode(&pfr); err != nil {
		return nil, err
	}
	return &pfr, nil
}

// ProcessFileSync submits a file for processing synchronously
func (c *Client) ProcessFileSync(pfp *ProcessFileParams) (*ProcessFileResponse, error) {
	req, err := http.NewRequest("POST", c.Endpoint+FilePath, nil)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(c.APIAuth.Key, c.APIAuth.Secret)
	req.Header.Set("Content-Type", "multipart/form-data")

	res, err := c.HTTPClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	var pfr ProcessFileResponse
	if err := json.NewDecoder(res.Body).Decode(&pfr); err != nil {
		return nil, err
	}
	return &pfr, nil
}

// ProcessFileAsync submits a file for processing synchronously
func (c *Client) ProcessFileAsync(pfp *ProcessFileParams) (*ProcessFileResponse, error) {
	req, err := http.NewRequest("POST", c.Endpoint+FileAsyncPath, nil)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(c.APIAuth.Key, c.APIAuth.Secret)
	req.Header.Set("Content-Type", "multipart/form-data")

	res, err := c.HTTPClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	var pfr ProcessFileResponse
	if err := json.NewDecoder(res.Body).Decode(&pfr); err != nil {
		return nil, err
	}
	return &pfr, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"gitlab.com/cjbarker/bucketscanner"
	"gopkg.in/alecthomas/kingpin.v2"
	"os"
	"strings"
	"sync"
	"time"
)

// Exit Codes
const (
	Success      = 0
	InvalidCloud = 100
)

// Provider
const (
	All           = "all"
	AwsProvider   = "aws"
	GcpProvider   = "gcp"
	AzureProvider = "azure"
)

// Config is struct representing the Commandline argument settings
type Config struct {
	BucketNames   *string
	Download      *bool
	Output        *string
	Verbose       *bool
	CloudProvider *string
	ThrottleMs    *int
	JSON          *bool
}

func (c Config) v(msg string) {
	if *c.Verbose {
		fmt.Printf("%s\n", msg)
	}
}

// Globals
var configPtr *Config

func getScanner(providerName *string) (scanners []bucketscanner.Scanner) {
	if providerName == nil || strings.Trim(*providerName, " ") == "" {
		return nil
	}

	//var scanners []*Scanner
	if strings.ToLower(*providerName) == All {
		scanners = append(scanners, &bucketscanner.AwsScanner{})
		scanners = append(scanners, &bucketscanner.GcpScanner{})
		scanners = append(scanners, &bucketscanner.AzureScanner{})
	} else if strings.ToLower(*providerName) == AwsProvider {
		scanners = append(scanners, &bucketscanner.AwsScanner{})
	} else if strings.ToLower(*providerName) == GcpProvider {
		scanners = append(scanners, &bucketscanner.GcpScanner{})
	} else if strings.ToLower(*providerName) == AzureProvider {
		scanners = append(scanners, &bucketscanner.AzureScanner{})
	} else {
		scanners = nil
	}

	return scanners
}

func main() {
	configPtr = new(Config)

	app := kingpin.New(os.Args[0], "Cloud command-line bucket (object) scanner.")
	app.Version("Version: " + bucketscanner.Version + "\nBuild: " + bucketscanner.Build)

	configPtr.BucketNames = app.Arg("bucket-name", "Bucket(s) name(s) to scan. Does support comma separated for multiple buckets.").Required().String()
	configPtr.CloudProvider = app.Flag("cloud", "Cloud provider to scan: aws, gcp, azure. Defaults to all.").Required().String()
	configPtr.ThrottleMs = app.Flag("throttle", "Time in milliseconds to throttle subsequent requests sent to a given provider.").Int()
	configPtr.Download = app.Flag("download", "Download bucket content(s).").Bool()
	configPtr.Output = app.Flag("output", "Download bucket content(s) destination directory. Defaults to current user's directory if none passed.").String()
	configPtr.JSON = app.Flag("JSON", "Output results in JSON.").Bool()
	configPtr.Verbose = app.Flag("verbose", "Verbose output messages. Defaults to quiet.").Bool()

	kingpin.MustParse(app.Parse(os.Args[1:]))

	// Default to all
	if strings.Trim(*configPtr.CloudProvider, " ") == "" {
		*configPtr.CloudProvider = All
	}

	// output settings
	configPtr.v(fmt.Sprintf("Cloud: %s", *configPtr.CloudProvider))
	configPtr.v(fmt.Sprintf("Buckets: % s", *configPtr.BucketNames))
	configPtr.v(fmt.Sprintf("ThrottleMS: %d", *configPtr.ThrottleMs))
	configPtr.v(fmt.Sprintf("Download: %t", *configPtr.Download))
	configPtr.v(fmt.Sprintf("Output: %t", *configPtr.Output))
	configPtr.v(fmt.Sprintf("JSON: %t", *configPtr.JSON))
	configPtr.v(fmt.Sprintf("Verbose: %t", *configPtr.Verbose))

	// space or command delim
	var bucketNames []string
	if strings.Index(*configPtr.BucketNames, ",") > -1 {
		bucketNames = strings.Split(*configPtr.BucketNames, ",")
	} else {
		bucketNames = strings.Split(*configPtr.BucketNames, " ")
	}

	var wg sync.WaitGroup
	var mutex = &sync.Mutex{}
	var buckets = []*bucketscanner.Bucket{}

	scanners := getScanner(configPtr.CloudProvider)

	for idx, scanner := range scanners {
		wg.Add(idx + 1)

		go func(scanner bucketscanner.Scanner) {
			defer wg.Done()

			for idx, bucketName := range bucketNames {
				bucketName = strings.Trim(bucketName, " ")

				if idx > 0 && *configPtr.ThrottleMs > 0 {
					configPtr.v(fmt.Sprintf("Throttle via sleep for %d MS", *configPtr.ThrottleMs))
					time.Sleep(time.Duration(*configPtr.ThrottleMs) * time.Millisecond)
				}

				configPtr.v(fmt.Sprintf("Getting from %s bucket: %s", scanner.GetProviderName(), bucketName))

				bucket, err := scanner.Read(bucketName)

				if err != nil {
					fmt.Println(err)
				} else {
					configPtr.v("Bucket response received")
					mutex.Lock()
					buckets = append(buckets, bucket)
					mutex.Unlock()
				}

				if *configPtr.Download || (configPtr.Output != nil && len(*configPtr.Output) > 0) {
					configPtr.v(fmt.Sprintf("Download bucket contents from %s ", bucketName))
					zipFile, err := bucket.Download(*configPtr.Output)
					if err != nil {
						configPtr.v(fmt.Sprintf("Unable to download bucket due to error: %s", err.Error()))
						//configPtr.v(fmt.Fprintln(os.Stderr, "Unable to download bucket due to error: %s", err.Error()))
					}
					fmt.Printf("Bucket downloaded successfully to %s\n", *zipFile)
				}
			}
		}(scanner)
	}

	wg.Wait()
	configPtr.v("*** Scan Completed ****")

	// Output Results
	for _, bucket := range buckets {
		if *configPtr.JSON {
			JSONStr, err := json.Marshal(bucket)
			if err != nil {
				fmt.Println(err)
				// TODO panic or not?
			}
			fmt.Printf("%s\n", string(JSONStr))
		} else {
			fmt.Printf("%v\n", bucket)
		}
	}

	os.Exit(Success)
}
<commit_msg>Created scan action flag<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"gitlab.com/cjbarker/bucketscanner"
	"gopkg.in/alecthomas/kingpin.v2"
	"os"
	"strings"
	"sync"
	"time"
)

// Exit Codes
const (
	Success      = 0
	InvalidCloud = 100
)

// Provider
const (
	All           = "all"
	AwsProvider   = "aws"
	GcpProvider   = "gcp"
	AzureProvider = "azure"
)

// scan actions
const (
	Read  = "r"
	Write = "w"
)

// Config is struct representing the Commandline argument settings
type Config struct {
	BucketNames   *string
	Action        *string
	Download      *bool
	Output        *string
	Verbose       *bool
	CloudProvider *string
	ThrottleMs    *int
	JSON          *bool
}

func (c Config) v(msg string) {
	if *c.Verbose {
		fmt.Printf("%s\n", msg)
	}
}

// Globals
var configPtr *Config

func getScanner(providerName *string) (scanners []bucketscanner.Scanner) {
	if providerName == nil || strings.Trim(*providerName, " ") == "" {
		return nil
	}

	//var scanners []*Scanner
	if strings.ToLower(*providerName) == All {
		scanners = append(scanners, &bucketscanner.AwsScanner{})
		scanners = append(scanners, &bucketscanner.GcpScanner{})
		scanners = append(scanners, &bucketscanner.AzureScanner{})
	} else if strings.ToLower(*providerName) == AwsProvider {
		scanners = append(scanners, &bucketscanner.AwsScanner{})
	} else if strings.ToLower(*providerName) == GcpProvider {
		scanners = append(scanners, &bucketscanner.GcpScanner{})
	} else if strings.ToLower(*providerName) == AzureProvider {
		scanners = append(scanners, &bucketscanner.AzureScanner{})
	} else {
		scanners = nil
	}

	return scanners
}

func main() {
	configPtr = new(Config)

	app := kingpin.New(os.Args[0], "Cloud command-line bucket (object) scanner.")
	app.Version("Version: " + bucketscanner.Version + "\nBuild: " + bucketscanner.Build)

	configPtr.BucketNames = app.Arg("bucket-name", "Bucket(s) name(s) to scan. Does support comma separated for multiple buckets.").Required().String()
	configPtr.CloudProvider = app.Flag("cloud", "Cloud provider to scan: aws, gcp, azure. Defaults to all.").Required().String()
	configPtr.Action = app.Flag("action", "Scan action to invoke against bucket: (r)ead, (w)rite, all. Defaults to all.").Required().String()
	configPtr.ThrottleMs = app.Flag("throttle", "Time in milliseconds to throttle subsequent requests sent to a given provider.").Int()
	configPtr.Download = app.Flag("download", "Download bucket content(s).").Bool()
	configPtr.Output = app.Flag("output", "Download bucket content(s) destination directory. Defaults to current user's directory if none passed.").String()
	configPtr.JSON = app.Flag("JSON", "Output results in JSON.").Bool()
	configPtr.Verbose = app.Flag("verbose", "Verbose output messages. Defaults to quiet.").Bool()

	kingpin.MustParse(app.Parse(os.Args[1:]))

	// Default to all
	if strings.Trim(*configPtr.CloudProvider, " ") == "" {
		*configPtr.CloudProvider = All
	}

	if strings.Trim(*configPtr.Action, " ") == "" {
		*configPtr.Action = All
	}

	// output settings
	configPtr.v(fmt.Sprintf("Cloud: %s", *configPtr.CloudProvider))
	configPtr.v(fmt.Sprintf("Buckets: % s", *configPtr.BucketNames))
	configPtr.v(fmt.Sprintf("Action: % s", *configPtr.Action))
	configPtr.v(fmt.Sprintf("ThrottleMS: %d", *configPtr.ThrottleMs))
	configPtr.v(fmt.Sprintf("Download: %t", *configPtr.Download))
	configPtr.v(fmt.Sprintf("Output: %t", *configPtr.Output))
	configPtr.v(fmt.Sprintf("JSON: %t", *configPtr.JSON))
	configPtr.v(fmt.Sprintf("Verbose: %t", *configPtr.Verbose))

	// space or command delim
	var bucketNames []string
	if strings.Index(*configPtr.BucketNames, ",") > -1 {
		bucketNames = strings.Split(*configPtr.BucketNames, ",")
	} else {
		bucketNames = strings.Split(*configPtr.BucketNames, " ")
	}

	var wg sync.WaitGroup
	var mutex = &sync.Mutex{}
	var buckets = []*bucketscanner.Bucket{}

	scanners := getScanner(configPtr.CloudProvider)

	for idx, scanner := range scanners {
		wg.Add(idx + 1)

		go func(scanner bucketscanner.Scanner) {
			defer wg.Done()

			for idx, bucketName := range bucketNames {
				bucketName = strings.Trim(bucketName, " ")

				if idx > 0 && *configPtr.ThrottleMs > 0 {
					configPtr.v(fmt.Sprintf("Throttle via sleep for %d MS", *configPtr.ThrottleMs))
					time.Sleep(time.Duration(*configPtr.ThrottleMs) * time.Millisecond)
				}

				configPtr.v(fmt.Sprintf("Getting from %s bucket: %s", scanner.GetProviderName(), bucketName))

				// TODO account for read, write or both - configPtr.Action
				bucket, err := scanner.Read(bucketName)

				if err != nil {
					fmt.Println(err)
				} else {
					configPtr.v("Bucket response received")
					mutex.Lock()
					buckets = append(buckets, bucket)
					mutex.Unlock()
				}

				if *configPtr.Download || (configPtr.Output != nil && len(*configPtr.Output) > 0) {
					configPtr.v(fmt.Sprintf("Download bucket contents from %s ", bucketName))
					zipFile, err := bucket.Download(*configPtr.Output)
					if err != nil {
						configPtr.v(fmt.Sprintf("Unable to download bucket due to error: %s", err.Error()))
						//configPtr.v(fmt.Fprintln(os.Stderr, "Unable to download bucket due to error: %s", err.Error()))
					}
					fmt.Printf("Bucket downloaded successfully to %s\n", *zipFile)
				}
			}
		}(scanner)
	}

	wg.Wait()
	configPtr.v("*** Scan Completed ****")

	// Output Results
	for _, bucket := range buckets {
		if *configPtr.JSON {
			JSONStr, err := json.Marshal(bucket)
			if err != nil {
				fmt.Println(err)
				// TODO panic or not?
			}
			fmt.Printf("%s\n", string(JSONStr))
		} else {
			fmt.Printf("%v\n", bucket)
		}
	}

	os.Exit(Success)
}
<|endoftext|>


<commit_before>package client

import (
	"github.com/hashicorp/nomad/nomad/structs"
	"log"
	"os"
	"testing"
	"time"
)

func TestMakeChecks(t *testing.T) {
	service := &structs.Service{
		Id:   "Foo",
		Name: "Bar",
		Checks: []structs.ServiceCheck{
			{
				Type:     "http",
				Path:     "/foo/bar",
				Interval: 10 * time.Second,
				Timeout:  2 * time.Second,
			},
			{
				Type:     "tcp",
				Interval: 10 * time.Second,
				Timeout:  2 * time.Second,
			},
		},
	}

	logger := log.New(os.Stdout, "logger: ", log.Lshortfile)

	c, _ := NewConsulClient(logger, "")
	checks := c.makeChecks(service, "10.10.0.1", 8090)

	if checks[0].HTTP != "http://10.10.0.1:8090/foo/bar" {
		t.Fatalf("Invalid http url for check: %v", checks[0].HTTP)
	}

	if checks[1].TCP != "10.10.0.1:8090" {
		t.Fatalf("Invalid tcp check: %v", checks[0].TCP)
	}
}
<commit_msg>Added a check for accpeting protocol information in consul check<commit_after>package client

import (
	"github.com/hashicorp/nomad/nomad/structs"
	"log"
	"os"
	"testing"
	"time"
)

func TestMakeChecks(t *testing.T) {
	service := &structs.Service{
		Id:   "Foo",
		Name: "Bar",
		Checks: []structs.ServiceCheck{
			{
				Type:     "http",
				Path:     "/foo/bar",
				Interval: 10 * time.Second,
				Timeout:  2 * time.Second,
			},
			{
				Type:     "http",
				Protocol: "https",
				Path:     "/foo/bar",
				Interval: 10 * time.Second,
				Timeout:  2 * time.Second,
			},
			{
				Type:     "tcp",
				Interval: 10 * time.Second,
				Timeout:  2 * time.Second,
			},
		},
	}

	logger := log.New(os.Stdout, "logger: ", log.Lshortfile)

	c, _ := NewConsulClient(logger, "")
	checks := c.makeChecks(service, "10.10.0.1", 8090)

	if checks[0].HTTP != "http://10.10.0.1:8090/foo/bar" {
		t.Fatalf("Invalid http url for check: %v", checks[0].HTTP)
	}

	if checks[1].HTTP != "https://10.10.0.1:8090/foo/bar" {
		t.Fatalf("Invalid http url for check: %v", checks[0].HTTP)
	}

	if checks[2].TCP != "10.10.0.1:8090" {
		t.Fatalf("Invalid tcp check: %v", checks[0].TCP)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Ajit Yagaty
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"

	"github.com/ajityagaty/go-kairosdb/builder"
	"github.com/ajityagaty/go-kairosdb/response"
)

var (
	api_version    = "/api/v1"
	datapoints_ep  = api_version + "/datapoints"
	query_ep       = api_version + "/datapoints/query"
	version_ep     = api_version + "/version"
	health_ep      = api_version + "/health/check"
	metricnames_ep = api_version + "/metricnames"
	tagnames_ep    = api_version + "/tagnames"
	tagvalues_ep   = api_version + "/tagvalues"
	delmetric_ep   = api_version + "/metric/"
)

// This is the type that implements the Client interface.
type httpClient struct {
	serverAddress string
}

func NewHttpClient(serverAddress string) Client {
	return &httpClient{
		serverAddress: serverAddress,
	}
}

// Returns a list of all metrics names.
func (hc *httpClient) GetMetricNames() (*response.GetResponse, error) {
	return hc.get(hc.serverAddress + metricnames_ep)
}

// Returns a list of all tag names.
func (hc *httpClient) GetTagNames() (*response.GetResponse, error) {
	return hc.get(hc.serverAddress + tagnames_ep)
}

// Returns a list of all tag values.
func (hc *httpClient) GetTagValues() (*response.GetResponse, error) {
	return hc.get(hc.serverAddress + tagvalues_ep)
}

// Queries KairosDB using the query built using builder.
func (hc *httpClient) Query(qb builder.QueryBuilder) (*response.QueryResponse, error) {
	// Get the JSON representation of the query.
	data, err := qb.Build()
	if err != nil {
		return nil, err
	}

	return hc.postQuery(hc.serverAddress+query_ep, data)
}

// Sends metrics from the builder to the KairosDB server.
func (hc *httpClient) PushMetrics(mb builder.MetricBuilder) (*response.Response, error) {
	data, err := mb.Build()
	if err != nil {
		return nil, err
	}

	return hc.postData(hc.serverAddress+datapoints_ep, data)
}

// Deletes a metric. This is the metric and all its datapoints.
func (hc *httpClient) DeleteMetric(name string) (*response.Response, error) {
	return hc.delete(hc.serverAddress + delmetric_ep + name)
}

// TODO: Deletes data in KairosDB using the query built by the builder.
func (hc *httpClient) Delete(builder builder.QueryBuilder) (*response.Response, error) {
	return nil, nil
}

// Checks the health of the KairosDB Server.
func (hc *httpClient) HealthCheck() (*response.Response, error) {
	resp, err := hc.sendRequest(hc.serverAddress+health_ep, "GET")
	if err != nil {
		return nil, err
	}

	r := &response.Response{}
	r.SetStatusCode(resp.StatusCode)
	return r, nil
}

func (hc *httpClient) sendRequest(url, method string) (*http.Response, error) {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("accept", "application/json")
	cli := &http.Client{}

	return cli.Do(req)
}

func (hc *httpClient) httpRespToResponse(httpResp *http.Response) (*response.Response, error) {
	resp := &response.Response{}
	resp.SetStatusCode(httpResp.StatusCode)

	if httpResp.StatusCode != http.StatusNoContent {
		// If the request has failed, then read the response body.
		defer httpResp.Body.Close()
		contents, err := ioutil.ReadAll(httpResp.Body)
		if err != nil {
			return nil, err
		} else {
			// Unmarshal the contents into Response object.
			err = json.Unmarshal(contents, resp)
			if err != nil {
				return nil, err
			}
		}
	}

	return resp, nil
}

func (hc *httpClient) httpRespToQueryResponse(httpResp *http.Response) (*response.QueryResponse, error) {
	// Read the HTTP response body.
	defer httpResp.Body.Close()
	contents, err := ioutil.ReadAll(httpResp.Body)
	if err != nil {
		return nil, err
	}

	qr := response.NewQueryResponse(httpResp.StatusCode)

	// Unmarshal the contents into QueryResponse object.
	err = json.Unmarshal(contents, qr)
	if err != nil {
		return nil, err
	}

	return qr, nil
}

func (hc *httpClient) get(url string) (*response.GetResponse, error) {
	resp, err := hc.sendRequest(url, "GET")
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	} else {
		gr := response.NewGetResponse(resp.StatusCode)

		err = json.Unmarshal(contents, gr)
		if err != nil {
			return nil, err
		}

		return gr, nil
	}
}

func (hc *httpClient) postData(url string, data []byte) (*response.Response, error) {
	resp, err := http.Post(url, "application/json", bytes.NewBuffer(data))
	if err != nil {
		return nil, err
	}

	return hc.httpRespToResponse(resp)
}

func (hc *httpClient) postQuery(url string, data []byte) (*response.QueryResponse, error) {
	resp, err := http.Post(url, "application/json", bytes.NewBuffer(data))
	if err != nil {
		return nil, err
	}

	return hc.httpRespToQueryResponse(resp)
}

func (hc *httpClient) delete(url string) (*response.Response, error) {
	resp, err := hc.sendRequest(url, "DELETE")
	if err != nil {
		return nil, err
	}

	return hc.httpRespToResponse(resp)
}
<commit_msg>implement delete datapoints<commit_after>// Copyright 2016 Ajit Yagaty
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"

	"github.com/ajityagaty/go-kairosdb/builder"
	"github.com/ajityagaty/go-kairosdb/response"
)

var (
	api_version      = "/api/v1"
	datapoints_ep    = api_version + "/datapoints"
	query_ep         = api_version + "/datapoints/query"
	version_ep       = api_version + "/version"
	health_ep        = api_version + "/health/check"
	metricnames_ep   = api_version + "/metricnames"
	tagnames_ep      = api_version + "/tagnames"
	tagvalues_ep     = api_version + "/tagvalues"
	delmetric_ep     = api_version + "/metric/"
	deldatapoints_ep = api_version + "/datapoints/delete"
)

// This is the type that implements the Client interface.
type httpClient struct {
	serverAddress string
}

func NewHttpClient(serverAddress string) Client {
	return &httpClient{
		serverAddress: serverAddress,
	}
}

// Returns a list of all metrics names.
func (hc *httpClient) GetMetricNames() (*response.GetResponse, error) {
	return hc.get(hc.serverAddress + metricnames_ep)
}

// Returns a list of all tag names.
func (hc *httpClient) GetTagNames() (*response.GetResponse, error) {
	return hc.get(hc.serverAddress + tagnames_ep)
}

// Returns a list of all tag values.
func (hc *httpClient) GetTagValues() (*response.GetResponse, error) {
	return hc.get(hc.serverAddress + tagvalues_ep)
}

// Queries KairosDB using the query built using builder.
func (hc *httpClient) Query(qb builder.QueryBuilder) (*response.QueryResponse, error) {
	// Get the JSON representation of the query.
	data, err := qb.Build()
	if err != nil {
		return nil, err
	}

	return hc.postQuery(hc.serverAddress+query_ep, data)
}

// Sends metrics from the builder to the KairosDB server.
func (hc *httpClient) PushMetrics(mb builder.MetricBuilder) (*response.Response, error) {
	data, err := mb.Build()
	if err != nil {
		return nil, err
	}

	return hc.postData(hc.serverAddress+datapoints_ep, data)
}

// Deletes a metric. This is the metric and all its datapoints.
func (hc *httpClient) DeleteMetric(name string) (*response.Response, error) {
	return hc.delete(hc.serverAddress + delmetric_ep + name)
}

// TODO: Deletes data in KairosDB using the query built by the builder.
func (hc *httpClient) Delete(qb builder.QueryBuilder) (*response.Response, error) {
	data, err := qb.Build()
	if err != nil {
		return nil, err
	}

	return hc.postData(hc.serverAddress+deldatapoints_ep, data)
}

// Checks the health of the KairosDB Server.
func (hc *httpClient) HealthCheck() (*response.Response, error) {
	resp, err := hc.sendRequest(hc.serverAddress+health_ep, "GET")
	if err != nil {
		return nil, err
	}

	r := &response.Response{}
	r.SetStatusCode(resp.StatusCode)
	return r, nil
}

func (hc *httpClient) sendRequest(url, method string) (*http.Response, error) {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("accept", "application/json")
	cli := &http.Client{}

	return cli.Do(req)
}

func (hc *httpClient) httpRespToResponse(httpResp *http.Response) (*response.Response, error) {
	resp := &response.Response{}
	resp.SetStatusCode(httpResp.StatusCode)

	if httpResp.StatusCode != http.StatusNoContent {
		// If the request has failed, then read the response body.
		defer httpResp.Body.Close()
		contents, err := ioutil.ReadAll(httpResp.Body)
		if err != nil {
			return nil, err
		} else {
			// Unmarshal the contents into Response object.
			err = json.Unmarshal(contents, resp)
			if err != nil {
				return nil, err
			}
		}
	}

	return resp, nil
}

func (hc *httpClient) httpRespToQueryResponse(httpResp *http.Response) (*response.QueryResponse, error) {
	// Read the HTTP response body.
	defer httpResp.Body.Close()
	contents, err := ioutil.ReadAll(httpResp.Body)
	if err != nil {
		return nil, err
	}

	qr := response.NewQueryResponse(httpResp.StatusCode)

	// Unmarshal the contents into QueryResponse object.
	err = json.Unmarshal(contents, qr)
	if err != nil {
		return nil, err
	}

	return qr, nil
}

func (hc *httpClient) get(url string) (*response.GetResponse, error) {
	resp, err := hc.sendRequest(url, "GET")
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	} else {
		gr := response.NewGetResponse(resp.StatusCode)

		err = json.Unmarshal(contents, gr)
		if err != nil {
			return nil, err
		}

		return gr, nil
	}
}

func (hc *httpClient) postData(url string, data []byte) (*response.Response, error) {
	resp, err := http.Post(url, "application/json", bytes.NewBuffer(data))
	if err != nil {
		return nil, err
	}

	return hc.httpRespToResponse(resp)
}

func (hc *httpClient) postQuery(url string, data []byte) (*response.QueryResponse, error) {
	resp, err := http.Post(url, "application/json", bytes.NewBuffer(data))
	if err != nil {
		return nil, err
	}

	return hc.httpRespToQueryResponse(resp)
}

func (hc *httpClient) delete(url string) (*response.Response, error) {
	resp, err := hc.sendRequest(url, "DELETE")
	if err != nil {
		return nil, err
	}

	return hc.httpRespToResponse(resp)
}
<|endoftext|>