<commit_before>package models

import (
	"errors"
	// "fmt"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
	"time"
)

type Notification struct {
	// unique identifier of Notification
	Id int64 `json:"id"`

	// notification recipient account id
	AccountId int64 `json:"accountId" sql:"NOT NULL"`

	// notification content foreign key
	NotificationContentId int64 `json:"notificationContentId" sql:"NOT NULL"`

	// glanced information
	Glanced bool `json:"glanced" sql:"NOT NULL"`

	// last notifier addition time
	UpdatedAt time.Time `json:"updatedAt" sql:"NOT NULL"`
}

const (
	Notification_TYPE_SUBSCRIBE   = "subscribe"
	Notification_TYPE_UNSUBSCRIBE = "unsubscribe"
)

func (n *Notification) GetId() int64 {
	return n.Id
}

func (n Notification) TableName() string {
	return "api.notification"
}

func NewNotification() *Notification {
	return &Notification{}
}

func (n *Notification) One(q *bongo.Query) error {
	return bongo.B.One(n, n, q)
}

func (n *Notification) Create() error {
	s := map[string]interface{}{
		"account_id":              n.AccountId,
		"notification_content_id": n.NotificationContentId,
	}
	q := bongo.NewQS(s)
	if err := n.One(q); err != nil {
		if err != gorm.RecordNotFound {
			return err
		}

		return bongo.B.Create(n)
	}

	n.Glanced = false

	return bongo.B.Update(n)
}

func (n *Notification) List(q *Query) (*NotificationResponse, error) {
	if q.Limit == 0 {
		return nil, errors.New("limit cannot be zero")
	}
	response := &NotificationResponse{}
	result, err := n.getDecoratedList(q)
	if err != nil {
		return response, err
	}

	response.Notifications = result
	response.UnreadCount = getUnreadNotificationCount(result)

	return response, nil
}

func (n *Notification) Some(data interface{}, q *bongo.Query) error {

	return bongo.B.Some(n, data, q)
}

func (n *Notification) fetchByAccountId(q *Query) ([]Notification, error) {
	var notifications []Notification
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"account_id": q.AccountId,
		},
		Sort: map[string]string{
			"updated_at": "desc",
		},
		Pagination: bongo.Pagination{
			Limit: q.Limit,
		},
	}
	if err := bongo.B.Some(n, &notifications, query); err != nil {
		return nil, err
	}

	return notifications, nil
}

// getDecoratedList fetches notifications of the given user and decorates it with
// notification activity actors
func (n *Notification) getDecoratedList(q *Query) ([]NotificationContainer, error) {
	result := make([]NotificationContainer, 0)

	nList, err := n.fetchByAccountId(q)
	if err != nil {
		return nil, err
	}
	// fetch all notification content relationships
	contentIds := deductContentIds(nList)

	nc := NewNotificationContent()
	ncMap, err := nc.FetchMapByIds(contentIds)
	if err != nil {
		return nil, err
	}

	na := NewNotificationActivity()
	naMap, err := na.FetchMapByContentIds(contentIds)
	if err != nil {
		return nil, err
	}

	for _, n := range nList {
		nc := ncMap[n.NotificationContentId]
		na := naMap[n.NotificationContentId]
		container := n.buildNotificationContainer(q.AccountId, &nc, na)
		result = append(result, container)
	}

	return result, nil
}

func (n *Notification) buildNotificationContainer(actorId int64, nc *NotificationContent, na []NotificationActivity) NotificationContainer {
	ct, err := CreateNotificationContentType(nc.TypeConstant)
	if err != nil {
		return NotificationContainer{}
	}

	ct.SetTargetId(nc.TargetId)
	ct.SetListerId(actorId)
	ac, err := ct.FetchActors(na)
	if err != nil {
		return NotificationContainer{}
	}
	return NotificationContainer{
		TargetId:              nc.TargetId,
		TypeConstant:          nc.TypeConstant,
		UpdatedAt:             n.UpdatedAt,
		Glanced:               n.Glanced,
		NotificationContentId: nc.Id,
		LatestActors:          ac.LatestActors,
		ActorCount:            ac.Count,
	}
}

func deductContentIds(nList []Notification) []int64 {
	notificationContentIds := make([]int64, 0)
	for _, n := range nList {
		notificationContentIds = append(notificationContentIds, n.NotificationContentId)
	}

	return notificationContentIds
}

func (n *Notification) FetchContent() (*NotificationContent, error) {
	nc := NewNotificationContent()
	if err := nc.ById(n.NotificationContentId); err != nil {
		return nil, err
	}

	return nc, nil
}

// func (n *Notification) Follow(a *NotificationActivity) error {
// 	// a.TypeConstant = NotificationContent_TYPE_FOLLOW
// 	// create NotificationActivity
// 	if err := a.Create(); err != nil {
// 		return err
// 	}

// 	fn := NewFollowNotification()
// 	fn.NotifierId = a.ActorId
// 	// fn.TargetId = a.TargetId

// 	return CreateNotificationContent(fn)
// }

// func (n *Notification) JoinGroup(a *NotificationActivity, admins []int64) error {
// 	// a.TypeConstant = NotificationContent_TYPE_JOIN

// 	return n.interactGroup(a, admins)
// }

// func (n *Notification) LeaveGroup(a *NotificationActivity, admins []int64) error {
// 	// a.TypeConstant = NotificationContent_TYPE_LEAVE

// 	return n.interactGroup(a, admins)
// }

// func (n *Notification) interactGroup(a *NotificationActivity, admins []int64) error {
// 	gn := NewGroupNotification(a.TypeConstant)
// 	gn.NotifierId = a.ActorId
// 	gn.TargetId = a.TargetId
// 	gn.Admins = admins

// 	if err := a.Create(); err != nil {
// 		return err
// 	}

// 	return CreateNotificationContent(gn)
// }

func (n *Notification) AfterCreate() {
	bongo.B.AfterCreate(n)
}

func (n *Notification) AfterUpdate() {
	bongo.B.AfterUpdate(n)
}

func (n *Notification) AfterDelete() {
	bongo.B.AfterDelete(n)
}

func (n *Notification) Glance() error {
	selector := map[string]interface{}{
		"glanced":    false,
		"account_id": n.AccountId,
	}

	set := map[string]interface{}{
		"glanced": true,
	}

	return bongo.B.UpdateMulti(n, selector, set)
}

func getUnreadNotificationCount(notificationList []NotificationContainer) int {
	unreadCount := 0
	for _, nc := range notificationList {
		if !nc.Glanced {
			unreadCount++
		}
	}

	return unreadCount
}
<commit_msg>Social: TypeConstant and SubscribedAt fields are added<commit_after>package models

import (
	"errors"
	// "fmt"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
	"time"
)

type Notification struct {
	// unique identifier of Notification
	Id int64 `json:"id"`

	// notification recipient account id
	AccountId int64 `json:"accountId" sql:"NOT NULL"`

	// notification content foreign key
	NotificationContentId int64 `json:"notificationContentId" sql:"NOT NULL"`

	// glanced information
	Glanced bool `json:"glanced" sql:"NOT NULL"`

	// last notifier addition time
	UpdatedAt time.Time `json:"updatedAt" sql:"NOT NULL"`

	// user's subscription time to related content
	SubscribedAt time.Time `json:"subscribedAt" sql:"NOT NULL"`

	// notification type as subscribed/unsubscribed
	TypeConstant string `json:"typeConstant" sql:"NOT NULL"`
}

const (
	Notification_TYPE_SUBSCRIBE   = "subscribe"
	Notification_TYPE_UNSUBSCRIBE = "unsubscribe"
)

func (n *Notification) GetId() int64 {
	return n.Id
}

func (n Notification) TableName() string {
	return "api.notification"
}

func NewNotification() *Notification {
	return &Notification{}
}

func (n *Notification) One(q *bongo.Query) error {
	return bongo.B.One(n, n, q)
}

func (n *Notification) Create() error {
	s := map[string]interface{}{
		"account_id":              n.AccountId,
		"notification_content_id": n.NotificationContentId,
	}
	q := bongo.NewQS(s)
	if err := n.One(q); err != nil {
		if err != gorm.RecordNotFound {
			return err
		}

		return bongo.B.Create(n)
	}

	n.Glanced = false

	return bongo.B.Update(n)
}

func (n *Notification) List(q *Query) (*NotificationResponse, error) {
	if q.Limit == 0 {
		return nil, errors.New("limit cannot be zero")
	}
	response := &NotificationResponse{}
	result, err := n.getDecoratedList(q)
	if err != nil {
		return response, err
	}

	response.Notifications = result
	response.UnreadCount = getUnreadNotificationCount(result)

	return response, nil
}

func (n *Notification) Some(data interface{}, q *bongo.Query) error {

	return bongo.B.Some(n, data, q)
}

func (n *Notification) fetchByAccountId(q *Query) ([]Notification, error) {
	var notifications []Notification
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"account_id": q.AccountId,
		},
		Sort: map[string]string{
			"updated_at": "desc",
		},
		Pagination: bongo.Pagination{
			Limit: q.Limit,
		},
	}
	if err := bongo.B.Some(n, &notifications, query); err != nil {
		return nil, err
	}

	return notifications, nil
}

// getDecoratedList fetches notifications of the given user and decorates it with
// notification activity actors
func (n *Notification) getDecoratedList(q *Query) ([]NotificationContainer, error) {
	result := make([]NotificationContainer, 0)

	nList, err := n.fetchByAccountId(q)
	if err != nil {
		return nil, err
	}
	// fetch all notification content relationships
	contentIds := deductContentIds(nList)

	nc := NewNotificationContent()
	ncMap, err := nc.FetchMapByIds(contentIds)
	if err != nil {
		return nil, err
	}

	na := NewNotificationActivity()
	naMap, err := na.FetchMapByContentIds(contentIds)
	if err != nil {
		return nil, err
	}

	for _, n := range nList {
		nc := ncMap[n.NotificationContentId]
		na := naMap[n.NotificationContentId]
		container := n.buildNotificationContainer(q.AccountId, &nc, na)
		result = append(result, container)
	}

	return result, nil
}

func (n *Notification) buildNotificationContainer(actorId int64, nc *NotificationContent, na []NotificationActivity) NotificationContainer {
	ct, err := CreateNotificationContentType(nc.TypeConstant)
	if err != nil {
		return NotificationContainer{}
	}

	ct.SetTargetId(nc.TargetId)
	ct.SetListerId(actorId)
	ac, err := ct.FetchActors(na)
	if err != nil {
		return NotificationContainer{}
	}
	return NotificationContainer{
		TargetId:              nc.TargetId,
		TypeConstant:          nc.TypeConstant,
		UpdatedAt:             n.UpdatedAt,
		Glanced:               n.Glanced,
		NotificationContentId: nc.Id,
		LatestActors:          ac.LatestActors,
		ActorCount:            ac.Count,
	}
}

func deductContentIds(nList []Notification) []int64 {
	notificationContentIds := make([]int64, 0)
	for _, n := range nList {
		notificationContentIds = append(notificationContentIds, n.NotificationContentId)
	}

	return notificationContentIds
}

func (n *Notification) FetchContent() (*NotificationContent, error) {
	nc := NewNotificationContent()
	if err := nc.ById(n.NotificationContentId); err != nil {
		return nil, err
	}

	return nc, nil
}

// func (n *Notification) Follow(a *NotificationActivity) error {
// 	// a.TypeConstant = NotificationContent_TYPE_FOLLOW
// 	// create NotificationActivity
// 	if err := a.Create(); err != nil {
// 		return err
// 	}

// 	fn := NewFollowNotification()
// 	fn.NotifierId = a.ActorId
// 	// fn.TargetId = a.TargetId

// 	return CreateNotificationContent(fn)
// }

// func (n *Notification) JoinGroup(a *NotificationActivity, admins []int64) error {
// 	// a.TypeConstant = NotificationContent_TYPE_JOIN

// 	return n.interactGroup(a, admins)
// }

// func (n *Notification) LeaveGroup(a *NotificationActivity, admins []int64) error {
// 	// a.TypeConstant = NotificationContent_TYPE_LEAVE

// 	return n.interactGroup(a, admins)
// }

// func (n *Notification) interactGroup(a *NotificationActivity, admins []int64) error {
// 	gn := NewGroupNotification(a.TypeConstant)
// 	gn.NotifierId = a.ActorId
// 	gn.TargetId = a.TargetId
// 	gn.Admins = admins

// 	if err := a.Create(); err != nil {
// 		return err
// 	}

// 	return CreateNotificationContent(gn)
// }

func (n *Notification) BeforeCreate() {
	if n.TypeConstant == "" {
		n.TypeConstant = Notification_TYPE_SUBSCRIBE
	}
}

func (n *Notification) AfterCreate() {
	bongo.B.AfterCreate(n)
}

func (n *Notification) AfterUpdate() {
	bongo.B.AfterUpdate(n)
}

func (n *Notification) AfterDelete() {
	bongo.B.AfterDelete(n)
}

func (n *Notification) Glance() error {
	selector := map[string]interface{}{
		"glanced":    false,
		"account_id": n.AccountId,
	}

	set := map[string]interface{}{
		"glanced": true,
	}

	return bongo.B.UpdateMulti(n, selector, set)
}

func getUnreadNotificationCount(notificationList []NotificationContainer) int {
	unreadCount := 0
	for _, nc := range notificationList {
		if !nc.Glanced {
			unreadCount++
		}
	}

	return unreadCount
}
<|endoftext|>


<commit_before>package vreplication

var (
	initialProductSchema = `
create table product(pid int, description varbinary(128), primary key(pid));
create table customer(cid int, name varbinary(128), typ enum('individual','soho','enterprise'), primary key(cid));
create table merchant(mname varchar(128), category varchar(128), primary key(mname)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
create table orders(oid int, cid int, pid int, mname varchar(128), price int, primary key(oid));
create table customer_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
create table order_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
`

	initialProductVSchema = `
{
  "tables": {
	"product": {},
	"customer": {},
	"merchant": {},
	"orders": {},
	"customer_seq": {
		"type": "sequence"
	},
	"order_seq": {
		"type": "sequence"
	}
  }
}
`
	customerSchema  = ""
	customerVSchema = `
{
  "sharded": true,
  "vindexes": {
	    "reverse_bits": {
	      "type": "reverse_bits"
	    }
	  },
   "tables": {
	    "customer": {
	      "column_vindexes": [
	        {
	          "column": "cid",
	          "name": "reverse_bits"
	        }
	      ],
	      "auto_increment": {
	        "column": "cid",
	        "sequence": "customer_seq"
	      }
	    }
   }

}
`
	merchantVSchema = `
{
  "sharded": true,
  "vindexes": {
    "md5": {
      "type": "unicode_loose_md5"
    }
  },
  "tables": {
    "merchant": {
      "column_vindexes": [
        {
          "column": "mname",
          "name": "md5"
        }
      ]
    }
  }
}
`
	//ordersSchema = "create table order_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';"
	ordersVSchema = `
{
  "sharded": true,
  "vindexes": {
    "reverse_bits": {
      "type": "reverse_bits"
    }
  },
  "tables": {
	"customer": {
	      "column_vindexes": [
	        {
	          "column": "cid",
	          "name": "reverse_bits"
	        }
	      ],
	      "auto_increment": {
	        "column": "cid",
	        "sequence": "customer_seq"
	      }
	},
    "orders": {
      "column_vindexes": [
        {
          "column": "oid",
          "name": "reverse_bits"
        }
      ],
      "auto_increment": {
        "column": "oid",
        "sequence": "order_seq"
      }

    }
  }
}
`
	materializeProductVSchema = `
{
  "sharded": true,
  "vindexes": {
    "reverse_bits": {
      "type": "reverse_bits"
    }
  },
  "tables": {
	"customer": {
	      "column_vindexes": [
	        {
	          "column": "cid",
	          "name": "reverse_bits"
	        }
	      ],
	      "auto_increment": {
	        "column": "cid",
	        "sequence": "customer_seq"
	      }
	},
    "orders": {
      "column_vindexes": [
        {
          "column": "oid",
          "name": "reverse_bits"
        }
      ],
      "auto_increment": {
        "column": "oid",
        "sequence": "order_seq"
      }

    },
	"cproduct": {
		"type": "reference"
	},
	"vproduct": {
		"type": "reference"
	}
  }
}
`
	materializeProductSpec = `
	{
	"workflow": "cproduct",
	"sourceKeyspace": "product",
	"targetKeyspace": "customer",
	"tableSettings": [{
		"targetTable": "cproduct",
		"sourceExpression": "select * from product",
		"create_ddl": "create table cproduct(pid bigint, description varchar(128), primary key(pid))"
	}]
}
`

	merchantOrdersVSchema = `
{
	  "sharded": true,
	  "vindexes": {
		"md5": {
		  "type": "unicode_loose_md5"
		}
	  },
	  "tables": {
		"merchant": {
		  "column_vindexes": [
			{
			  "column": "mname",
			  "name": "md5"
			}
		  ]
      	},
	  	"morders": {
		  "column_vindexes": [
			{
			  "column": "mname",
			  "name": "md5"
			}
		  ],
		  "auto_increment": {
			"column": "oid",
			"sequence": "order_seq"
		  }
	  	},
	  	"msales": {
		  "column_vindexes": [
			{
			  "column": "merchant_name",
			  "name": "md5"
			}
		  ]
	  	}
      }
}
`
	materializeMerchantOrdersSpec = `
{
  "workflow": "morders",
  "sourceKeyspace": "customer",
  "targetKeyspace": "merchant",
  "tableSettings": [{
    "targetTable": "morders",
    "sourceExpression": "select * from orders",
    "create_ddl": "create table morders(oid int, cid int, mname varchar(128), pid int, price int, primary key(oid))"
  }]
}
`

	materializeMerchantSalesSpec = `
{
  "workflow": "msales",
  "sourceKeyspace": "customer",
  "targetKeyspace": "merchant",
  "tableSettings": [{
    "targetTable": "msales",
	"sourceExpression": "select mname as merchant_name, count(*) as kount, sum(price) as amount from orders group by merchant_name",
    "create_ddl": "create table msales(merchant_name varchar(128), kount int, amount int, primary key(merchant_name))"
  }]
}
`

	materializeSalesVSchema = `
{
  "tables": {
    "product": {},
    "sales": {},
    "customer_seq": {
      "type": "sequence"
    },
    "order_seq": {
      "type": "sequence"
    }
  }
}
`
	materializeSalesSpec = `
{
  "workflow": "sales",
  "sourceKeyspace": "customer",
  "targetKeyspace": "product",
  "tableSettings": [{
    "targetTable": "sales",
    "sourceExpression": "select pid, count(*) as kount, sum(price) as amount from orders group by pid",
    "create_ddl": "create table sales(pid int, kount int, amount int, primary key(pid))"
  }]
}
`
	materializeRollupSpec = `
{
  "workflow": "rollup",
  "sourceKeyspace": "product",
  "targetKeyspace": "product",
  "tableSettings": [{
    "targetTable": "rollup",
    "sourceExpression": "select 'total' as rollupname, count(*) as kount from product group by rollupname",
    "create_ddl": "create table rollup(rollupname varchar(100), kount int, primary key (rollupname))"
  }]
}
`
)
<commit_msg>Add a timestamp column to vrepl e2e to test a complete workflow with timestamps<commit_after>package vreplication

var (
	initialProductSchema = `
create table product(pid int, description varbinary(128), primary key(pid));
create table customer(cid int, name varbinary(128), typ enum('individual','soho','enterprise'), ts timestamp not null default current_timestamp, primary key(cid));
create table merchant(mname varchar(128), category varchar(128), primary key(mname)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
create table orders(oid int, cid int, pid int, mname varchar(128), price int, primary key(oid));
create table customer_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
create table order_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
`

	initialProductVSchema = `
{
  "tables": {
	"product": {},
	"customer": {},
	"merchant": {},
	"orders": {},
	"customer_seq": {
		"type": "sequence"
	},
	"order_seq": {
		"type": "sequence"
	}
  }
}
`
	customerSchema  = ""
	customerVSchema = `
{
  "sharded": true,
  "vindexes": {
	    "reverse_bits": {
	      "type": "reverse_bits"
	    }
	  },
   "tables": {
	    "customer": {
	      "column_vindexes": [
	        {
	          "column": "cid",
	          "name": "reverse_bits"
	        }
	      ],
	      "auto_increment": {
	        "column": "cid",
	        "sequence": "customer_seq"
	      }
	    }
   }

}
`
	merchantVSchema = `
{
  "sharded": true,
  "vindexes": {
    "md5": {
      "type": "unicode_loose_md5"
    }
  },
  "tables": {
    "merchant": {
      "column_vindexes": [
        {
          "column": "mname",
          "name": "md5"
        }
      ]
    }
  }
}
`
	//ordersSchema = "create table order_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';"
	ordersVSchema = `
{
  "sharded": true,
  "vindexes": {
    "reverse_bits": {
      "type": "reverse_bits"
    }
  },
  "tables": {
	"customer": {
	      "column_vindexes": [
	        {
	          "column": "cid",
	          "name": "reverse_bits"
	        }
	      ],
	      "auto_increment": {
	        "column": "cid",
	        "sequence": "customer_seq"
	      }
	},
    "orders": {
      "column_vindexes": [
        {
          "column": "oid",
          "name": "reverse_bits"
        }
      ],
      "auto_increment": {
        "column": "oid",
        "sequence": "order_seq"
      }

    }
  }
}
`
	materializeProductVSchema = `
{
  "sharded": true,
  "vindexes": {
    "reverse_bits": {
      "type": "reverse_bits"
    }
  },
  "tables": {
	"customer": {
	      "column_vindexes": [
	        {
	          "column": "cid",
	          "name": "reverse_bits"
	        }
	      ],
	      "auto_increment": {
	        "column": "cid",
	        "sequence": "customer_seq"
	      }
	},
    "orders": {
      "column_vindexes": [
        {
          "column": "oid",
          "name": "reverse_bits"
        }
      ],
      "auto_increment": {
        "column": "oid",
        "sequence": "order_seq"
      }

    },
	"cproduct": {
		"type": "reference"
	},
	"vproduct": {
		"type": "reference"
	}
  }
}
`
	materializeProductSpec = `
	{
	"workflow": "cproduct",
	"sourceKeyspace": "product",
	"targetKeyspace": "customer",
	"tableSettings": [{
		"targetTable": "cproduct",
		"sourceExpression": "select * from product",
		"create_ddl": "create table cproduct(pid bigint, description varchar(128), primary key(pid))"
	}]
}
`

	merchantOrdersVSchema = `
{
	  "sharded": true,
	  "vindexes": {
		"md5": {
		  "type": "unicode_loose_md5"
		}
	  },
	  "tables": {
		"merchant": {
		  "column_vindexes": [
			{
			  "column": "mname",
			  "name": "md5"
			}
		  ]
      	},
	  	"morders": {
		  "column_vindexes": [
			{
			  "column": "mname",
			  "name": "md5"
			}
		  ],
		  "auto_increment": {
			"column": "oid",
			"sequence": "order_seq"
		  }
	  	},
	  	"msales": {
		  "column_vindexes": [
			{
			  "column": "merchant_name",
			  "name": "md5"
			}
		  ]
	  	}
      }
}
`
	materializeMerchantOrdersSpec = `
{
  "workflow": "morders",
  "sourceKeyspace": "customer",
  "targetKeyspace": "merchant",
  "tableSettings": [{
    "targetTable": "morders",
    "sourceExpression": "select * from orders",
    "create_ddl": "create table morders(oid int, cid int, mname varchar(128), pid int, price int, primary key(oid))"
  }]
}
`

	materializeMerchantSalesSpec = `
{
  "workflow": "msales",
  "sourceKeyspace": "customer",
  "targetKeyspace": "merchant",
  "tableSettings": [{
    "targetTable": "msales",
	"sourceExpression": "select mname as merchant_name, count(*) as kount, sum(price) as amount from orders group by merchant_name",
    "create_ddl": "create table msales(merchant_name varchar(128), kount int, amount int, primary key(merchant_name))"
  }]
}
`

	materializeSalesVSchema = `
{
  "tables": {
    "product": {},
    "sales": {},
    "customer_seq": {
      "type": "sequence"
    },
    "order_seq": {
      "type": "sequence"
    }
  }
}
`
	materializeSalesSpec = `
{
  "workflow": "sales",
  "sourceKeyspace": "customer",
  "targetKeyspace": "product",
  "tableSettings": [{
    "targetTable": "sales",
    "sourceExpression": "select pid, count(*) as kount, sum(price) as amount from orders group by pid",
    "create_ddl": "create table sales(pid int, kount int, amount int, primary key(pid))"
  }]
}
`
	materializeRollupSpec = `
{
  "workflow": "rollup",
  "sourceKeyspace": "product",
  "targetKeyspace": "product",
  "tableSettings": [{
    "targetTable": "rollup",
    "sourceExpression": "select 'total' as rollupname, count(*) as kount from product group by rollupname",
    "create_ddl": "create table rollup(rollupname varchar(100), kount int, primary key (rollupname))"
  }]
}
`
)
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"net/url"
	"os"
	"strings"
	"testing"

	"github.com/docker/api/azure"
	"github.com/docker/api/azure/login"

	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources"
	azure_storage "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/storage/mgmt/storage"
	"github.com/Azure/azure-storage-file-go/azfile"
	"github.com/Azure/go-autorest/autorest/to"
	. "github.com/onsi/gomega"
	log "github.com/sirupsen/logrus"
	"github.com/stretchr/testify/suite"

	"github.com/docker/api/context/store"
	"github.com/docker/api/tests/aci-e2e/storage"
	. "github.com/docker/api/tests/framework"
)

const (
	resourceGroupName = "resourceGroupTest"
	location          = "westeurope"
	contextName       = "acitest"

	testContainerName = "testcontainername"
)

var (
	subscriptionID string
)

type E2eACISuite struct {
	Suite
}

func (s *E2eACISuite) TestContextDefault() {
	It("should be initialized with default context", func() {
		_, err := s.NewCommand("docker", "context", "rm", "-f", contextName).Exec()
		if err == nil {
			log.Println("Cleaning existing test context")
		}

		s.NewCommand("docker", "context", "use", "default").ExecOrDie()
		output := s.NewCommand("docker", "context", "ls").ExecOrDie()
		Expect(output).To(Not(ContainSubstring(contextName)))
		Expect(output).To(ContainSubstring("default *"))
	})
}

func (s *E2eACISuite) TestACIBackend() {
	It("Logs in azure using service principal credentials", func() {
		login, err := login.NewAzureLoginService()
		Expect(err).To(BeNil())
		// in order to create new service principal and get these 3 values : `az ad sp create-for-rbac --name 'TestServicePrincipal' --sdk-auth`
		clientID := os.Getenv("AZURE_CLIENT_ID")
		clientSecret := os.Getenv("AZURE_CLIENT_SECRET")
		tenantID := os.Getenv("AZURE_TENANT_ID")
		err = login.TestLoginFromServicePrincipal(clientID, clientSecret, tenantID)
		Expect(err).To(BeNil())
	})

	It("creates a new aci context for tests", func() {
		setupTestResourceGroup(resourceGroupName)
		helper := azure.NewACIResourceGroupHelper()
		models, err := helper.GetSubscriptionIDs(context.TODO())
		Expect(err).To(BeNil())
		subscriptionID = *models[0].SubscriptionID

		s.NewDockerCommand("context", "create", "aci", contextName, "--subscription-id", subscriptionID, "--resource-group", resourceGroupName, "--location", location).ExecOrDie()
		// Expect(output).To(ContainSubstring("ACI context acitest created"))
	})

	defer deleteResourceGroup(resourceGroupName)

	It("uses the aci context", func() {
		currentContext := s.NewCommand("docker", "context", "use", contextName).ExecOrDie()
		Expect(currentContext).To(ContainSubstring(contextName))
		output := s.NewCommand("docker", "context", "ls").ExecOrDie()
		Expect(output).To(ContainSubstring("acitest *"))
	})

	It("ensures no container is running initially", func() {
		output := s.NewDockerCommand("ps").ExecOrDie()
		Expect(len(Lines(output))).To(Equal(1))
	})

	It("runs nginx on port 80", func() {
		aciContext := store.AciContext{
			SubscriptionID: subscriptionID,
			Location:       location,
			ResourceGroup:  resourceGroupName,
		}
		createStorageAccount(aciContext, testStorageAccountName)
		defer deleteStorageAccount(aciContext)
		keys := getStorageKeys(aciContext, testStorageAccountName)
		firstKey := *keys[0].Value
		credential, u := createFileShare(firstKey, testShareName)
		uploadFile(credential, u.String(), testFileName, testFileContent)

		mountTarget := "/usr/share/nginx/html"
		output := s.NewDockerCommand("run", "nginx",
			"-v", fmt.Sprintf("%s:%s@%s:%s",
				testStorageAccountName, firstKey, testShareName, mountTarget),
			"-p", "80:80",
			"--name", testContainerName).ExecOrDie()
		Expect(output).To(ContainSubstring(testContainerName))
		output = s.NewDockerCommand("ps").ExecOrDie()
		lines := Lines(output)
		Expect(len(lines)).To(Equal(2))

		containerFields := Columns(lines[1])
		Expect(containerFields[1]).To(Equal("nginx"))
		Expect(containerFields[2]).To(Equal("Running"))
		exposedIP := containerFields[3]
		containerID := containerFields[0]
		Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))

		publishedURL := strings.ReplaceAll(exposedIP, "->80/tcp", "")
		output = s.NewCommand("curl", publishedURL).ExecOrDie()
		Expect(output).To(ContainSubstring(testFileContent))

		output = s.NewDockerCommand("logs", containerID).ExecOrDie()
		Expect(output).To(ContainSubstring("GET"))
	})

	It("removes container nginx", func() {
		output := s.NewDockerCommand("rm", testContainerName).ExecOrDie()
		Expect(Lines(output)[0]).To(Equal(testContainerName))
	})

	var exposedURL string
	const composeFile = "../composefiles/aci-demo/aci_demo_port.yaml"
	const composeFileMultiplePorts = "../composefiles/aci-demo/aci_demo_multi_port.yaml"
	const serverContainer = "acidemo_web"
	const wordsContainer = "acidemo_words"
	It("deploys a compose app", func() {
		s.NewDockerCommand("compose", "up", "-f", composeFile, "--project-name", "acidemo").ExecOrDie()
		// Expect(output).To(ContainSubstring("Successfully deployed"))
		output := s.NewDockerCommand("ps").ExecOrDie()
		Lines := Lines(output)
		Expect(len(Lines)).To(Equal(4))
		webChecked := false

		for _, line := range Lines[1:] {
			Expect(line).To(ContainSubstring("Running"))
			if strings.Contains(line, serverContainer) {
				webChecked = true
				containerFields := Columns(line)
				exposedIP := containerFields[3]
				Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))

				exposedURL = strings.ReplaceAll(exposedIP, "->80/tcp", "")
				output = s.NewCommand("curl", exposedURL).ExecOrDie()
				Expect(output).To(ContainSubstring("Docker Compose demo"))
				output = s.NewCommand("curl", exposedURL+"/words/noun").ExecOrDie()
				Expect(output).To(ContainSubstring("\"word\":"))
			}
		}

		Expect(webChecked).To(BeTrue())
	})

	It("get logs from web service", func() {
		output := s.NewDockerCommand("logs", serverContainer).ExecOrDie()
		Expect(output).To(ContainSubstring("Listening on port 80"))
	})

	It("updates a compose app", func() {
		s.NewDockerCommand("compose", "up", "-f", composeFileMultiplePorts, "--project-name", "acidemo").ExecOrDie()
		// Expect(output).To(ContainSubstring("Successfully deployed"))
		output := s.NewDockerCommand("ps").ExecOrDie()
		Lines := Lines(output)
		Expect(len(Lines)).To(Equal(4))
		webChecked := false
		wordsChecked := false

		for _, line := range Lines[1:] {
			Expect(line).To(ContainSubstring("Running"))
			if strings.Contains(line, serverContainer) {
				webChecked = true
				containerFields := Columns(line)
				exposedIP := containerFields[3]
				Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))

				url := strings.ReplaceAll(exposedIP, "->80/tcp", "")
				Expect(exposedURL).To(Equal(url))
			}
			if strings.Contains(line, wordsContainer) {
				wordsChecked = true
				containerFields := Columns(line)
				exposedIP := containerFields[3]
				Expect(exposedIP).To(ContainSubstring(":8080->8080/tcp"))

				url := strings.ReplaceAll(exposedIP, "->8080/tcp", "")
				output = s.NewCommand("curl", url+"/noun").ExecOrDie()
				Expect(output).To(ContainSubstring("\"word\":"))
			}
		}

		Expect(webChecked).To(BeTrue())
		Expect(wordsChecked).To(BeTrue())
	})

	It("shutdown compose app", func() {
		s.NewDockerCommand("compose", "down", "-f", composeFile, "--project-name", "acidemo").ExecOrDie()
	})
	It("switches back to default context", func() {
		output := s.NewCommand("docker", "context", "use", "default").ExecOrDie()
		Expect(output).To(ContainSubstring("default"))
	})

	It("deletes test context", func() {
		output := s.NewCommand("docker", "context", "rm", contextName).ExecOrDie()
		Expect(output).To(ContainSubstring(contextName))
	})
}

const (
	testStorageAccountName = "dockertestaccount"
	testShareName          = "dockertestshare"
	testFileContent        = "Volume mounted with success!"
	testFileName           = "index.html"
)

func createStorageAccount(aciContext store.AciContext, accountName string) azure_storage.Account {
	log.Println("Creating storage account " + accountName)
	storageAccount, err := storage.CreateStorageAccount(context.TODO(), aciContext, accountName)
	Expect(err).To(BeNil())
	Expect(*storageAccount.Name).To(Equal(accountName))
	return storageAccount
}

func getStorageKeys(aciContext store.AciContext, storageAccountName string) []azure_storage.AccountKey {
	list, err := storage.ListKeys(context.TODO(), aciContext, storageAccountName)
	Expect(err).To(BeNil())
	Expect(list.Keys).ToNot(BeNil())
	Expect(len(*list.Keys)).To(BeNumerically(">", 0))

	return *list.Keys
}

func deleteStorageAccount(aciContext store.AciContext) {
	log.Println("Deleting storage account " + testStorageAccountName)
	_, err := storage.DeleteStorageAccount(context.TODO(), aciContext, testStorageAccountName)
	Expect(err).To(BeNil())
}

func createFileShare(key, shareName string) (azfile.SharedKeyCredential, url.URL) {
	// Create a ShareURL object that wraps a soon-to-be-created share's URL and a default pipeline.
	u, _ := url.Parse(fmt.Sprintf("https://%s.file.core.windows.net/%s", testStorageAccountName, shareName))
	credential, err := azfile.NewSharedKeyCredential(testStorageAccountName, key)
	Expect(err).To(BeNil())

	shareURL := azfile.NewShareURL(*u, azfile.NewPipeline(credential, azfile.PipelineOptions{}))
	_, err = shareURL.Create(context.TODO(), azfile.Metadata{}, 0)
	Expect(err).To(BeNil())

	return *credential, *u
}

func uploadFile(credential azfile.SharedKeyCredential, baseURL, fileName, fileContent string) {
	fURL, err := url.Parse(baseURL + "/" + fileName)
	Expect(err).To(BeNil())
	fileURL := azfile.NewFileURL(*fURL, azfile.NewPipeline(&credential, azfile.PipelineOptions{}))
	err = azfile.UploadBufferToAzureFile(context.TODO(), []byte(fileContent), fileURL, azfile.UploadToAzureFileOptions{})
	Expect(err).To(BeNil())
}

func TestE2eACI(t *testing.T) {
	suite.Run(t, new(E2eACISuite))
}

func setupTestResourceGroup(groupName string) {
	log.Println("Creating resource group " + resourceGroupName)
	ctx := context.TODO()
	helper := azure.NewACIResourceGroupHelper()
	models, err := helper.GetSubscriptionIDs(ctx)
	Expect(err).To(BeNil())
	_, err = helper.CreateOrUpdate(ctx, *models[0].SubscriptionID, groupName, resources.Group{
		Location: to.StringPtr(location),
	})
	Expect(err).To(BeNil())
}

func deleteResourceGroup(groupName string) {
	log.Println("Deleting resource group " + resourceGroupName)
	ctx := context.TODO()
	helper := azure.NewACIResourceGroupHelper()
	models, err := helper.GetSubscriptionIDs(ctx)
	Expect(err).To(BeNil())
	err = helper.Delete(ctx, *models[0].SubscriptionID, groupName)
	Expect(err).To(BeNil())
}
<commit_msg>Random ACI group name for tests, otherwise we are conflicting with each other + the CI.  Same for storage account name (outside of the scope of resource group)<commit_after>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"math/rand"
	"net/url"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources"
	azure_storage "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/storage/mgmt/storage"
	"github.com/Azure/azure-storage-file-go/azfile"
	"github.com/Azure/go-autorest/autorest/to"
	. "github.com/onsi/gomega"
	log "github.com/sirupsen/logrus"
	"github.com/stretchr/testify/suite"

	"github.com/docker/api/azure"
	"github.com/docker/api/azure/login"
	"github.com/docker/api/context/store"
	"github.com/docker/api/tests/aci-e2e/storage"
	. "github.com/docker/api/tests/framework"
)

const (
	location          = "westeurope"
	contextName       = "acitest"
	testContainerName = "testcontainername"
	testShareName     = "dockertestshare"
	testFileContent   = "Volume mounted with success!"
	testFileName      = "index.html"
)

var (
	subscriptionID         string
	resourceGroupName      = "resourceGroupTestE2E-" + RandStringBytes(10)
	testStorageAccountName = "storageteste2e" + RandStringBytes(6) // "between 3 and 24 characters in length and use numbers and lower-case letters only"
)

type E2eACISuite struct {
	Suite
}

func (s *E2eACISuite) TestContextDefault() {
	It("should be initialized with default context", func() {
		_, err := s.NewCommand("docker", "context", "rm", "-f", contextName).Exec()
		if err == nil {
			log.Println("Cleaning existing test context")
		}

		s.NewCommand("docker", "context", "use", "default").ExecOrDie()
		output := s.NewCommand("docker", "context", "ls").ExecOrDie()
		Expect(output).To(Not(ContainSubstring(contextName)))
		Expect(output).To(ContainSubstring("default *"))
	})
}

func (s *E2eACISuite) TestACIBackend() {
	It("Logs in azure using service principal credentials", func() {
		login, err := login.NewAzureLoginService()
		Expect(err).To(BeNil())
		// in order to create new service principal and get these 3 values : `az ad sp create-for-rbac --name 'TestServicePrincipal' --sdk-auth`
		clientID := os.Getenv("AZURE_CLIENT_ID")
		clientSecret := os.Getenv("AZURE_CLIENT_SECRET")
		tenantID := os.Getenv("AZURE_TENANT_ID")
		err = login.TestLoginFromServicePrincipal(clientID, clientSecret, tenantID)
		Expect(err).To(BeNil())
	})

	It("creates a new aci context for tests", func() {
		setupTestResourceGroup(resourceGroupName)
		helper := azure.NewACIResourceGroupHelper()
		models, err := helper.GetSubscriptionIDs(context.TODO())
		Expect(err).To(BeNil())
		subscriptionID = *models[0].SubscriptionID

		s.NewDockerCommand("context", "create", "aci", contextName, "--subscription-id", subscriptionID, "--resource-group", resourceGroupName, "--location", location).ExecOrDie()
		// Expect(output).To(ContainSubstring("ACI context acitest created"))
	})

	defer deleteResourceGroup(resourceGroupName)

	It("uses the aci context", func() {
		currentContext := s.NewCommand("docker", "context", "use", contextName).ExecOrDie()
		Expect(currentContext).To(ContainSubstring(contextName))
		output := s.NewCommand("docker", "context", "ls").ExecOrDie()
		Expect(output).To(ContainSubstring("acitest *"))
	})

	It("ensures no container is running initially", func() {
		output := s.NewDockerCommand("ps").ExecOrDie()
		Expect(len(Lines(output))).To(Equal(1))
	})

	It("runs nginx on port 80", func() {
		aciContext := store.AciContext{
			SubscriptionID: subscriptionID,
			Location:       location,
			ResourceGroup:  resourceGroupName,
		}
		createStorageAccount(aciContext, testStorageAccountName)
		defer deleteStorageAccount(aciContext)
		keys := getStorageKeys(aciContext, testStorageAccountName)
		firstKey := *keys[0].Value
		credential, u := createFileShare(firstKey, testShareName)
		uploadFile(credential, u.String(), testFileName, testFileContent)

		mountTarget := "/usr/share/nginx/html"
		output := s.NewDockerCommand("run", "nginx",
			"-v", fmt.Sprintf("%s:%s@%s:%s",
				testStorageAccountName, firstKey, testShareName, mountTarget),
			"-p", "80:80",
			"--name", testContainerName).ExecOrDie()
		Expect(output).To(ContainSubstring(testContainerName))
		output = s.NewDockerCommand("ps").ExecOrDie()
		lines := Lines(output)
		Expect(len(lines)).To(Equal(2))

		containerFields := Columns(lines[1])
		Expect(containerFields[1]).To(Equal("nginx"))
		Expect(containerFields[2]).To(Equal("Running"))
		exposedIP := containerFields[3]
		containerID := containerFields[0]
		Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))

		publishedURL := strings.ReplaceAll(exposedIP, "->80/tcp", "")
		output = s.NewCommand("curl", publishedURL).ExecOrDie()
		Expect(output).To(ContainSubstring(testFileContent))

		output = s.NewDockerCommand("logs", containerID).ExecOrDie()
		Expect(output).To(ContainSubstring("GET"))
	})

	It("removes container nginx", func() {
		output := s.NewDockerCommand("rm", testContainerName).ExecOrDie()
		Expect(Lines(output)[0]).To(Equal(testContainerName))
	})

	var exposedURL string
	const composeFile = "../composefiles/aci-demo/aci_demo_port.yaml"
	const composeFileMultiplePorts = "../composefiles/aci-demo/aci_demo_multi_port.yaml"
	const serverContainer = "acidemo_web"
	const wordsContainer = "acidemo_words"
	It("deploys a compose app", func() {
		s.NewDockerCommand("compose", "up", "-f", composeFile, "--project-name", "acidemo").ExecOrDie()
		// Expect(output).To(ContainSubstring("Successfully deployed"))
		output := s.NewDockerCommand("ps").ExecOrDie()
		Lines := Lines(output)
		Expect(len(Lines)).To(Equal(4))
		webChecked := false

		for _, line := range Lines[1:] {
			Expect(line).To(ContainSubstring("Running"))
			if strings.Contains(line, serverContainer) {
				webChecked = true
				containerFields := Columns(line)
				exposedIP := containerFields[3]
				Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))

				exposedURL = strings.ReplaceAll(exposedIP, "->80/tcp", "")
				output = s.NewCommand("curl", exposedURL).ExecOrDie()
				Expect(output).To(ContainSubstring("Docker Compose demo"))
				output = s.NewCommand("curl", exposedURL+"/words/noun").ExecOrDie()
				Expect(output).To(ContainSubstring("\"word\":"))
			}
		}

		Expect(webChecked).To(BeTrue())
	})

	It("get logs from web service", func() {
		output := s.NewDockerCommand("logs", serverContainer).ExecOrDie()
		Expect(output).To(ContainSubstring("Listening on port 80"))
	})

	It("updates a compose app", func() {
		s.NewDockerCommand("compose", "up", "-f", composeFileMultiplePorts, "--project-name", "acidemo").ExecOrDie()
		// Expect(output).To(ContainSubstring("Successfully deployed"))
		output := s.NewDockerCommand("ps").ExecOrDie()
		Lines := Lines(output)
		Expect(len(Lines)).To(Equal(4))
		webChecked := false
		wordsChecked := false

		for _, line := range Lines[1:] {
			Expect(line).To(ContainSubstring("Running"))
			if strings.Contains(line, serverContainer) {
				webChecked = true
				containerFields := Columns(line)
				exposedIP := containerFields[3]
				Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))

				url := strings.ReplaceAll(exposedIP, "->80/tcp", "")
				Expect(exposedURL).To(Equal(url))
			}
			if strings.Contains(line, wordsContainer) {
				wordsChecked = true
				containerFields := Columns(line)
				exposedIP := containerFields[3]
				Expect(exposedIP).To(ContainSubstring(":8080->8080/tcp"))

				url := strings.ReplaceAll(exposedIP, "->8080/tcp", "")
				output = s.NewCommand("curl", url+"/noun").ExecOrDie()
				Expect(output).To(ContainSubstring("\"word\":"))
			}
		}

		Expect(webChecked).To(BeTrue())
		Expect(wordsChecked).To(BeTrue())
	})

	It("shutdown compose app", func() {
		s.NewDockerCommand("compose", "down", "-f", composeFile, "--project-name", "acidemo").ExecOrDie()
	})
	It("switches back to default context", func() {
		output := s.NewCommand("docker", "context", "use", "default").ExecOrDie()
		Expect(output).To(ContainSubstring("default"))
	})

	It("deletes test context", func() {
		output := s.NewCommand("docker", "context", "rm", contextName).ExecOrDie()
		Expect(output).To(ContainSubstring(contextName))
	})
}

func createStorageAccount(aciContext store.AciContext, accountName string) azure_storage.Account {
	log.Println("Creating storage account " + accountName)
	storageAccount, err := storage.CreateStorageAccount(context.TODO(), aciContext, accountName)
	Expect(err).To(BeNil())
	Expect(*storageAccount.Name).To(Equal(accountName))
	return storageAccount
}

func getStorageKeys(aciContext store.AciContext, storageAccountName string) []azure_storage.AccountKey {
	list, err := storage.ListKeys(context.TODO(), aciContext, storageAccountName)
	Expect(err).To(BeNil())
	Expect(list.Keys).ToNot(BeNil())
	Expect(len(*list.Keys)).To(BeNumerically(">", 0))

	return *list.Keys
}

func deleteStorageAccount(aciContext store.AciContext) {
	log.Println("Deleting storage account " + testStorageAccountName)
	_, err := storage.DeleteStorageAccount(context.TODO(), aciContext, testStorageAccountName)
	Expect(err).To(BeNil())
}

func createFileShare(key, shareName string) (azfile.SharedKeyCredential, url.URL) {
	// Create a ShareURL object that wraps a soon-to-be-created share's URL and a default pipeline.
	u, _ := url.Parse(fmt.Sprintf("https://%s.file.core.windows.net/%s", testStorageAccountName, shareName))
	credential, err := azfile.NewSharedKeyCredential(testStorageAccountName, key)
	Expect(err).To(BeNil())

	shareURL := azfile.NewShareURL(*u, azfile.NewPipeline(credential, azfile.PipelineOptions{}))
	_, err = shareURL.Create(context.TODO(), azfile.Metadata{}, 0)
	Expect(err).To(BeNil())

	return *credential, *u
}

func uploadFile(credential azfile.SharedKeyCredential, baseURL, fileName, fileContent string) {
	fURL, err := url.Parse(baseURL + "/" + fileName)
	Expect(err).To(BeNil())
	fileURL := azfile.NewFileURL(*fURL, azfile.NewPipeline(&credential, azfile.PipelineOptions{}))
	err = azfile.UploadBufferToAzureFile(context.TODO(), []byte(fileContent), fileURL, azfile.UploadToAzureFileOptions{})
	Expect(err).To(BeNil())
}

func TestE2eACI(t *testing.T) {
	suite.Run(t, new(E2eACISuite))
}

func setupTestResourceGroup(groupName string) {
	log.Println("Creating resource group " + resourceGroupName)
	ctx := context.TODO()
	helper := azure.NewACIResourceGroupHelper()
	models, err := helper.GetSubscriptionIDs(ctx)
	Expect(err).To(BeNil())
	_, err = helper.CreateOrUpdate(ctx, *models[0].SubscriptionID, groupName, resources.Group{
		Location: to.StringPtr(location),
	})
	Expect(err).To(BeNil())
}

func deleteResourceGroup(groupName string) {
	log.Println("Deleting resource group " + resourceGroupName)
	ctx := context.TODO()
	helper := azure.NewACIResourceGroupHelper()
	models, err := helper.GetSubscriptionIDs(ctx)
	Expect(err).To(BeNil())
	err = helper.Delete(ctx, *models[0].SubscriptionID, groupName)
	Expect(err).To(BeNil())
}

func RandStringBytes(n int) string {
	rand.Seed(time.Now().UnixNano())
	const digits = "0123456789"
	b := make([]byte, n)
	for i := range b {
		b[i] = digits[rand.Intn(len(digits))]
	}
	return string(b)
}
<|endoftext|>


<commit_before>package registry

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"strconv"

	"github.com/docker/docker/pkg/log"
	"github.com/docker/docker/utils"
	"github.com/gorilla/mux"
)

func newV2RegistryRouter() *mux.Router {
	router := mux.NewRouter()

	v2Router := router.PathPrefix("/v2/").Subrouter()

	// Version Info
	v2Router.Path("/version").Name("version")

	// Image Manifests
	v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests")

	// List Image Tags
	v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags")

	// Download a blob
	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")

	// Upload a blob
	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob")

	// Mounting a blob in an image
	v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")

	return router
}

// APIVersion2 /v2/
var v2HTTPRoutes = newV2RegistryRouter()

func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) {
	route := v2HTTPRoutes.Get(routeName)
	if route == nil {
		return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName)
	}

	varReplace := make([]string, 0, len(vars)*2)
	for key, val := range vars {
		varReplace = append(varReplace, key, val)
	}

	routePath, err := route.URLPath(varReplace...)
	if err != nil {
		return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err)
	}
	u, err := url.Parse(REGISTRYSERVER)
	if err != nil {
		return nil, fmt.Errorf("invalid registry url: %s", err)
	}

	return &url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
		Path:   routePath.Path,
	}, nil
}

// V2 Provenance POC

func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) {
	routeURL, err := getV2URL(r.indexEndpoint, "version", nil)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res)
	}

	decoder := json.NewDecoder(res.Body)
	versionInfo := new(RegistryInfo)

	err = decoder.Decode(versionInfo)
	if err != nil {
		return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err)
	}

	return versionInfo, nil
}

//
// 1) Check if TarSum of each layer exists /v2/
//  1.a) if 200, continue
//  1.b) if 300, then push the
//  1.c) if anything else, err
// 2) PUT the created/signed manifest
//
func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) {
	vars := map[string]string{
		"imagename": imageName,
		"tagname":   tagName,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res)
	}

	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Error while reading the http response: %s", err)
	}
	return buf, nil
}

// - Succeeded to mount for this image scope
// - Failed with no error (So continue to Push the Blob)
// - Failed with error
func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
		"sum":       sum,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars)
	if err != nil {
		return false, err
	}

	method := "POST"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return false, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return false, err
	}
	res.Body.Close() // close early, since we're not needing a body on this call .. yet?
	switch res.StatusCode {
	case 200:
		// return something indicating no push needed
		return true, nil
	case 300:
		// return something indicating blob push needed
		return false, nil
	}
	return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode)
}

func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
		"sum":       sum,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
	if err != nil {
		return err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
	}

	_, err = io.Copy(blobWrtr, res.Body)
	return err
}

func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
		"sum":       sum,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
	if err != nil {
		return nil, 0, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, 0, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, 0, err
	}
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, 0, errLoginRequired
		}
		return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
	}
	lenStr := res.Header.Get("Content-Length")
	l, err := strconv.ParseInt(lenStr, 10, 64)
	if err != nil {
		return nil, 0, err
	}

	return res.Body, l, err
}

// Push the image to the server for storage.
// 'layer' is an uncompressed reader of the blob to be pushed.
// The server will generate it's own checksum calculation.
func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars)
	if err != nil {
		return "", err
	}

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr)
	if err != nil {
		return "", err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	if res.StatusCode != 201 {
		if res.StatusCode == 401 {
			return "", errLoginRequired
		}
		return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res)
	}

	type sumReturn struct {
		Checksum string `json:"checksum"`
	}

	decoder := json.NewDecoder(res.Body)
	var sumInfo sumReturn

	err = decoder.Decode(&sumInfo)
	if err != nil {
		return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err)
	}

	// XXX this is a json struct from the registry, with its checksum
	return sumInfo.Checksum, nil
}

// Finally Push the (signed) manifest of the blobs we've just pushed
func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error {
	vars := map[string]string{
		"imagename": imageName,
		"tagname":   tagName,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
	if err != nil {
		return err
	}

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr)
	if err != nil {
		return err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	res.Body.Close()
	if res.StatusCode != 201 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res)
	}

	return nil
}

// Given a repository name, returns a json array of string tags
func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) {
	vars := map[string]string{
		"imagename": imageName,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "tags", vars)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res)
	}

	decoder := json.NewDecoder(res.Body)
	var tags []string
	err = decoder.Decode(&tags)
	if err != nil {
		return nil, fmt.Errorf("Error while decoding the http response: %s", err)
	}
	return tags, nil
}
<commit_msg>Support tarsum dev version to fix issue with mtime<commit_after>package registry

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"strconv"

	"github.com/docker/docker/pkg/log"
	"github.com/docker/docker/utils"
	"github.com/gorilla/mux"
)

func newV2RegistryRouter() *mux.Router {
	router := mux.NewRouter()

	v2Router := router.PathPrefix("/v2/").Subrouter()

	// Version Info
	v2Router.Path("/version").Name("version")

	// Image Manifests
	v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests")

	// List Image Tags
	v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags")

	// Download a blob
	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")

	// Upload a blob
	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob")

	// Mounting a blob in an image
	v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")

	return router
}

// APIVersion2 /v2/
var v2HTTPRoutes = newV2RegistryRouter()

func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) {
	route := v2HTTPRoutes.Get(routeName)
	if route == nil {
		return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName)
	}

	varReplace := make([]string, 0, len(vars)*2)
	for key, val := range vars {
		varReplace = append(varReplace, key, val)
	}

	routePath, err := route.URLPath(varReplace...)
	if err != nil {
		return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err)
	}
	u, err := url.Parse(REGISTRYSERVER)
	if err != nil {
		return nil, fmt.Errorf("invalid registry url: %s", err)
	}

	return &url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
		Path:   routePath.Path,
	}, nil
}

// V2 Provenance POC

func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) {
	routeURL, err := getV2URL(r.indexEndpoint, "version", nil)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res)
	}

	decoder := json.NewDecoder(res.Body)
	versionInfo := new(RegistryInfo)

	err = decoder.Decode(versionInfo)
	if err != nil {
		return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err)
	}

	return versionInfo, nil
}

//
// 1) Check if TarSum of each layer exists /v2/
//  1.a) if 200, continue
//  1.b) if 300, then push the
//  1.c) if anything else, err
// 2) PUT the created/signed manifest
//
func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) {
	vars := map[string]string{
		"imagename": imageName,
		"tagname":   tagName,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res)
	}

	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Error while reading the http response: %s", err)
	}
	return buf, nil
}

// - Succeeded to mount for this image scope
// - Failed with no error (So continue to Push the Blob)
// - Failed with error
func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
		"sum":       sum,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars)
	if err != nil {
		return false, err
	}

	method := "POST"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return false, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return false, err
	}
	res.Body.Close() // close early, since we're not needing a body on this call .. yet?
	switch res.StatusCode {
	case 200:
		// return something indicating no push needed
		return true, nil
	case 300:
		// return something indicating blob push needed
		return false, nil
	}
	return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode)
}

func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
		"sum":       sum,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
	if err != nil {
		return err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
	}

	_, err = io.Copy(blobWrtr, res.Body)
	return err
}

func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
		"sum":       sum,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
	if err != nil {
		return nil, 0, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, 0, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, 0, err
	}
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, 0, errLoginRequired
		}
		return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
	}
	lenStr := res.Header.Get("Content-Length")
	l, err := strconv.ParseInt(lenStr, 10, 64)
	if err != nil {
		return nil, 0, err
	}

	return res.Body, l, err
}

// Push the image to the server for storage.
// 'layer' is an uncompressed reader of the blob to be pushed.
// The server will generate it's own checksum calculation.
func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) {
	vars := map[string]string{
		"imagename": imageName,
		"sumtype":   sumType,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars)
	if err != nil {
		return "", err
	}

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr)
	if err != nil {
		return "", err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	if res.StatusCode != 201 {
		if res.StatusCode == 401 {
			return "", errLoginRequired
		}
		return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res)
	}

	type sumReturn struct {
		Checksum string `json:"checksum"`
	}

	decoder := json.NewDecoder(res.Body)
	var sumInfo sumReturn

	err = decoder.Decode(&sumInfo)
	if err != nil {
		return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err)
	}

	// XXX this is a json struct from the registry, with its checksum
	return sumInfo.Checksum, nil
}

// Finally Push the (signed) manifest of the blobs we've just pushed
func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error {
	vars := map[string]string{
		"imagename": imageName,
		"tagname":   tagName,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
	if err != nil {
		return err
	}

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
	req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr)
	if err != nil {
		return err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	res.Body.Close()
	if res.StatusCode != 201 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res)
	}

	return nil
}

// Given a repository name, returns a json array of string tags
func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) {
	vars := map[string]string{
		"imagename": imageName,
	}

	routeURL, err := getV2URL(r.indexEndpoint, "tags", vars)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL.String())

	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
	if err != nil {
		return nil, err
	}
	setTokenAuth(req, token)
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res)
	}

	decoder := json.NewDecoder(res.Body)
	var tags []string
	err = decoder.Decode(&tags)
	if err != nil {
		return nil, fmt.Errorf("Error while decoding the http response: %s", err)
	}
	return tags, nil
}
<|endoftext|>


<commit_before>package rolling

import (
	"time"
)

type Point struct {
	Time  time.Time
	Value int64
}

// Got to be a better name than this.
type Window struct {
	x [][]Point
}

func New(widths []int) *Window {

}

func (w *Window) Push(p Point) {

}

func (w *Window) Flush(level int) {

}

func (w *Window) Get(level int) []Point {

}

func (w *Window) Close() {

}

func (w *Window) Subscribe(level int) chan []Point {

}
<commit_msg>Skeleton<commit_after>package rolling

import (
	"time"
)

type Point struct {
	Time  time.Time
	Value int64
}

type Aggregator func(point []Point) Point

// Got to be a better name than this.
type Window struct {
	rows []row
	aggregator Aggregator
}

type row struct {
	points []Point
	next int
}

func New(aggregator Aggregator, widths []int) *Window {
	w := Window{
		aggregator: aggregator,
		rows: make([]row, len(widths)),
	}

	for i := range widths {
		if widths[i] <= 0 {
			panic("Widths must be >= 0")
		}
		w.rows[i] = row{points:make([]Point, widths[i])}
	}

	return &w
}

func (w *Window) Push(p Point) {

}

func (w *Window) Flush(level int) {

}

func (w *Window) Get(level int) []Point {

}

func (w *Window) Close() {

}

func (w *Window) Subscribe(level int) chan []Point {

}
<|endoftext|>


<commit_before>package consumer

import (
	"flume-bridge/consumer/pool"
	"flume-bridge/rpc/flume"
	"github.com/momotech/GoRedis/libs/stdlog"
	"math/rand"
	"net"
	"os"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

const (
	batchSize = 1000
)

// 用于向flume中作为sink 通过thrift客户端写入日志
type SourceServer struct {
	clientPools       []*pool.FlumePoolLink
	clientPoolsRemote []*pool.FlumePoolLink
	clientPoolsLocal  []*pool.FlumePoolLink
	isStop            bool
	monitorCount      counter
	business          string
	batchSize         int
	buffChannel       chan *flume.ThriftFlumeEvent
	sourceLog         stdlog.Logger
	flushWorkNum      chan byte //发送线程数
	flushChan         chan []*flume.ThriftFlumeEvent
}

func newSourceServer(business string, clientPools []*pool.FlumePoolLink, sourceLog stdlog.Logger) (server *SourceServer) {
	buffChannel := make(chan *flume.ThriftFlumeEvent)
	sourceServer := &SourceServer{
		business:    business,
		clientPools: clientPools,
		batchSize:   batchSize,
		buffChannel: buffChannel,
		sourceLog:   sourceLog}

	sourceServer.spliteLocal()
	sourceServer.flushWorkNum = make(chan byte, 10)
	sourceServer.flushChan = make(chan []*flume.ThriftFlumeEvent, 2000)
	return sourceServer
}

//分离出本机和remote的flume pool
func (self *SourceServer) spliteLocal() {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		panic(err)
	}

	flumePoolRemote := make([]*pool.FlumePoolLink, 0)
	flumePoolLocal := make([]*pool.FlumePoolLink, 0)

	hostname, _ := os.Hostname()
	for _, pool := range self.clientPools {
		host := pool.FlumePool.GetHostPort().Host
		for _, addr := range addrs {
			self.sourceLog.Printf("[%s] [%s] [%s]\n", host, addr.String(), hostname)
			if host == addr.String() || host == hostname {
				flumePoolLocal = append(flumePoolLocal, pool)
				continue
			}
			flumePoolRemote = append(flumePoolRemote, pool)
		}
	}

	self.sourceLog.Printf("flume nodes|Remote :[%s] Local :[%s]\n", flumePoolRemote, flumePoolLocal)
	self.clientPoolsRemote = flumePoolRemote
	self.clientPoolsLocal = flumePoolLocal
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize, arrayPool int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	arrayPool = len(self.flushWorkNum)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false
	//消费
	go self.consume()
	//消息转移
	go self.transfer()
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) consume() {

	//开启flush的操作
	go func() {
		for !self.isStop {
			events := <-self.flushChan
			self.flushWorkNum <- 1
			go func(p []*flume.ThriftFlumeEvent) {
				defer func() {
					<-self.flushWorkNum
				}()
				self.flush(p)
			}(events)
		}
	}()
}

const (
	MAX_THRIFT_PACKET_SIZE = 12 * 1024 * 1024
)

//转移到消费的channel
func (self *SourceServer) transfer() {
	// transfer
	byteSize := 0
	tick := time.NewTicker(5 * time.Second)
	packets := make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
	for !self.isStop {
		select {
		case event := <-self.buffChannel:
			//未达到批量提交的值并且当前的bytesSize 没有到最大THRIFT的包大小则继续append
			//否则强制提交
			if len(packets) < self.batchSize && byteSize < MAX_THRIFT_PACKET_SIZE {
				packets = append(packets, event)
				byteSize += len(event.Body)
			} else {
				self.flushChan <- packets
				packets = make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
				packets = append(packets, event)
				byteSize = len(event.Body)
			}
		case <-tick.C:
			//超时如果有数据则直接flush
			if len(packets) > 0 {
				self.flushChan <- packets[0:len(packets)]
				packets = make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
			}
			byteSize = 0
		}
	}

	//最后的flush出去
	if len(packets) > 0 {
		self.flushChan <- packets[0:len(packets)]
	}

}

func (self *SourceServer) flush(events []*flume.ThriftFlumeEvent) {

	start := time.Now().UnixNano()
	pool := self.getFlumeClientPool()
	if nil == pool {
		self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENTPOOL|FAIL|%s\n", self.business)
		return
	}
	flumeclient, err := pool.Get(2 * time.Second)
	if nil != err || nil == flumeclient {
		self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s\n",
			self.business, err.Error())
		return
	}

	defer func() {
		if err := recover(); nil != err {
			//回收这个坏的连接
			pool.ReleaseBroken(flumeclient)
		} else {
			pool.Release(flumeclient)
		}
	}()
	err = flumeclient.AppendBatch(events)

	if nil != err {
		atomic.AddInt64(&self.monitorCount.currFailValue, int64(len(events)))
		self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|%s\n",
			self.business, flumeclient.HostPort(), err.Error())
	} else {
		atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))

	}

	end := time.Now().UnixNano()
	if rand.Intn(1000) == 0 {
		self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|COST|%s|%s|cost:%d\n",
			self.business, flumeclient.HostPort(), end-start)
	}
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(1 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for _, v := range self.clientPools {
		v.DetachBusiness(self.business)
	}
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {
	//先从本地选择
	localCnt := len(self.clientPoolsLocal)
	if localCnt > 0 {
		idx := rand.Intn(localCnt)
		if idx < len(self.clientPoolsLocal) {
			return self.clientPoolsLocal[idx].FlumePool
		}
	}
	remoteCnt := len(self.clientPoolsRemote)
	if remoteCnt > 0 {
		idx := rand.Intn(remoteCnt)
		if idx < len(self.clientPoolsRemote) {
			return self.clientPoolsRemote[idx].FlumePool
		}
	}
	return nil
}
<commit_msg>	modified:   consumer/log_source.go<commit_after>package consumer

import (
	"flume-bridge/consumer/pool"
	"flume-bridge/rpc/flume"
	"github.com/momotech/GoRedis/libs/stdlog"
	"math/rand"
	"net"
	"os"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

const (
	batchSize = 1000
)

// 用于向flume中作为sink 通过thrift客户端写入日志
type SourceServer struct {
	clientPools       []*pool.FlumePoolLink
	clientPoolsRemote []*pool.FlumePoolLink
	clientPoolsLocal  []*pool.FlumePoolLink
	isStop            bool
	monitorCount      counter
	business          string
	batchSize         int
	buffChannel       chan *flume.ThriftFlumeEvent
	sourceLog         stdlog.Logger
	flushWorkNum      chan byte //发送线程数
	flushChan         chan []*flume.ThriftFlumeEvent
}

func newSourceServer(business string, clientPools []*pool.FlumePoolLink, sourceLog stdlog.Logger) (server *SourceServer) {
	buffChannel := make(chan *flume.ThriftFlumeEvent)
	sourceServer := &SourceServer{
		business:    business,
		clientPools: clientPools,
		batchSize:   batchSize,
		buffChannel: buffChannel,
		sourceLog:   sourceLog}

	sourceServer.spliteLocal()
	sourceServer.flushWorkNum = make(chan byte, 10)
	sourceServer.flushChan = make(chan []*flume.ThriftFlumeEvent, 2000)
	return sourceServer
}

//分离出本机和remote的flume pool
func (self *SourceServer) spliteLocal() {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		panic(err)
	}

	flumePoolRemote := make([]*pool.FlumePoolLink, 0)
	flumePoolLocal := make([]*pool.FlumePoolLink, 0)

	hostname, _ := os.Hostname()
	for _, pool := range self.clientPools {
		host := pool.FlumePool.GetHostPort().Host
		for _, addr := range addrs {
			self.sourceLog.Printf("[%s] [%s] [%s]\n", host, addr.String(), hostname)
			if host == addr.String() || host == hostname {
				flumePoolLocal = append(flumePoolLocal, pool)
				continue
			}
			flumePoolRemote = append(flumePoolRemote, pool)
		}
	}

	self.sourceLog.Printf("flume nodes|Remote :[%s] Local :[%s]\n", flumePoolRemote, flumePoolLocal)
	self.clientPoolsRemote = flumePoolRemote
	self.clientPoolsLocal = flumePoolLocal
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize, arrayPool int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	arrayPool = len(self.flushWorkNum)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false
	//消费
	go self.consume()
	//消息转移
	go self.transfer()
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) consume() {

	//开启flush的操作
	go func() {
		for !self.isStop {
			events := <-self.flushChan
			self.flushWorkNum <- 1
			go func(p []*flume.ThriftFlumeEvent) {
				defer func() {
					<-self.flushWorkNum
				}()
				self.flush(p)
			}(events)
		}
	}()
}

const (
	MAX_THRIFT_PACKET_SIZE = 12 * 1024 * 1024
)

//转移到消费的channel
func (self *SourceServer) transfer() {
	// transfer
	byteSize := 0
	tick := time.NewTicker(5 * time.Second)
	packets := make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
	for !self.isStop {
		select {
		case event := <-self.buffChannel:
			//未达到批量提交的值并且当前的bytesSize 没有到最大THRIFT的包大小则继续append
			//否则强制提交
			if len(packets) < self.batchSize && byteSize < MAX_THRIFT_PACKET_SIZE {
				packets = append(packets, event)
				byteSize += len(event.Body)
			} else {
				self.flushChan <- packets
				packets = packets[:0]
				packets = append(packets, event)
				byteSize = len(event.Body)
			}
		case <-tick.C:
			//超时如果有数据则直接flush
			if len(packets) > 0 {
				self.flushChan <- packets[0:len(packets)]
				packets = packets[:0]
			}
			byteSize = 0
		}
	}

	//最后的flush出去
	if len(packets) > 0 {
		self.flushChan <- packets[0:len(packets)]
	}

}

func (self *SourceServer) flush(events []*flume.ThriftFlumeEvent) {

	start := time.Now().UnixNano()
	pool := self.getFlumeClientPool()
	if nil == pool {
		self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENTPOOL|FAIL|%s\n", self.business)
		return
	}
	flumeclient, err := pool.Get(2 * time.Second)
	if nil != err || nil == flumeclient {
		self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s\n",
			self.business, err.Error())
		return
	}

	defer func() {
		if err := recover(); nil != err {
			//回收这个坏的连接
			pool.ReleaseBroken(flumeclient)
		} else {
			pool.Release(flumeclient)
		}
	}()
	err = flumeclient.AppendBatch(events)

	if nil != err {
		atomic.AddInt64(&self.monitorCount.currFailValue, int64(len(events)))
		self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|%s\n",
			self.business, flumeclient.HostPort(), err.Error())
	} else {
		atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))

	}

	end := time.Now().UnixNano()
	if rand.Intn(1000) == 0 {
		self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|COST|%s|%s|cost:%d\n",
			self.business, flumeclient.HostPort(), end-start)
	}
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(1 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for _, v := range self.clientPools {
		v.DetachBusiness(self.business)
	}
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {
	//先从本地选择
	localCnt := len(self.clientPoolsLocal)
	if localCnt > 0 {
		idx := rand.Intn(localCnt)
		if idx < len(self.clientPoolsLocal) {
			return self.clientPoolsLocal[idx].FlumePool
		}
	}
	remoteCnt := len(self.clientPoolsRemote)
	if remoteCnt > 0 {
		idx := rand.Intn(remoteCnt)
		if idx < len(self.clientPoolsRemote) {
			return self.clientPoolsRemote[idx].FlumePool
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package reverse_test

import (
	"io/ioutil"
	"testing"

	"github.com/rfjakob/gocryptfs/internal/ctlsock"
	"github.com/rfjakob/gocryptfs/tests/test_helpers"
)

const xxx = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

/*
tree exclude_test_fs
exclude_test_fs/
├── dir1
│   ├── file1
│   ├── file2
│   ├── longfile1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── longfile2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
├── dir2
│   ├── file
│   ├── longdir1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   │   └── file
│   ├── longfile.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── subdir
│       └── file
├── file1
├── file2
├── longdir1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── file
├── longdir2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── file
├── longfile1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
└── longfile2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
*/

func ctlsockEncryptPath(t *testing.T, sock string, path string) string {
	req := ctlsock.RequestStruct{EncryptPath: path}
	response := test_helpers.QueryCtlSock(t, sock, req)
	if response.ErrNo != 0 {
		t.Fatal(response)
	}
	return response.Result
}

func testExclude(t *testing.T, flag string) {
	pOk := []string{
		"file2",
		"dir1/file1",
		"dir1/longfile1" + xxx,
		"longdir1" + xxx,
		"longdir1" + xxx + "/file",
		"longfile1" + xxx,
	}
	pExclude := []string{
		"file1",
		"dir1/file2",
		"dir1/longfile2" + xxx,
		"dir2",
		"dir2/file",
		"dir2/file/xxx",
		"dir2/subdir",
		"dir2/subdir/file",
		"dir2/longdir1" + xxx + "/file",
		"dir2/longfile." + xxx,
		"longfile2" + xxx,
	}
	// Mount reverse fs
	mnt, err := ioutil.TempDir(test_helpers.TmpDir, "TestExclude")
	if err != nil {
		t.Fatal(err)
	}
	sock := mnt + ".sock"
	cliArgs := []string{"-reverse", "-extpass", "echo test", "-ctlsock", sock}
	for _, v := range pExclude {
		cliArgs = append(cliArgs, flag, v)
	}
	if plaintextnames {
		cliArgs = append(cliArgs, "-config", "exclude_test_fs/.gocryptfs.reverse.conf.plaintextnames")
	}
	test_helpers.MountOrFatal(t, "exclude_test_fs", mnt, cliArgs...)
	defer test_helpers.UnmountPanic(mnt)
	// Get encrypted version of "ok" and "excluded" paths
	cOk := make([]string, len(pOk))
	cExclude := make([]string, len(pExclude))
	for i, v := range pOk {
		cOk[i] = ctlsockEncryptPath(t, sock, v)
	}
	for i, v := range pExclude {
		cExclude[i] = ctlsockEncryptPath(t, sock, v)
	}
	// Check that "excluded" paths are not there and "ok" paths are there
	for i, v := range cExclude {
		if test_helpers.VerifyExistence(mnt + "/" + v) {
			t.Errorf("File %q / %q is visible, but should be excluded", pExclude[i], v)
		}
	}
	for i, v := range cOk {
		if !test_helpers.VerifyExistence(mnt + "/" + v) {
			t.Errorf("File %q / %q is hidden, but should be visible", pOk[i], v)
		}
	}
}

func TestExclude(t *testing.T) {
	testExclude(t, "-exclude")
	testExclude(t, "-e")
}
<commit_msg>tests: reverse: verify that longname .name files are exluded as well<commit_after>package reverse_test

import (
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/rfjakob/gocryptfs/internal/ctlsock"
	"github.com/rfjakob/gocryptfs/internal/nametransform"
	"github.com/rfjakob/gocryptfs/tests/test_helpers"
)

const xxx = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

/*
tree exclude_test_fs
exclude_test_fs/
├── dir1
│   ├── file1
│   ├── file2
│   ├── longfile1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── longfile2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
├── dir2
│   ├── file
│   ├── longdir1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   │   └── file
│   ├── longfile.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── subdir
│       └── file
├── file1
├── file2
├── longdir1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── file
├── longdir2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
│   └── file
├── longfile1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
└── longfile2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
*/

func ctlsockEncryptPath(t *testing.T, sock string, path string) string {
	req := ctlsock.RequestStruct{EncryptPath: path}
	response := test_helpers.QueryCtlSock(t, sock, req)
	if response.ErrNo != 0 {
		t.Fatal(response)
	}
	return response.Result
}

func testExclude(t *testing.T, flag string) {
	pOk := []string{
		"file2",
		"dir1/file1",
		"dir1/longfile1" + xxx,
		"longdir1" + xxx,
		"longdir1" + xxx + "/file",
		"longfile1" + xxx,
	}
	pExclude := []string{
		"file1",
		"dir1/file2",
		"dir1/longfile2" + xxx,
		"dir2",
		"dir2/file",
		"dir2/file/xxx",
		"dir2/subdir",
		"dir2/subdir/file",
		"dir2/longdir1" + xxx + "/file",
		"dir2/longfile." + xxx,
		"longfile2" + xxx,
	}
	// Mount reverse fs
	mnt, err := ioutil.TempDir(test_helpers.TmpDir, "TestExclude")
	if err != nil {
		t.Fatal(err)
	}
	sock := mnt + ".sock"
	cliArgs := []string{"-reverse", "-extpass", "echo test", "-ctlsock", sock}
	for _, v := range pExclude {
		cliArgs = append(cliArgs, flag, v)
	}
	if plaintextnames {
		cliArgs = append(cliArgs, "-config", "exclude_test_fs/.gocryptfs.reverse.conf.plaintextnames")
	}
	test_helpers.MountOrFatal(t, "exclude_test_fs", mnt, cliArgs...)
	defer test_helpers.UnmountPanic(mnt)
	// Get encrypted version of "ok" and "excluded" paths
	cOk := encryptExcludeTestPaths(t, sock, pOk)
	cExclude := encryptExcludeTestPaths(t, sock, pExclude)
	// Check that "excluded" paths are not there and "ok" paths are there
	for _, v := range cExclude {
		if test_helpers.VerifyExistence(mnt + "/" + v) {
			t.Errorf("File %q is visible, but should be excluded", v)
		}
		if nametransform.IsLongContent(filepath.Base(v)) {

		}
	}
	for _, v := range cOk {
		if !test_helpers.VerifyExistence(mnt + "/" + v) {
			t.Errorf("File %q is hidden, but should be visible", v)
		}
	}
}

// encryptExcludeTestPaths is used by testExclude() to encrypt the lists of
// testcase paths
func encryptExcludeTestPaths(t *testing.T, socket string, pRelPaths []string) (out []string) {
	for _, pRelPath := range pRelPaths {
		cRelPath := ctlsockEncryptPath(t, socket, pRelPath)
		out = append(out, cRelPath)
		if !plaintextnames && nametransform.IsLongContent(filepath.Base(cRelPath)) {
			// If we exclude
			//   gocryptfs.longname.3vZ_r3eDPb1_fL3j5VA4rd_bcKWLKT9eaxOVIGK5HFA
			// we should also exclude
			//   gocryptfs.longname.3vZ_r3eDPb1_fL3j5VA4rd_bcKWLKT9eaxOVIGK5HFA.name
			out = append(out, cRelPath+nametransform.LongNameSuffix)
		}
	}
	return out
}

func TestExclude(t *testing.T) {
	testExclude(t, "-exclude")
	testExclude(t, "-e")
}
<|endoftext|>


<commit_before>package dnsr

import (
	"testing"

	"github.com/nbio/st"
)

func TestRRStringStandard(t *testing.T) {
	rr := RR{
		Name:  "example.com.",
		Type:  "A",
		Value: "203.0.113.1",
	}
	result := rr.String()
	st.Expect(t, result, "example.com.	      3600	IN	A	203.0.113.1")
}
<commit_msg>Test expiring RR string<commit_after>package dnsr

import (
	"testing"
	"time"

	"github.com/nbio/st"
)

func TestRRStringStandard(t *testing.T) {
	rr := RR{
		Name:  "example.com.",
		Type:  "A",
		Value: "203.0.113.1",
	}
	result := rr.String()
	st.Expect(t, result, "example.com.	      3600	IN	A	203.0.113.1")
}

func TestRRStringExpiring(t *testing.T) {
	ttl := 86400 * time.Second
	rr := RR{
		Name:   "example.com.",
		Type:   "A",
		Value:  "203.0.113.1",
		TTL:    ttl,
		Expiry: time.Now().Add(ttl),
	}
	result := rr.String()
	st.Expect(t, result, "example.com.	     86400	IN	A	203.0.113.1")
}
<|endoftext|>


<commit_before>package rsa

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"encoding/base64"
	"errors"
	"strings"
	"io/ioutil"
)

type Key interface {
	PublicKey() *rsa.PublicKey
	PrivateKey() *rsa.PrivateKey
	Modulus() int
}

func ParsePKCS8Key(publicKey, privateKey []byte) (Key, error) {
	puk, err := x509.ParsePKIXPublicKey(publicKey)
	if err != nil {
		return nil, err
	}

	prk, err := x509.ParsePKCS8PrivateKey(privateKey)
	if err != nil {
		return nil, err
	}
	return &key{publicKey:puk.(*rsa.PublicKey), privateKey:prk.(*rsa.PrivateKey)}, nil
}

func ParsePKCS8KeyWithBase64(publicKey, privateKey string) (Key, error) {
	puk, err := base64.StdEncoding.DecodeString(publicKey)
	if err != nil {
		return nil, err
	}
	prk, err := base64.StdEncoding.DecodeString(privateKey)
	if err != nil {
		return nil, err
	}
	return ParsePKCS8Key(puk, prk)
}

func ParsePKCS8KeyWithPEMEncoding(publicKey, privateKey string) (Key, error) {

	puk, _ := pem.Decode([]byte(publicKey))
	prk, _ := pem.Decode([]byte(privateKey))

	if puk == nil || prk == nil {
		return nil, errors.New("is not pem formate")
	}
	return ParsePKCS8Key(puk.Bytes, prk.Bytes)
}

func LoadPKCS8KeyFromPEMFile(publicKeyFilePath, privateKeyFilePath string) (Key, error) {

	//TODO 断言如果入参为"" ，则直接报错

	publicKeyFilePath = strings.TrimSpace(publicKeyFilePath)

	pukBytes, err := ioutil.ReadFile(publicKeyFilePath)
	if err != nil {
		return nil, err
	}

	puk, _ := pem.Decode(pukBytes)
	if puk == nil {
		return nil, errors.New("publicKey is not pem formate")
	}

	privateKeyFilePath = strings.TrimSpace(privateKeyFilePath)

	prkBytes, err := ioutil.ReadFile(privateKeyFilePath)
	if err != nil {
		return nil, err
	}

	prk, _ := pem.Decode(prkBytes)
	if prk == nil {
		return nil, errors.New("privateKey is not pem formate")
	}

	return ParsePKCS8Key(puk.Bytes, prk.Bytes)
}

type key struct {
	publicKey  *rsa.PublicKey
	privateKey *rsa.PrivateKey
}

func (key *key) Modulus() int {
	return len(key.publicKey.N.Bytes())
}

func (key *key) PublicKey() *rsa.PublicKey {
	return key.publicKey
}

func (key *key) PrivateKey() *rsa.PrivateKey {
	return key.privateKey
}
<commit_msg>新增了对pkcs1 秘钥的支持<commit_after>package rsa

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"strings"
	"io/ioutil"
)

type Key interface {
	PublicKey() *rsa.PublicKey
	PrivateKey() *rsa.PrivateKey
	Modulus() int
}

func ParsePKCS8Key(publicKey, privateKey []byte) (Key, error) {
	puk, err := x509.ParsePKIXPublicKey(publicKey)
	if err != nil {
		return nil, err
	}

	prk, err := x509.ParsePKCS8PrivateKey(privateKey)
	if err != nil {
		return nil, err
	}
	return &key{publicKey:puk.(*rsa.PublicKey), privateKey:prk.(*rsa.PrivateKey)}, nil
}

func ParsePKCS1Key(publicKey, privateKey []byte) (Key, error) {
	puk, err := x509.ParsePKIXPublicKey(publicKey)
	if err != nil {
		return nil, err
	}

	prk, err := x509.ParsePKCS1PrivateKey(privateKey)
	if err != nil {
		return nil, err
	}
	return &key{publicKey:puk.(*rsa.PublicKey), privateKey:prk.(*rsa.PrivateKey)}, nil
}

func LoadPKCS8KeyFromPEMFile(publicKeyFilePath, privateKeyFilePath string) (Key, error) {

	//TODO 断言如果入参为"" ，则直接报错

	publicKeyFilePath = strings.TrimSpace(publicKeyFilePath)

	pukBytes, err := ioutil.ReadFile(publicKeyFilePath)
	if err != nil {
		return nil, err
	}

	puk, _ := pem.Decode(pukBytes)
	if puk == nil {
		return nil, errors.New("publicKey is not pem formate")
	}

	privateKeyFilePath = strings.TrimSpace(privateKeyFilePath)

	prkBytes, err := ioutil.ReadFile(privateKeyFilePath)
	if err != nil {
		return nil, err
	}

	prk, _ := pem.Decode(prkBytes)
	if prk == nil {
		return nil, errors.New("privateKey is not pem formate")
	}

	return ParsePKCS8Key(puk.Bytes, prk.Bytes)
}

func LoadPKCS1KeyFromPEMFile(publicKeyFilePath, privateKeyFilePath string) (Key, error) {

	//TODO 断言如果入参为"" ，则直接报错

	publicKeyFilePath = strings.TrimSpace(publicKeyFilePath)

	pukBytes, err := ioutil.ReadFile(publicKeyFilePath)
	if err != nil {
		return nil, err
	}

	puk, _ := pem.Decode(pukBytes)
	if puk == nil {
		return nil, errors.New("publicKey is not pem formate")
	}

	privateKeyFilePath = strings.TrimSpace(privateKeyFilePath)

	prkBytes, err := ioutil.ReadFile(privateKeyFilePath)
	if err != nil {
		return nil, err
	}

	prk, _ := pem.Decode(prkBytes)
	if prk == nil {
		return nil, errors.New("privateKey is not pem formate")
	}

	return ParsePKCS1Key(puk.Bytes, prk.Bytes)
}

type key struct {
	publicKey  *rsa.PublicKey
	privateKey *rsa.PrivateKey
}

func (key *key) Modulus() int {
	return len(key.publicKey.N.Bytes())
}

func (key *key) PublicKey() *rsa.PublicKey {
	return key.publicKey
}

func (key *key) PrivateKey() *rsa.PrivateKey {
	return key.privateKey
}
<|endoftext|>