<commit_before>package handlers

import (
	"fmt"
	"github.com/dmonay/okra/common"
	"github.com/gin-gonic/gin"
	// "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func (dw *DoWorkResource) CreateOrg(c *gin.Context) {

	var reqBody common.OrganizationJson

	c.Bind(&reqBody)

	org := reqBody.Organization
	objId := bson.ObjectIdHex(reqBody.UserId)

	// 1. Create collection with members document
	member := reqBody.UserName
	memArray := []string{member}
	membersDoc := &OrgMembers{memArray, "membersArray"}
	err2 := dw.mongo.C(org).Insert(membersDoc)
	CheckErr(err2, "Mongo failed to create collection with the empty members array")

	// 2. Add organization to user's document in Users
	colQuerier := bson.M{"_id": objId}
	updateTimeframe := bson.M{"$push": bson.M{"orgs": org}}
	err := dw.mongo.C("Users").Update(colQuerier, updateTimeframe)
	CheckErr(err, "Mongo failed to add organization to user's document in Users")

	c.JSON(201, "You have successfully created an organization")
}

type OrgMembers struct {
	Members []string `bson:"members"`
	Name    string   `bson:"name"`
}

func (dw *DoWorkResource) CreateTree(c *gin.Context) {

	var reqBody common.TreeJson
	c.Bind(&reqBody)

	org := c.Params.ByName("organization")
	timeframe := reqBody.Timeframe
	treeName := reqBody.TreeName
	objId := bson.ObjectIdHex(reqBody.UserId)
	member := reqBody.UserName
	members := []string{member}
	var emptyObj ObjectiveMongo
	id := bson.NewObjectId()

	// 1. Create tree and upsert it into the org collection
	treeStruct := &OkrTree{
		id,
		"tree",
		org,
		"",
		members,
		true,
		timeframe,
		treeName,
		emptyObj,
		emptyObj,
		emptyObj,
		emptyObj,
		emptyObj,
	}
	colQuerier := bson.M{"treename": treeName}
	upsertTree := bson.M{"$set": treeStruct}
	info, err := dw.mongo.C(org).Upsert(colQuerier, upsertTree)
	CheckErr(err, "Mongo failed to create tree for "+org+" organization")

	// 2. Update user's doc in Users with the ObjId and name of the tree
	treeId := info.UpsertedId.(bson.ObjectId)
	tree := common.UserTree{treeName, treeId}
	colQuerier2 := bson.M{"_id": objId}
	updateTimeframe := bson.M{"$push": bson.M{"trees": tree}}
	err2 := dw.mongo.C("Users").Update(colQuerier2, updateTimeframe)
	CheckErr(err2, "Mongo failed to add tree to user's document in Users")

	c.JSON(201, "You have successfully created a tree with the "+timeframe+" timeframe for the "+org+" organization")
}

func (dw *DoWorkResource) UpdateMission(c *gin.Context) {

	org := c.Params.ByName("organization")
	var reqBody common.MissionJson

	c.Bind(&reqBody)

	mission := reqBody.Mission
	treeId := bson.ObjectIdHex(reqBody.TreeId)

	colQuerier := bson.M{"_id": treeId}
	setMission := bson.M{"$set": bson.M{"mission": mission}}
	err := dw.mongo.C(org).Update(colQuerier, setMission)
	CheckErr(err, "Mongo failed to update mission")

	c.JSON(201, mission)
}

func (dw *DoWorkResource) AddMembers(c *gin.Context) {
	org := c.Params.ByName("organization")
	var reqBody common.MembersJson
	c.Bind(&reqBody)

	// update tree's members array if tree ID provided
	if reqBody.UpdateTree {
		treeId := bson.ObjectIdHex(reqBody.TreeId)

		colQuerier := bson.M{"_id": treeId}
		for _, value := range reqBody.Members {
			addMembers := bson.M{"$push": bson.M{"members": value}}
			err := dw.mongo.C(org).Update(colQuerier, addMembers)
			CheckErr(err, "Mongo failed to add members to the provided tree")

			// Update user's doc in Users with the ObjId and name of the tree
			memberId := bson.ObjectIdHex(value.UserId)
			tree := common.UserTree{reqBody.TreeName, treeId}
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$push": bson.M{"trees": tree}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to add tree to user's document in Users")
		}

		c.JSON(201, "You have successfully added members to the tree")

		// otherwise update org's members array
	} else {
		colQuerier := bson.M{"name": "membersArray"}
		for _, value := range reqBody.Members {
			addMembers := bson.M{"$push": bson.M{"members": value}}
			err := dw.mongo.C(org).Update(colQuerier, addMembers)
			CheckErr(err, "Mongo failed to add members to "+org+" organization")

			// Update user's doc in Users with the name of the org
			memberId := bson.ObjectIdHex(value.UserId)
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$push": bson.M{"orgs": org}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to add org to user's document in Users")
		}
		c.JSON(201, "You have successfully added members to the "+org+" organization")
	}
}

func (dw *DoWorkResource) DeleteMembers(c *gin.Context) {
	org := c.Params.ByName("organization")
	var reqBody common.MembersJsonDelete
	c.Bind(&reqBody)

	// remove member from tree's members array if tree ID provided
	if reqBody.UpdateTree {

		id := bson.ObjectIdHex(reqBody.TreeId)

		colQuerier := bson.M{"_id": id}
		for _, value := range reqBody.Members {
			fmt.Println("user's id: ", value)
			removeMembers := bson.M{"$pull": bson.M{"members": bson.M{"userid": value}}}
			err := dw.mongo.C(org).Update(colQuerier, removeMembers)
			CheckErr(err, "Mongo failed to remove members from the provided tree")

			// Remove trees from user's doc in Users
			memberId := bson.ObjectIdHex(value)
			treeId := bson.ObjectIdHex(reqBody.TreeId)
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$pull": bson.M{"trees": bson.M{"treeid": treeId}}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to remove tree from user's document in Users")
		}
		c.JSON(200, "You have successfully removed members from the tree")

		// otherwise remove member from org's members array
	} else {
		colQuerier := bson.M{"name": "membersArray"}
		for _, value := range reqBody.Members {
			removeMembers := bson.M{"$pull": bson.M{"members": bson.M{"userid": value}}}
			err := dw.mongo.C(org).Update(colQuerier, removeMembers)
			CheckErr(err, "Mongo failed to remove members from "+org+" organization")

			// Remove org from user's doc in Users
			memberId := bson.ObjectIdHex(value)
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$pull": bson.M{"orgs": org}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to remove org from user's document in Users")
		}
		c.JSON(200, "You have successfully removed members from the "+org+" organization")
	}
}

func (dw *DoWorkResource) UpdateObjective(c *gin.Context) {

	org := c.Params.ByName("organization")
	var reqBody common.ObjectiveJson

	c.Bind(&reqBody)

	id := reqBody.Id
	treeId := bson.ObjectIdHex(reqBody.TreeId)
	obj := ObjectiveMongo{
		Name:    reqBody.Name,
		Body:    reqBody.Body,
		Active:  reqBody.Active,
		Members: reqBody.Members,
	}

	colQuerier := bson.M{"_id": treeId}
	addObjective := bson.M{"$set": bson.M{id: obj}}
	err := dw.mongo.C(org).Update(colQuerier, addObjective)
	CheckErr(err, "Mongo failed to add objective")

	c.JSON(201, "You have successfully added an objective to the "+org+" organization")
}

func (dw *DoWorkResource) CreateKeyResult(c *gin.Context) {

	org := c.Params.ByName("organization")
	obj := c.Params.ByName("objective")
	var reqBody common.KeyResultJson

	c.Bind(&reqBody)

	id := reqBody.Id
	resultName := obj + "." + id

	colQuerier := bson.M{"orgname": org}
	addKeyResult := bson.M{"$set": bson.M{resultName: reqBody}}
	err := dw.mongo.C(org).Update(colQuerier, addKeyResult)
	CheckErr(err, "Mongo failed to add key result")

	c.JSON(201, "You have successfully added a key result to "+obj)
}

func (dw *DoWorkResource) GetTrees(c *gin.Context) {
	org := c.Params.ByName("organization")
	treeId := c.Params.ByName("treeid")
	id := bson.ObjectIdHex(treeId)

	var result OkrTree

	err := dw.mongo.C(org).Find(bson.M{"_id": id}).One(&result)
	result.Id = id
	CheckErr(err, "Failed to retrieve tree from Mongo")

	c.JSON(200, result)
}

func (dw *DoWorkResource) GetAllTrees(c *gin.Context) {
	org := c.Params.ByName("organization")

	var intermResult []OkrTree
	err4 := dw.mongo.C(org).Find(bson.M{"type": "tree"}).All(&intermResult)
	CheckErr(err4, "Failed to retrieve trees in organization "+org+" from Mongo")
	length := len(intermResult)

	result := make([]TreeInOrg, length)

	for key, value := range intermResult {
		treeName := value.TreeName
		treeId := value.Id
		active := value.Active

		result[key].Name = treeName
		result[key].Id = treeId
		result[key].Active = active
	}

	c.JSON(200, result)
}

type OkrTree struct {
	Id         bson.ObjectId `bson:"_id"`
	Type       string
	OrgName    string
	Mission    string
	Members    []string
	Active     bool
	Timeframe  string
	TreeName   string
	Objective1 ObjectiveMongo
	Objective2 ObjectiveMongo
	Objective3 ObjectiveMongo
	Objective4 ObjectiveMongo
	Objective5 ObjectiveMongo
}

type ObjectiveMongo struct {
	Name    string                       `json:"name"`
	Body    string                       `json:"body"`
	Active  bool                         `json:"active"`
	Members map[string]map[string]string `json:"members"`
}

type TreeInOrg struct {
	Name   string        `bson:"treename"`
	Id     bson.ObjectId `bson:"_id"`
	Active bool          `bson:"active"`
}
<commit_msg>add CORS header for GET requests<commit_after>package handlers

import (
	"fmt"
	"github.com/dmonay/okra/common"
	"github.com/gin-gonic/gin"
	// "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func (dw *DoWorkResource) CreateOrg(c *gin.Context) {

	var reqBody common.OrganizationJson

	c.Bind(&reqBody)

	org := reqBody.Organization
	objId := bson.ObjectIdHex(reqBody.UserId)

	// 1. Create collection with members document
	member := reqBody.UserName
	memArray := []string{member}
	membersDoc := &OrgMembers{memArray, "membersArray"}
	err2 := dw.mongo.C(org).Insert(membersDoc)
	CheckErr(err2, "Mongo failed to create collection with the empty members array")

	// 2. Add organization to user's document in Users
	colQuerier := bson.M{"_id": objId}
	updateTimeframe := bson.M{"$push": bson.M{"orgs": org}}
	err := dw.mongo.C("Users").Update(colQuerier, updateTimeframe)
	CheckErr(err, "Mongo failed to add organization to user's document in Users")

	c.JSON(201, "You have successfully created an organization")
}

type OrgMembers struct {
	Members []string `bson:"members"`
	Name    string   `bson:"name"`
}

func (dw *DoWorkResource) CreateTree(c *gin.Context) {

	var reqBody common.TreeJson
	c.Bind(&reqBody)

	org := c.Params.ByName("organization")
	timeframe := reqBody.Timeframe
	treeName := reqBody.TreeName
	objId := bson.ObjectIdHex(reqBody.UserId)
	member := reqBody.UserName
	members := []string{member}
	var emptyObj ObjectiveMongo
	id := bson.NewObjectId()

	// 1. Create tree and upsert it into the org collection
	treeStruct := &OkrTree{
		id,
		"tree",
		org,
		"",
		members,
		true,
		timeframe,
		treeName,
		emptyObj,
		emptyObj,
		emptyObj,
		emptyObj,
		emptyObj,
	}
	colQuerier := bson.M{"treename": treeName}
	upsertTree := bson.M{"$set": treeStruct}
	info, err := dw.mongo.C(org).Upsert(colQuerier, upsertTree)
	CheckErr(err, "Mongo failed to create tree for "+org+" organization")

	// 2. Update user's doc in Users with the ObjId and name of the tree
	treeId := info.UpsertedId.(bson.ObjectId)
	tree := common.UserTree{treeName, treeId}
	colQuerier2 := bson.M{"_id": objId}
	updateTimeframe := bson.M{"$push": bson.M{"trees": tree}}
	err2 := dw.mongo.C("Users").Update(colQuerier2, updateTimeframe)
	CheckErr(err2, "Mongo failed to add tree to user's document in Users")

	c.JSON(201, "You have successfully created a tree with the "+timeframe+" timeframe for the "+org+" organization")
}

func (dw *DoWorkResource) UpdateMission(c *gin.Context) {

	org := c.Params.ByName("organization")
	var reqBody common.MissionJson

	c.Bind(&reqBody)

	mission := reqBody.Mission
	treeId := bson.ObjectIdHex(reqBody.TreeId)

	colQuerier := bson.M{"_id": treeId}
	setMission := bson.M{"$set": bson.M{"mission": mission}}
	err := dw.mongo.C(org).Update(colQuerier, setMission)
	CheckErr(err, "Mongo failed to update mission")

	c.JSON(201, mission)
}

func (dw *DoWorkResource) AddMembers(c *gin.Context) {
	org := c.Params.ByName("organization")
	var reqBody common.MembersJson
	c.Bind(&reqBody)

	// update tree's members array if tree ID provided
	if reqBody.UpdateTree {
		treeId := bson.ObjectIdHex(reqBody.TreeId)

		colQuerier := bson.M{"_id": treeId}
		for _, value := range reqBody.Members {
			addMembers := bson.M{"$push": bson.M{"members": value}}
			err := dw.mongo.C(org).Update(colQuerier, addMembers)
			CheckErr(err, "Mongo failed to add members to the provided tree")

			// Update user's doc in Users with the ObjId and name of the tree
			memberId := bson.ObjectIdHex(value.UserId)
			tree := common.UserTree{reqBody.TreeName, treeId}
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$push": bson.M{"trees": tree}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to add tree to user's document in Users")
		}

		c.JSON(201, "You have successfully added members to the tree")

		// otherwise update org's members array
	} else {
		colQuerier := bson.M{"name": "membersArray"}
		for _, value := range reqBody.Members {
			addMembers := bson.M{"$push": bson.M{"members": value}}
			err := dw.mongo.C(org).Update(colQuerier, addMembers)
			CheckErr(err, "Mongo failed to add members to "+org+" organization")

			// Update user's doc in Users with the name of the org
			memberId := bson.ObjectIdHex(value.UserId)
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$push": bson.M{"orgs": org}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to add org to user's document in Users")
		}
		c.JSON(201, "You have successfully added members to the "+org+" organization")
	}
}

func (dw *DoWorkResource) DeleteMembers(c *gin.Context) {
	org := c.Params.ByName("organization")
	var reqBody common.MembersJsonDelete
	c.Bind(&reqBody)

	// remove member from tree's members array if tree ID provided
	if reqBody.UpdateTree {

		id := bson.ObjectIdHex(reqBody.TreeId)

		colQuerier := bson.M{"_id": id}
		for _, value := range reqBody.Members {
			fmt.Println("user's id: ", value)
			removeMembers := bson.M{"$pull": bson.M{"members": bson.M{"userid": value}}}
			err := dw.mongo.C(org).Update(colQuerier, removeMembers)
			CheckErr(err, "Mongo failed to remove members from the provided tree")

			// Remove trees from user's doc in Users
			memberId := bson.ObjectIdHex(value)
			treeId := bson.ObjectIdHex(reqBody.TreeId)
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$pull": bson.M{"trees": bson.M{"treeid": treeId}}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to remove tree from user's document in Users")
		}
		c.JSON(200, "You have successfully removed members from the tree")

		// otherwise remove member from org's members array
	} else {
		colQuerier := bson.M{"name": "membersArray"}
		for _, value := range reqBody.Members {
			removeMembers := bson.M{"$pull": bson.M{"members": bson.M{"userid": value}}}
			err := dw.mongo.C(org).Update(colQuerier, removeMembers)
			CheckErr(err, "Mongo failed to remove members from "+org+" organization")

			// Remove org from user's doc in Users
			memberId := bson.ObjectIdHex(value)
			colQuerier2 := bson.M{"_id": memberId}
			updateUsersDoc := bson.M{"$pull": bson.M{"orgs": org}}
			err2 := dw.mongo.C("Users").Update(colQuerier2, updateUsersDoc)
			CheckErr(err2, "Mongo failed to remove org from user's document in Users")
		}
		c.JSON(200, "You have successfully removed members from the "+org+" organization")
	}
}

func (dw *DoWorkResource) UpdateObjective(c *gin.Context) {

	org := c.Params.ByName("organization")
	var reqBody common.ObjectiveJson

	c.Bind(&reqBody)

	id := reqBody.Id
	treeId := bson.ObjectIdHex(reqBody.TreeId)
	obj := ObjectiveMongo{
		Name:    reqBody.Name,
		Body:    reqBody.Body,
		Active:  reqBody.Active,
		Members: reqBody.Members,
	}

	colQuerier := bson.M{"_id": treeId}
	addObjective := bson.M{"$set": bson.M{id: obj}}
	err := dw.mongo.C(org).Update(colQuerier, addObjective)
	CheckErr(err, "Mongo failed to add objective")

	c.JSON(201, "You have successfully added an objective to the "+org+" organization")
}

func (dw *DoWorkResource) CreateKeyResult(c *gin.Context) {

	org := c.Params.ByName("organization")
	obj := c.Params.ByName("objective")
	var reqBody common.KeyResultJson

	c.Bind(&reqBody)

	id := reqBody.Id
	resultName := obj + "." + id

	colQuerier := bson.M{"orgname": org}
	addKeyResult := bson.M{"$set": bson.M{resultName: reqBody}}
	err := dw.mongo.C(org).Update(colQuerier, addKeyResult)
	CheckErr(err, "Mongo failed to add key result")

	c.JSON(201, "You have successfully added a key result to "+obj)
}

func (dw *DoWorkResource) GetTrees(c *gin.Context) {
	org := c.Params.ByName("organization")
	treeId := c.Params.ByName("treeid")
	id := bson.ObjectIdHex(treeId)

	var result OkrTree

	err := dw.mongo.C(org).Find(bson.M{"_id": id}).One(&result)
	result.Id = id
	CheckErr(err, "Failed to retrieve tree from Mongo")

	origin := c.Request.Header.Get("Origin")
	c.Writer.Header().Set("Access-Control-Allow-Origin", origin)
	c.JSON(200, result)
}

func (dw *DoWorkResource) GetAllTrees(c *gin.Context) {
	org := c.Params.ByName("organization")

	var intermResult []OkrTree
	err4 := dw.mongo.C(org).Find(bson.M{"type": "tree"}).All(&intermResult)
	CheckErr(err4, "Failed to retrieve trees in organization "+org+" from Mongo")
	length := len(intermResult)

	result := make([]TreeInOrg, length)

	for key, value := range intermResult {
		treeName := value.TreeName
		treeId := value.Id
		active := value.Active

		result[key].Name = treeName
		result[key].Id = treeId
		result[key].Active = active
	}

	origin := c.Request.Header.Get("Origin")
	c.Writer.Header().Set("Access-Control-Allow-Origin", origin)
	c.JSON(200, result)
}

type OkrTree struct {
	Id         bson.ObjectId `bson:"_id"`
	Type       string
	OrgName    string
	Mission    string
	Members    []string
	Active     bool
	Timeframe  string
	TreeName   string
	Objective1 ObjectiveMongo
	Objective2 ObjectiveMongo
	Objective3 ObjectiveMongo
	Objective4 ObjectiveMongo
	Objective5 ObjectiveMongo
}

type ObjectiveMongo struct {
	Name    string                       `json:"name"`
	Body    string                       `json:"body"`
	Active  bool                         `json:"active"`
	Members map[string]map[string]string `json:"members"`
}

type TreeInOrg struct {
	Name   string        `bson:"treename"`
	Id     bson.ObjectId `bson:"_id"`
	Active bool          `bson:"active"`
}
<|endoftext|>


<commit_before>package lb

import (
	"container/list"
	"github.com/domgoer/hiruok/pkg/collection"
	"github.com/fagongzi/gateway/pkg/pb/metapb"
	"sync/atomic"

	"github.com/valyala/fasthttp"
)

// RoundRobin round robin loadBalance impl
type RoundRobin struct {
	ops *uint64
}

// NewRoundRobin create a RoundRobin
func NewRoundRobin() LoadBalance {
	var ops uint64
	ops = 0

	return RoundRobin{
		ops: &ops,
	}
}

// Select select a server from servers using RoundRobin
func (rr RoundRobin) Select(req *fasthttp.Request, servers *list.List) uint64 {
	l := uint64(servers.Len())

	if 0 >= l {
		return 0
	}

	idx := int(atomic.AddUint64(rr.ops, 1) % l)

	v := collection.Get(servers, idx).Value
	if v == nil {
		return 0
	}

	return v.(*metapb.Server).ID
}
<commit_msg>fix: import error package<commit_after>package lb

import (
	"container/list"
	"github.com/fagongzi/gateway/pkg/pb/metapb"
	"github.com/fagongzi/util/collection"
	"sync/atomic"

	"github.com/valyala/fasthttp"
)

// RoundRobin round robin loadBalance impl
type RoundRobin struct {
	ops *uint64
}

// NewRoundRobin create a RoundRobin
func NewRoundRobin() LoadBalance {
	var ops uint64
	ops = 0

	return RoundRobin{
		ops: &ops,
	}
}

// Select select a server from servers using RoundRobin
func (rr RoundRobin) Select(req *fasthttp.Request, servers *list.List) uint64 {
	l := uint64(servers.Len())

	if 0 >= l {
		return 0
	}

	idx := int(atomic.AddUint64(rr.ops, 1) % l)

	v := collection.Get(servers, idx).Value
	if v == nil {
		return 0
	}

	return v.(*metapb.Server).ID
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2022 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package reports

import (
	"database/sql"
	"fmt"
	"strings"
	"time"

	"github.com/sapcc/limes/pkg/core"
	"github.com/sapcc/limes/pkg/db"
)

var scrapeErrorsQuery = db.SimplifyWhitespaceInSQL(`
	SELECT d.uuid, d.name, p.uuid, p.name, ps.type, ps.checked_at, ps.scrape_error_message
	  FROM projects p
	  LEFT OUTER JOIN domains d ON d.id = p.domain_id
	  LEFT OUTER JOIN project_services ps ON ps.project_id = p.id
	WHERE %s GROUP BY d.uuid, d.name, p.uuid, p.name, ps.type, ps.checked_at, ps.scrape_error_message
	HAVING ps.scrape_error_message != ''
	ORDER BY d.name, p.name, ps.type, ps.scrape_error_message
`)

type ScrapeError struct {
	Project struct {
		ID     string `json:"id"`
		Name   string `json:"name"`
		Domain struct {
			ID   string `json:"id"`
			Name string `json:"name"`
		} `json:"domain"`
	} `json:"project"`
	ServiceType string `json:"service_type"`
	CheckedAt   *int64 `json:"checked_at"`
	Message     string `json:"message"`
}

func GetScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	return getScrapeErrors(cluster, dbi, filter, scrapeErrorsQuery)
}

func GetRateScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	dbQuery := strings.ReplaceAll(scrapeErrorsQuery, "scrape_error_message", "rates_scrape_error_message")
	dbQuery = strings.ReplaceAll(dbQuery, "checked_at", "rates_checked_at")
	return getScrapeErrors(cluster, dbi, filter, dbQuery)
}

func getScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter, dbQuery string) ([]ScrapeError, error) {
	fields := map[string]interface{}{"d.cluster_id": cluster.ID}

	var result []ScrapeError
	queryStr, joinArgs := filter.PrepareQuery(dbQuery)
	whereStr, whereArgs := db.BuildSimpleWhereClause(fields, len(joinArgs))
	err := db.ForeachRow(dbi, fmt.Sprintf(queryStr, whereStr), append(joinArgs, whereArgs...), func(rows *sql.Rows) error {
		sErr := ScrapeError{}
		var checkedAtAsTime *time.Time
		err := rows.Scan(
			&sErr.Project.Domain.ID, &sErr.Project.Domain.Name, &sErr.Project.ID,
			&sErr.Project.Name, &sErr.ServiceType, &checkedAtAsTime, &sErr.Message,
		)
		if err != nil {
			return err
		}

		if checkedAtAsTime != nil {
			v := checkedAtAsTime.Unix()
			sErr.CheckedAt = &v
		}
		result = append(result, sErr)

		return nil
	})
	if err != nil {
		return nil, err
	}

	if len(result) == 0 {
		//Ensure that empty list gets serialized as `[]` rather than as `null`.
		return []ScrapeError{}, nil
	}

	//To avoid excessively large responses, we group identical scrape errors for multiple
	//project services of the same type into one item with a dummy project ID.
	uniqueErrors := make(map[string]map[string]ScrapeError) //map[serviceType]map[Message]ScrapeError
	for _, v := range result {
		if _, found := uniqueErrors[v.ServiceType][v.Message]; found {
			if v.Project.Name == "dummy-project" {
				continue
			}
			v.Project.ID = "uuid-for-dummy-project"
			v.Project.Name = "dummy-project"
			v.Project.Domain.ID = "uuid-for-dummy-domain"
			v.Project.Domain.Name = "dummy-domain"
		}

		if _, ok := uniqueErrors[v.ServiceType]; !ok {
			uniqueErrors[v.ServiceType] = make(map[string]ScrapeError)
		}
		uniqueErrors[v.ServiceType][v.Message] = v
	}
	var new []ScrapeError
	for _, errMsgs := range uniqueErrors {
		for _, sErr := range errMsgs {
			new = append(new, sErr)
		}
	}
	result = new

	return result, nil
}
<commit_msg>pkg/reports: deterministic ordering for unit tests<commit_after>/*******************************************************************************
*
* Copyright 2022 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package reports

import (
	"database/sql"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/sapcc/limes/pkg/core"
	"github.com/sapcc/limes/pkg/db"
)

var scrapeErrorsQuery = db.SimplifyWhitespaceInSQL(`
	SELECT d.uuid, d.name, p.uuid, p.name, ps.type, ps.checked_at, ps.scrape_error_message
	  FROM projects p
	  LEFT OUTER JOIN domains d ON d.id = p.domain_id
	  LEFT OUTER JOIN project_services ps ON ps.project_id = p.id
	WHERE %s GROUP BY d.uuid, d.name, p.uuid, p.name, ps.type, ps.checked_at, ps.scrape_error_message
	HAVING ps.scrape_error_message != ''
	ORDER BY d.name, p.name, ps.type, ps.scrape_error_message
`)

type ScrapeError struct {
	Project struct {
		ID     string `json:"id"`
		Name   string `json:"name"`
		Domain struct {
			ID   string `json:"id"`
			Name string `json:"name"`
		} `json:"domain"`
	} `json:"project"`
	ServiceType string `json:"service_type"`
	CheckedAt   *int64 `json:"checked_at"`
	Message     string `json:"message"`
}

func GetScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	return getScrapeErrors(cluster, dbi, filter, scrapeErrorsQuery)
}

func GetRateScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	dbQuery := strings.ReplaceAll(scrapeErrorsQuery, "scrape_error_message", "rates_scrape_error_message")
	dbQuery = strings.ReplaceAll(dbQuery, "checked_at", "rates_checked_at")
	return getScrapeErrors(cluster, dbi, filter, dbQuery)
}

func getScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter, dbQuery string) ([]ScrapeError, error) {
	fields := map[string]interface{}{"d.cluster_id": cluster.ID}

	var result []ScrapeError
	queryStr, joinArgs := filter.PrepareQuery(dbQuery)
	whereStr, whereArgs := db.BuildSimpleWhereClause(fields, len(joinArgs))
	err := db.ForeachRow(dbi, fmt.Sprintf(queryStr, whereStr), append(joinArgs, whereArgs...), func(rows *sql.Rows) error {
		sErr := ScrapeError{}
		var checkedAtAsTime *time.Time
		err := rows.Scan(
			&sErr.Project.Domain.ID, &sErr.Project.Domain.Name, &sErr.Project.ID,
			&sErr.Project.Name, &sErr.ServiceType, &checkedAtAsTime, &sErr.Message,
		)
		if err != nil {
			return err
		}

		if checkedAtAsTime != nil {
			v := checkedAtAsTime.Unix()
			sErr.CheckedAt = &v
		}
		result = append(result, sErr)

		return nil
	})
	if err != nil {
		return nil, err
	}

	if len(result) == 0 {
		//Ensure that empty list gets serialized as `[]` rather than as `null`.
		return []ScrapeError{}, nil
	}

	//To avoid excessively large responses, we group identical scrape errors for multiple
	//project services of the same type into one item with a dummy project ID.
	uniqueErrors := make(map[string]map[string]ScrapeError) //map[serviceType]map[Message]ScrapeError
	for _, v := range result {
		if _, found := uniqueErrors[v.ServiceType][v.Message]; found {
			if v.Project.Name == "dummy-project" {
				continue
			}
			v.Project.ID = "uuid-for-dummy-project"
			v.Project.Name = "dummy-project"
			v.Project.Domain.ID = "uuid-for-dummy-domain"
			v.Project.Domain.Name = "dummy-domain"
		}

		if _, ok := uniqueErrors[v.ServiceType]; !ok {
			uniqueErrors[v.ServiceType] = make(map[string]ScrapeError)
		}
		uniqueErrors[v.ServiceType][v.Message] = v
	}
	var new []ScrapeError
	for _, errMsgs := range uniqueErrors {
		for _, sErr := range errMsgs {
			new = append(new, sErr)
		}
	}
	//Deterministic ordering for unit tests
	sort.Slice(new, func(i, j int) bool {
		srvType1 := new[i].ServiceType
		srvType2 := new[j].ServiceType
		if srvType1 != srvType2 {
			return srvType1 < srvType2
		}
		pID1 := new[i].Project.ID
		pID2 := new[i].Project.ID
		return pID1 < pID2
	})
	result = new

	return result, nil
}
<|endoftext|>


<commit_before>package sharing

import (
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/lock"
)

// SetupReceiver is used on the receivers' cozy to make sure the cozy can
// receive the shared documents.
func (s *Sharing) SetupReceiver(inst *instance.Instance) error {
	if err := couchdb.EnsureDBExist(inst, consts.Shared); err != nil {
		return err
	}
	if s.HasFiles() {
		return EnsureSharedWithMeDir(inst)
	}
	return nil
}

// Setup is used when a member accepts a sharing to prepare the io.cozy.shared
// database and start an initial replication. It is meant to be used in a new
// goroutine and, as such, does not return errors but log them.
func (s *Sharing) Setup(inst *instance.Instance, m *Member) {
	// Don't do the setup for most tests
	if !inst.OnboardingFinished {
		return
	}

	mu := lock.ReadWrite(inst.Domain + "/sharings/" + s.SID)
	mu.Lock()
	defer mu.Unlock()

	if err := couchdb.EnsureDBExist(inst, consts.Shared); err != nil {
		inst.Logger().Warnf("[sharing] Can't ensure io.cozy.shared exists (%s): %s", s.SID, err)
	}

	if err := s.AddTrackTriggers(inst); err != nil {
		inst.Logger().Warnf("[sharing] Errors on setup of track triggers (%s): %s", s.SID, err)
	}
	// TODO add triggers for rules that can revoke the sharing
	for i, rule := range s.Rules {
		if err := s.InitialCopy(inst, rule, i); err != nil {
			inst.Logger().Warnf("[sharing] Error on initial copy for %s (%s): %s", rule.Title, s.SID, err)
		}
	}
	if err := s.AddReplicateTrigger(inst); err != nil {
		inst.Logger().Warnf("[sharing] Error on setup replicate trigger (%s): %s", s.SID, err)
	}
	if err := s.ReplicateTo(inst, m, true); err != nil {
		inst.Logger().Warnf("[sharing] Error on initial replication (%s): %s", s.SID, err)
		s.retryReplicate(inst, 1)
	}
}

// AddTrackTriggers creates the share-track triggers for each rule of the
// sharing that will update the io.cozy.shared database.
func (s *Sharing) AddTrackTriggers(inst *instance.Instance) error {
	if s.Triggers.Track {
		return nil
	}
	sched := jobs.System()
	for i, rule := range s.Rules {
		args := rule.TriggerArgs(s.Owner)
		if args == "" {
			continue
		}
		msg, err := jobs.NewMessage(&TrackMessage{
			SharingID: s.SID,
			RuleIndex: i,
		})
		if err != nil {
			return err
		}
		t, err := jobs.NewTrigger(&jobs.TriggerInfos{
			Domain:     inst.Domain,
			Type:       "@event",
			WorkerType: "share-track",
			Message:    msg,
			Arguments:  args,
		})
		if err != nil {
			return err
		}
		if err = sched.AddTrigger(t); err != nil {
			return err
		}
	}
	s.Triggers.Track = true
	return couchdb.UpdateDoc(inst, s)
}

// AddReplicateTrigger creates the share-replicate trigger for this sharing:
// it will starts the replicator when some changes are made to the
// io.cozy.shared database.
func (s *Sharing) AddReplicateTrigger(inst *instance.Instance) error {
	if s.Triggers.Replicate {
		return nil
	}
	msg, err := jobs.NewMessage(&ReplicateMsg{
		SharingID: s.SID,
		Errors:    0,
	})
	if err != nil {
		return err
	}
	args := consts.Shared + ":CREATED,UPDATED:" + s.SID + ":sharing"
	t, err := jobs.NewTrigger(&jobs.TriggerInfos{
		Domain:     inst.Domain,
		Type:       "@event",
		WorkerType: "share-replicate",
		Message:    msg,
		Arguments:  args,
		Debounce:   "5s",
	})
	if err != nil {
		return err
	}
	sched := jobs.System()
	if err = sched.AddTrigger(t); err != nil {
		return err
	}
	s.Triggers.Replicate = true
	return couchdb.UpdateDoc(inst, s)
}

// InitialCopy lists the shared documents and put a reference in the
// io.cozy.shared database
func (s *Sharing) InitialCopy(inst *instance.Instance, rule Rule, r int) error {
	if rule.Local || len(rule.Values) == 0 {
		return nil
	}

	mu := lock.ReadWrite(inst.Domain + "/shared")
	mu.Lock()
	defer mu.Unlock()

	docs, err := findDocsToCopy(inst, rule)
	if err != nil {
		return err
	}
	refs, err := s.buildReferences(inst, rule, r, docs)
	if err != nil {
		return err
	}
	refs = compactSlice(refs)
	if len(refs) == 0 {
		return nil
	}
	return couchdb.BulkUpdateDocs(inst, consts.Shared, refs)
}

// findDocsToCopy finds the documents that match the given rule
func findDocsToCopy(inst *instance.Instance, rule Rule) ([]couchdb.JSONDoc, error) {
	var docs []couchdb.JSONDoc
	if rule.Selector == "" || rule.Selector == "id" {
		req := &couchdb.AllDocsRequest{
			Keys: rule.Values,
		}
		if err := couchdb.GetAllDocs(inst, rule.DocType, req, &docs); err != nil {
			return nil, err
		}
	} else {
		// Create index based on selector to retrieve documents to share
		name := "by-" + rule.Selector
		idx := mango.IndexOnFields(rule.DocType, name, []string{rule.Selector})
		if err := couchdb.DefineIndex(inst, idx); err != nil {
			return nil, err
		}
		// Request the index for all values
		for _, val := range rule.Values {
			var results []couchdb.JSONDoc
			req := &couchdb.FindRequest{
				UseIndex: name,
				Selector: mango.Equal(rule.Selector, val),
			}
			if err := couchdb.FindDocs(inst, rule.DocType, req, &results); err != nil {
				return nil, err
			}
			docs = append(docs, results...)
		}
	}
	return docs, nil
}

// buildReferences build the SharedRef to add/update the given docs in the
// io.cozy.shared database
func (s *Sharing) buildReferences(inst *instance.Instance, rule Rule, r int, docs []couchdb.JSONDoc) ([]interface{}, error) {
	ids := make([]string, len(docs))
	for i, doc := range docs {
		ids[i] = rule.DocType + "/" + doc.ID()
	}
	srefs, err := FindReferences(inst, ids)
	if err != nil {
		return nil, err
	}

	refs := make([]interface{}, len(docs))
	for i, doc := range docs {
		rev := doc.Rev()
		if srefs[i] == nil {
			refs[i] = SharedRef{
				SID:       rule.DocType + "/" + doc.ID(),
				Revisions: []string{rev},
				Infos: map[string]SharedInfo{
					s.SID: {Rule: r},
				},
			}
		} else {
			found := false
			for _, revision := range srefs[i].Revisions {
				if revision == rev {
					found = true
					break
				}
			}
			if found {
				if _, ok := srefs[i].Infos[s.SID]; ok {
					continue
				}
			} else {
				srefs[i].Revisions = append(srefs[i].Revisions, rev)
			}
			srefs[i].Infos[s.SID] = SharedInfo{Rule: r}
			refs[i] = *srefs[i]
		}
	}

	return refs, nil
}

// compactSlice returns the given slice without the nil values
// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
func compactSlice(a []interface{}) []interface{} {
	b := a[:0]
	for _, x := range a {
		if x != nil {
			b = append(b, x)
		}
	}
	return b
}
<commit_msg>Add recovery on sharing setup goroutine<commit_after>package sharing

import (
	"fmt"
	"runtime"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/lock"
)

// SetupReceiver is used on the receivers' cozy to make sure the cozy can
// receive the shared documents.
func (s *Sharing) SetupReceiver(inst *instance.Instance) error {
	if err := couchdb.EnsureDBExist(inst, consts.Shared); err != nil {
		return err
	}
	if s.HasFiles() {
		return EnsureSharedWithMeDir(inst)
	}
	return nil
}

// Setup is used when a member accepts a sharing to prepare the io.cozy.shared
// database and start an initial replication. It is meant to be used in a new
// goroutine and, as such, does not return errors but log them.
func (s *Sharing) Setup(inst *instance.Instance, m *Member) {
	defer func() {
		if r := recover(); r != nil {
			var err error
			switch r := r.(type) {
			case error:
				err = r
			default:
				err = fmt.Errorf("%v", r)
			}
			stack := make([]byte, 4<<10) // 4 KB
			length := runtime.Stack(stack, false)
			log := inst.Logger().WithField("panic", true)
			log.Errorf("PANIC RECOVER %s: %s", err.Error(), stack[:length])
		}
	}()

	// Don't do the setup for most tests
	if !inst.OnboardingFinished {
		return
	}

	mu := lock.ReadWrite(inst.Domain + "/sharings/" + s.SID)
	mu.Lock()
	defer mu.Unlock()

	if err := couchdb.EnsureDBExist(inst, consts.Shared); err != nil {
		inst.Logger().Warnf("[sharing] Can't ensure io.cozy.shared exists (%s): %s", s.SID, err)
	}

	if err := s.AddTrackTriggers(inst); err != nil {
		inst.Logger().Warnf("[sharing] Errors on setup of track triggers (%s): %s", s.SID, err)
	}
	// TODO add triggers for rules that can revoke the sharing
	for i, rule := range s.Rules {
		if err := s.InitialCopy(inst, rule, i); err != nil {
			inst.Logger().Warnf("[sharing] Error on initial copy for %s (%s): %s", rule.Title, s.SID, err)
		}
	}
	if err := s.AddReplicateTrigger(inst); err != nil {
		inst.Logger().Warnf("[sharing] Error on setup replicate trigger (%s): %s", s.SID, err)
	}
	if err := s.ReplicateTo(inst, m, true); err != nil {
		inst.Logger().Warnf("[sharing] Error on initial replication (%s): %s", s.SID, err)
		s.retryReplicate(inst, 1)
	}
}

// AddTrackTriggers creates the share-track triggers for each rule of the
// sharing that will update the io.cozy.shared database.
func (s *Sharing) AddTrackTriggers(inst *instance.Instance) error {
	if s.Triggers.Track {
		return nil
	}
	sched := jobs.System()
	for i, rule := range s.Rules {
		args := rule.TriggerArgs(s.Owner)
		if args == "" {
			continue
		}
		msg, err := jobs.NewMessage(&TrackMessage{
			SharingID: s.SID,
			RuleIndex: i,
		})
		if err != nil {
			return err
		}
		t, err := jobs.NewTrigger(&jobs.TriggerInfos{
			Domain:     inst.Domain,
			Type:       "@event",
			WorkerType: "share-track",
			Message:    msg,
			Arguments:  args,
		})
		if err != nil {
			return err
		}
		if err = sched.AddTrigger(t); err != nil {
			return err
		}
	}
	s.Triggers.Track = true
	return couchdb.UpdateDoc(inst, s)
}

// AddReplicateTrigger creates the share-replicate trigger for this sharing:
// it will starts the replicator when some changes are made to the
// io.cozy.shared database.
func (s *Sharing) AddReplicateTrigger(inst *instance.Instance) error {
	if s.Triggers.Replicate {
		return nil
	}
	msg, err := jobs.NewMessage(&ReplicateMsg{
		SharingID: s.SID,
		Errors:    0,
	})
	if err != nil {
		return err
	}
	args := consts.Shared + ":CREATED,UPDATED:" + s.SID + ":sharing"
	t, err := jobs.NewTrigger(&jobs.TriggerInfos{
		Domain:     inst.Domain,
		Type:       "@event",
		WorkerType: "share-replicate",
		Message:    msg,
		Arguments:  args,
		Debounce:   "5s",
	})
	if err != nil {
		return err
	}
	sched := jobs.System()
	if err = sched.AddTrigger(t); err != nil {
		return err
	}
	s.Triggers.Replicate = true
	return couchdb.UpdateDoc(inst, s)
}

// InitialCopy lists the shared documents and put a reference in the
// io.cozy.shared database
func (s *Sharing) InitialCopy(inst *instance.Instance, rule Rule, r int) error {
	if rule.Local || len(rule.Values) == 0 {
		return nil
	}

	mu := lock.ReadWrite(inst.Domain + "/shared")
	mu.Lock()
	defer mu.Unlock()

	docs, err := findDocsToCopy(inst, rule)
	if err != nil {
		return err
	}
	refs, err := s.buildReferences(inst, rule, r, docs)
	if err != nil {
		return err
	}
	refs = compactSlice(refs)
	if len(refs) == 0 {
		return nil
	}
	return couchdb.BulkUpdateDocs(inst, consts.Shared, refs)
}

// findDocsToCopy finds the documents that match the given rule
func findDocsToCopy(inst *instance.Instance, rule Rule) ([]couchdb.JSONDoc, error) {
	var docs []couchdb.JSONDoc
	if rule.Selector == "" || rule.Selector == "id" {
		req := &couchdb.AllDocsRequest{
			Keys: rule.Values,
		}
		if err := couchdb.GetAllDocs(inst, rule.DocType, req, &docs); err != nil {
			return nil, err
		}
	} else {
		// Create index based on selector to retrieve documents to share
		name := "by-" + rule.Selector
		idx := mango.IndexOnFields(rule.DocType, name, []string{rule.Selector})
		if err := couchdb.DefineIndex(inst, idx); err != nil {
			return nil, err
		}
		// Request the index for all values
		for _, val := range rule.Values {
			var results []couchdb.JSONDoc
			req := &couchdb.FindRequest{
				UseIndex: name,
				Selector: mango.Equal(rule.Selector, val),
			}
			if err := couchdb.FindDocs(inst, rule.DocType, req, &results); err != nil {
				return nil, err
			}
			docs = append(docs, results...)
		}
	}
	return docs, nil
}

// buildReferences build the SharedRef to add/update the given docs in the
// io.cozy.shared database
func (s *Sharing) buildReferences(inst *instance.Instance, rule Rule, r int, docs []couchdb.JSONDoc) ([]interface{}, error) {
	ids := make([]string, len(docs))
	for i, doc := range docs {
		ids[i] = rule.DocType + "/" + doc.ID()
	}
	srefs, err := FindReferences(inst, ids)
	if err != nil {
		return nil, err
	}

	refs := make([]interface{}, len(docs))
	for i, doc := range docs {
		rev := doc.Rev()
		if srefs[i] == nil {
			refs[i] = SharedRef{
				SID:       rule.DocType + "/" + doc.ID(),
				Revisions: []string{rev},
				Infos: map[string]SharedInfo{
					s.SID: {Rule: r},
				},
			}
		} else {
			found := false
			for _, revision := range srefs[i].Revisions {
				if revision == rev {
					found = true
					break
				}
			}
			if found {
				if _, ok := srefs[i].Infos[s.SID]; ok {
					continue
				}
			} else {
				srefs[i].Revisions = append(srefs[i].Revisions, rev)
			}
			srefs[i].Infos[s.SID] = SharedInfo{Rule: r}
			refs[i] = *srefs[i]
		}
	}

	return refs, nil
}

// compactSlice returns the given slice without the nil values
// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
func compactSlice(a []interface{}) []interface{} {
	b := a[:0]
	for _, x := range a {
		if x != nil {
			b = append(b, x)
		}
	}
	return b
}
<|endoftext|>


<commit_before>package s3

import (
	"fmt"
	"io"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"

	"github.com/sjansen/hoggle/pkg/storage"
)

type Factory struct {
	Region   string
	Bucket   string
	Prefix   string
	Endpoint string
	Profile  string
}

type Bucket struct {
	bucket     string
	prefix     string
	uploader   *s3manager.Uploader
	downloader *s3manager.Downloader
}

func (f *Factory) New() (storage.Container, error) {
	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
	}
	config := &opts.Config
	if f.Endpoint != "" {
		config.Region = aws.String(f.Region)
		config.Endpoint = aws.String(f.Endpoint)
		if strings.HasPrefix(f.Endpoint, "http://") {
			config.DisableSSL = aws.Bool(true)
		}
		config.S3ForcePathStyle = aws.Bool(true)
	} else if f.Region != "" {
		config.Region = aws.String(f.Region)
	}
	if f.Profile != "" {
		opts.Profile = f.Profile
	}
	sess := session.Must(session.NewSessionWithOptions(opts))
	prefix := f.Prefix
	if len(prefix) > 0 && prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}
	b := &Bucket{
		bucket:     f.Bucket,
		prefix:     prefix,
		uploader:   s3manager.NewUploader(sess),
		downloader: s3manager.NewDownloader(sess),
	}
	return b, nil
}

func (b *Bucket) Download(oid string, dst io.WriterAt) error {
	key := b.oid2key(oid)
	params := &s3.GetObjectInput{
		Bucket: aws.String(b.bucket),
		Key:    aws.String(key),
	}
	if _, err := b.downloader.Download(dst, params); err != nil {
		return err
	} else {
		return nil
	}
}

func (b *Bucket) Upload(oid string, src io.ReadSeeker) error {
	key := b.oid2key(oid)
	params := &s3manager.UploadInput{
		Bucket: aws.String(b.bucket),
		Key:    aws.String(key),
		Body:   src,
	}
	if _, err := b.uploader.Upload(params); err != nil {
		return err
	} else {
		return nil
	}
}

func (b *Bucket) oid2key(oid string) (key string) {
	if len(oid) < 4 {
		key = fmt.Sprintf("%sobjects/%s", b.prefix, oid)
	} else {
		key = fmt.Sprintf("%sobjects/%s/%s/%s", b.prefix, oid[0:2], oid[2:4], oid)
	}
	return
}
<commit_msg>hold onto S3 session to use it for adhoc requests<commit_after>package s3

import (
	"fmt"
	"io"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"

	"github.com/sjansen/hoggle/pkg/storage"
)

type Factory struct {
	Region   string
	Bucket   string
	Prefix   string
	Endpoint string
	Profile  string
}

type Bucket struct {
	bucket     string
	prefix     string
	session    *session.Session
	uploader   *s3manager.Uploader
	downloader *s3manager.Downloader
}

func (f *Factory) New() (storage.Container, error) {
	prefix := f.Prefix
	if len(prefix) > 0 && prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
	}
	if f.Profile != "" {
		opts.Profile = f.Profile
	}

	config := &opts.Config
	if f.Endpoint != "" {
		config.Region = aws.String(f.Region)
		config.Endpoint = aws.String(f.Endpoint)
		if strings.HasPrefix(f.Endpoint, "http://") {
			config.DisableSSL = aws.Bool(true)
		}
		config.S3ForcePathStyle = aws.Bool(true)
	} else if f.Region != "" {
		config.Region = aws.String(f.Region)
	}

	sess := session.Must(session.NewSessionWithOptions(opts))
	b := &Bucket{
		bucket:     f.Bucket,
		prefix:     prefix,
		session:    sess,
		uploader:   s3manager.NewUploader(sess),
		downloader: s3manager.NewDownloader(sess),
	}
	return b, nil
}

func (b *Bucket) Download(oid string, dst io.WriterAt) error {
	key := b.oid2key(oid)
	params := &s3.GetObjectInput{
		Bucket: aws.String(b.bucket),
		Key:    aws.String(key),
	}
	if _, err := b.downloader.Download(dst, params); err != nil {
		return err
	} else {
		return nil
	}
}

func (b *Bucket) Upload(oid string, src io.ReadSeeker) error {
	key := b.oid2key(oid)
	params := &s3manager.UploadInput{
		Bucket: aws.String(b.bucket),
		Key:    aws.String(key),
		Body:   src,
	}
	if _, err := b.uploader.Upload(params); err != nil {
		return err
	} else {
		return nil
	}
}

func (b *Bucket) oid2key(oid string) (key string) {
	if len(oid) < 4 {
		key = fmt.Sprintf("%sobjects/%s", b.prefix, oid)
	} else {
		key = fmt.Sprintf("%sobjects/%s/%s/%s", b.prefix, oid[0:2], oid[2:4], oid)
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"compress/bzip2"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/docker/docker/pkg/archive"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Tar knows how to write files to a tar file.
type Tar struct {
	hardlinks map[uint64]string
	w         *tar.Writer
}

// NewTar will create an instance of Tar that can write files to the writer at f.
func NewTar(f io.Writer) Tar {
	w := tar.NewWriter(f)
	return Tar{
		w:         w,
		hardlinks: map[uint64]string{},
	}
}

// Close will close any open streams used by Tar.
func (t *Tar) Close() {
	t.w.Close()
}

// AddFileToTar adds the file at path p to the tar
func (t *Tar) AddFileToTar(p string) error {
	i, err := os.Lstat(p)
	if err != nil {
		return fmt.Errorf("Failed to get file info for %s: %s", p, err)
	}
	linkDst := ""
	if i.Mode()&os.ModeSymlink != 0 {
		var err error
		linkDst, err = os.Readlink(p)
		if err != nil {
			return err
		}
	}
	if i.Mode()&os.ModeSocket != 0 {
		logrus.Infof("ignoring socket %s, not adding to tar", i.Name())
		return nil
	}
	hdr, err := tar.FileInfoHeader(i, linkDst)
	if err != nil {
		return err
	}

	if p != "/" {
		// Docker uses no leading / in the tarball
		hdr.Name = strings.TrimLeft(p, "/")
	} else {
		// allow entry for / to preserve permission changes etc. (currently ignored anyway by Docker runtime)
		hdr.Name = p
	}

	hardlink, linkDst := t.checkHardlink(p, i)
	if hardlink {
		hdr.Linkname = linkDst
		hdr.Typeflag = tar.TypeLink
		hdr.Size = 0
	}
	if err := t.w.WriteHeader(hdr); err != nil {
		return err
	}
	if !(i.Mode().IsRegular()) || hardlink {
		return nil
	}
	r, err := os.Open(p)
	if err != nil {
		return err
	}
	defer r.Close()
	if _, err := io.Copy(t.w, r); err != nil {
		return err
	}
	return nil
}

func (t *Tar) Whiteout(p string) error {
	dir := filepath.Dir(p)
	name := ".wh." + filepath.Base(p)

	th := &tar.Header{
		Name: filepath.Join(dir, name),
		Size: 0,
	}
	if err := t.w.WriteHeader(th); err != nil {
		return err
	}

	return nil
}

// Returns true if path is hardlink, and the link destination
func (t *Tar) checkHardlink(p string, i os.FileInfo) (bool, string) {
	hardlink := false
	linkDst := ""
	stat := getSyscallStatT(i)
	if stat != nil {
		nlinks := stat.Nlink
		if nlinks > 1 {
			inode := stat.Ino
			if original, exists := t.hardlinks[inode]; exists && original != p {
				hardlink = true
				logrus.Debugf("%s inode exists in hardlinks map, linking to %s", p, original)
				linkDst = original
			} else {
				t.hardlinks[inode] = p
			}
		}
	}
	return hardlink, linkDst
}

func getSyscallStatT(i os.FileInfo) *syscall.Stat_t {
	if sys := i.Sys(); sys != nil {
		if stat, ok := sys.(*syscall.Stat_t); ok {
			return stat
		}
	}
	return nil
}

// UnpackLocalTarArchive unpacks the tar archive at path to the directory dest
// Returns the files extracted from the tar archive
func UnpackLocalTarArchive(path, dest string) ([]string, error) {
	// First, we need to check if the path is a local tar archive
	if compressed, compressionLevel := fileIsCompressedTar(path); compressed {
		file, err := os.Open(path)
		if err != nil {
			return nil, err
		}
		defer file.Close()
		if compressionLevel == archive.Gzip {
			return nil, UnpackCompressedTar(path, dest)
		} else if compressionLevel == archive.Bzip2 {
			bzr := bzip2.NewReader(file)
			return unTar(bzr, dest)
		}
	}
	if fileIsUncompressedTar(path) {
		file, err := os.Open(path)
		if err != nil {
			return nil, err
		}
		defer file.Close()
		return unTar(file, dest)
	}
	return nil, errors.New("path does not lead to local tar archive")
}

//IsFileLocalTarArchive returns true if the file is a local tar archive
func IsFileLocalTarArchive(src string) bool {
	compressed, _ := fileIsCompressedTar(src)
	uncompressed := fileIsUncompressedTar(src)
	return compressed || uncompressed
}

func fileIsCompressedTar(src string) (bool, archive.Compression) {
	r, err := os.Open(src)
	if err != nil {
		return false, -1
	}
	defer r.Close()
	buf, err := ioutil.ReadAll(r)
	if err != nil {
		return false, -1
	}
	compressionLevel := archive.DetectCompression(buf)
	return (compressionLevel > 0), compressionLevel
}

func fileIsUncompressedTar(src string) bool {
	r, err := os.Open(src)
	if err != nil {
		return false
	}
	defer r.Close()
	fi, err := os.Lstat(src)
	if err != nil {
		return false
	}
	if fi.Size() == 0 {
		return false
	}
	tr := tar.NewReader(r)
	if tr == nil {
		return false
	}
	_, err = tr.Next()
	return err == nil
}

// UnpackCompressedTar unpacks the compressed tar at path to dir
func UnpackCompressedTar(path, dir string) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()
	gzr, err := gzip.NewReader(file)
	if err != nil {
		return err
	}
	defer gzr.Close()
	_, err = unTar(gzr, dir)
	return err
}
<commit_msg>Also remove leading / from paths to file which are marked as deleted<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"compress/bzip2"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/docker/docker/pkg/archive"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Tar knows how to write files to a tar file.
type Tar struct {
	hardlinks map[uint64]string
	w         *tar.Writer
}

// NewTar will create an instance of Tar that can write files to the writer at f.
func NewTar(f io.Writer) Tar {
	w := tar.NewWriter(f)
	return Tar{
		w:         w,
		hardlinks: map[uint64]string{},
	}
}

// Close will close any open streams used by Tar.
func (t *Tar) Close() {
	t.w.Close()
}

// AddFileToTar adds the file at path p to the tar
func (t *Tar) AddFileToTar(p string) error {
	i, err := os.Lstat(p)
	if err != nil {
		return fmt.Errorf("Failed to get file info for %s: %s", p, err)
	}
	linkDst := ""
	if i.Mode()&os.ModeSymlink != 0 {
		var err error
		linkDst, err = os.Readlink(p)
		if err != nil {
			return err
		}
	}
	if i.Mode()&os.ModeSocket != 0 {
		logrus.Infof("ignoring socket %s, not adding to tar", i.Name())
		return nil
	}
	hdr, err := tar.FileInfoHeader(i, linkDst)
	if err != nil {
		return err
	}

	if p != "/" {
		// Docker uses no leading / in the tarball
		hdr.Name = strings.TrimLeft(p, "/")
	} else {
		// allow entry for / to preserve permission changes etc. (currently ignored anyway by Docker runtime)
		hdr.Name = p
	}

	hardlink, linkDst := t.checkHardlink(p, i)
	if hardlink {
		hdr.Linkname = linkDst
		hdr.Typeflag = tar.TypeLink
		hdr.Size = 0
	}
	if err := t.w.WriteHeader(hdr); err != nil {
		return err
	}
	if !(i.Mode().IsRegular()) || hardlink {
		return nil
	}
	r, err := os.Open(p)
	if err != nil {
		return err
	}
	defer r.Close()
	if _, err := io.Copy(t.w, r); err != nil {
		return err
	}
	return nil
}

func (t *Tar) Whiteout(p string) error {
	dir := filepath.Dir(p)
	name := ".wh." + filepath.Base(p)

	th := &tar.Header{
		// Docker uses no leading / in the tarball
		Name: strings.TrimLeft(filepath.Join(dir, name), "/"),
		Size: 0,
	}
	if err := t.w.WriteHeader(th); err != nil {
		return err
	}

	return nil
}

// Returns true if path is hardlink, and the link destination
func (t *Tar) checkHardlink(p string, i os.FileInfo) (bool, string) {
	hardlink := false
	linkDst := ""
	stat := getSyscallStatT(i)
	if stat != nil {
		nlinks := stat.Nlink
		if nlinks > 1 {
			inode := stat.Ino
			if original, exists := t.hardlinks[inode]; exists && original != p {
				hardlink = true
				logrus.Debugf("%s inode exists in hardlinks map, linking to %s", p, original)
				linkDst = original
			} else {
				t.hardlinks[inode] = p
			}
		}
	}
	return hardlink, linkDst
}

func getSyscallStatT(i os.FileInfo) *syscall.Stat_t {
	if sys := i.Sys(); sys != nil {
		if stat, ok := sys.(*syscall.Stat_t); ok {
			return stat
		}
	}
	return nil
}

// UnpackLocalTarArchive unpacks the tar archive at path to the directory dest
// Returns the files extracted from the tar archive
func UnpackLocalTarArchive(path, dest string) ([]string, error) {
	// First, we need to check if the path is a local tar archive
	if compressed, compressionLevel := fileIsCompressedTar(path); compressed {
		file, err := os.Open(path)
		if err != nil {
			return nil, err
		}
		defer file.Close()
		if compressionLevel == archive.Gzip {
			return nil, UnpackCompressedTar(path, dest)
		} else if compressionLevel == archive.Bzip2 {
			bzr := bzip2.NewReader(file)
			return unTar(bzr, dest)
		}
	}
	if fileIsUncompressedTar(path) {
		file, err := os.Open(path)
		if err != nil {
			return nil, err
		}
		defer file.Close()
		return unTar(file, dest)
	}
	return nil, errors.New("path does not lead to local tar archive")
}

//IsFileLocalTarArchive returns true if the file is a local tar archive
func IsFileLocalTarArchive(src string) bool {
	compressed, _ := fileIsCompressedTar(src)
	uncompressed := fileIsUncompressedTar(src)
	return compressed || uncompressed
}

func fileIsCompressedTar(src string) (bool, archive.Compression) {
	r, err := os.Open(src)
	if err != nil {
		return false, -1
	}
	defer r.Close()
	buf, err := ioutil.ReadAll(r)
	if err != nil {
		return false, -1
	}
	compressionLevel := archive.DetectCompression(buf)
	return (compressionLevel > 0), compressionLevel
}

func fileIsUncompressedTar(src string) bool {
	r, err := os.Open(src)
	if err != nil {
		return false
	}
	defer r.Close()
	fi, err := os.Lstat(src)
	if err != nil {
		return false
	}
	if fi.Size() == 0 {
		return false
	}
	tr := tar.NewReader(r)
	if tr == nil {
		return false
	}
	_, err = tr.Next()
	return err == nil
}

// UnpackCompressedTar unpacks the compressed tar at path to dir
func UnpackCompressedTar(path, dir string) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()
	gzr, err := gzip.NewReader(file)
	if err != nil {
		return err
	}
	defer gzr.Close()
	_, err = unTar(gzr, dir)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vmtest

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"testing"

	"github.com/u-root/u-root/pkg/golang"
	"github.com/u-root/u-root/pkg/uio"
	"github.com/u-root/u-root/pkg/uroot"
	"github.com/u-root/u-root/pkg/vmtest/internal/json2test"
)

// GolangTest compiles the unit tests found in pkgs and runs them in a QEMU VM.
func GolangTest(t *testing.T, pkgs []string, o *Options) {
	SkipWithoutQEMU(t)
	// TODO: support arm
	if TestArch() != "amd64" && TestArch() != "arm64" {
		t.Skipf("test not supported on %s", TestArch())
	}

	if o == nil {
		o = &Options{}
	}

	// Create a temporary directory.
	if len(o.TmpDir) == 0 {
		tmpDir, err := ioutil.TempDir("", "uroot-integration")
		if err != nil {
			t.Fatal(err)
		}
		o.TmpDir = tmpDir
	}

	// Set up u-root build options.
	env := golang.Default()
	env.CgoEnabled = false
	env.GOARCH = TestArch()
	o.BuildOpts.Env = env

	// Statically build tests and add them to the temporary directory.
	var tests []string
	os.Setenv("CGO_ENABLED", "0")
	os.Setenv("GOARCH", TestArch())
	testDir := filepath.Join(o.TmpDir, "tests")
	for _, pkg := range pkgs {
		pkgDir := filepath.Join(testDir, pkg)
		if err := os.MkdirAll(pkgDir, 0755); err != nil {
			t.Fatal(err)
		}

		testFile := filepath.Join(pkgDir, fmt.Sprintf("%s.test", path.Base(pkg)))

		cmd := exec.Command("go", "test",
			"-gcflags=all=-l",
			"-ldflags", "-s -w",
			"-c", pkg,
			"-o", testFile,
		)
		if stderr, err := cmd.CombinedOutput(); err != nil {
			t.Fatalf("could not build %s: %v\n%s", pkg, err, string(stderr))
		}

		// When a package does not contain any tests, the test
		// executable is not generated, so it is not included in the
		// `tests` list.
		if _, err := os.Stat(testFile); !os.IsNotExist(err) {
			tests = append(tests, pkg)

			p, err := o.BuildOpts.Env.Package(pkg)
			if err != nil {
				t.Fatal(err)
			}
			// Optimistically copy any files in the pkg's
			// directory, in case e.g. a testdata dir is there.
			if err := copyRelativeFiles(p.Dir, filepath.Join(testDir, pkg)); err != nil {
				t.Fatal(err)
			}
		}
	}

	// Create the CPIO and start QEMU.
	o.BuildOpts.AddBusyBoxCommands("github.com/u-root/u-root/cmds/core/*")
	o.BuildOpts.AddCommands(uroot.BinaryCmds("cmd/test2json")...)

	// Specify the custom gotest uinit.
	o.Uinit = "github.com/u-root/u-root/integration/testcmd/gotest/uinit"

	tc := json2test.NewTestCollector()
	serial := []io.Writer{
		// Collect JSON test events in tc.
		json2test.EventParser(tc),
		// Write non-JSON output to log.
		JSONLessTestLineWriter(t, "serial"),
	}
	if o.QEMUOpts.SerialOutput != nil {
		serial = append(serial, o.QEMUOpts.SerialOutput)
	}
	o.QEMUOpts.SerialOutput = uio.MultiWriteCloser(serial...)

	q, cleanup := QEMUTest(t, o)
	defer cleanup()

	if err := q.Expect("TESTS PASSED MARKER"); err != nil {
		t.Errorf("Waiting for 'TESTS PASSED MARKER' signal: %v", err)
	}

	// TODO: check that tc.Tests == tests
	for pkg, test := range tc.Tests {
		switch test.State {
		case json2test.StateFail:
			t.Errorf("Test %v failed:\n%v", pkg, test.FullOutput)
		case json2test.StateSkip:
			t.Logf("Test %v skipped", pkg)
		case json2test.StatePass:
			// Nothing.
		default:
			t.Errorf("Test %v left in state %v:\n%v", pkg, test.State, test.FullOutput)
		}
	}
}

func copyRelativeFiles(src string, dst string) error {
	return filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		if fi.Mode().IsDir() {
			return os.MkdirAll(filepath.Join(dst, rel), fi.Mode().Perm())
		} else if fi.Mode().IsRegular() {
			srcf, err := os.Open(path)
			if err != nil {
				return err
			}
			defer srcf.Close()
			dstf, err := os.Create(filepath.Join(dst, rel))
			if err != nil {
				return err
			}
			defer dstf.Close()
			_, err = io.Copy(dstf, srcf)
			return err
		}
		return nil
	})
}
<commit_msg>vmtest: use same Go version to compile initramfs and Go tests<commit_after>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vmtest

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"testing"

	"github.com/u-root/u-root/pkg/golang"
	"github.com/u-root/u-root/pkg/uio"
	"github.com/u-root/u-root/pkg/uroot"
	"github.com/u-root/u-root/pkg/vmtest/internal/json2test"
)

// GolangTest compiles the unit tests found in pkgs and runs them in a QEMU VM.
func GolangTest(t *testing.T, pkgs []string, o *Options) {
	SkipWithoutQEMU(t)
	// TODO: support arm
	if TestArch() != "amd64" && TestArch() != "arm64" {
		t.Skipf("test not supported on %s", TestArch())
	}

	if o == nil {
		o = &Options{}
	}

	// Create a temporary directory.
	if len(o.TmpDir) == 0 {
		tmpDir, err := ioutil.TempDir("", "uroot-integration")
		if err != nil {
			t.Fatal(err)
		}
		o.TmpDir = tmpDir
	}

	// Set up u-root build options.
	env := golang.Default()
	env.CgoEnabled = false
	env.GOARCH = TestArch()
	o.BuildOpts.Env = env

	// Statically build tests and add them to the temporary directory.
	var tests []string
	testDir := filepath.Join(o.TmpDir, "tests")
	for _, pkg := range pkgs {
		pkgDir := filepath.Join(testDir, pkg)
		if err := os.MkdirAll(pkgDir, 0755); err != nil {
			t.Fatal(err)
		}

		testFile := filepath.Join(pkgDir, fmt.Sprintf("%s.test", path.Base(pkg)))

		cmd := env.GoCmd("test",
			"-gcflags=all=-l",
			"-ldflags", "-s -w",
			"-c", pkg,
			"-o", testFile,
		)
		if stderr, err := cmd.CombinedOutput(); err != nil {
			t.Fatalf("could not build %s: %v\n%s", pkg, err, string(stderr))
		}

		// When a package does not contain any tests, the test
		// executable is not generated, so it is not included in the
		// `tests` list.
		if _, err := os.Stat(testFile); !os.IsNotExist(err) {
			tests = append(tests, pkg)

			p, err := o.BuildOpts.Env.Package(pkg)
			if err != nil {
				t.Fatal(err)
			}
			// Optimistically copy any files in the pkg's
			// directory, in case e.g. a testdata dir is there.
			if err := copyRelativeFiles(p.Dir, filepath.Join(testDir, pkg)); err != nil {
				t.Fatal(err)
			}
		}
	}

	// Create the CPIO and start QEMU.
	o.BuildOpts.AddBusyBoxCommands("github.com/u-root/u-root/cmds/core/*")
	o.BuildOpts.AddCommands(uroot.BinaryCmds("cmd/test2json")...)

	// Specify the custom gotest uinit.
	o.Uinit = "github.com/u-root/u-root/integration/testcmd/gotest/uinit"

	tc := json2test.NewTestCollector()
	serial := []io.Writer{
		// Collect JSON test events in tc.
		json2test.EventParser(tc),
		// Write non-JSON output to log.
		JSONLessTestLineWriter(t, "serial"),
	}
	if o.QEMUOpts.SerialOutput != nil {
		serial = append(serial, o.QEMUOpts.SerialOutput)
	}
	o.QEMUOpts.SerialOutput = uio.MultiWriteCloser(serial...)

	q, cleanup := QEMUTest(t, o)
	defer cleanup()

	if err := q.Expect("TESTS PASSED MARKER"); err != nil {
		t.Errorf("Waiting for 'TESTS PASSED MARKER' signal: %v", err)
	}

	// TODO: check that tc.Tests == tests
	for pkg, test := range tc.Tests {
		switch test.State {
		case json2test.StateFail:
			t.Errorf("Test %v failed:\n%v", pkg, test.FullOutput)
		case json2test.StateSkip:
			t.Logf("Test %v skipped", pkg)
		case json2test.StatePass:
			// Nothing.
		default:
			t.Errorf("Test %v left in state %v:\n%v", pkg, test.State, test.FullOutput)
		}
	}
}

func copyRelativeFiles(src string, dst string) error {
	return filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		if fi.Mode().IsDir() {
			return os.MkdirAll(filepath.Join(dst, rel), fi.Mode().Perm())
		} else if fi.Mode().IsRegular() {
			srcf, err := os.Open(path)
			if err != nil {
				return err
			}
			defer srcf.Close()
			dstf, err := os.Create(filepath.Join(dst, rel))
			if err != nil {
				return err
			}
			defer dstf.Close()
			_, err = io.Copy(dstf, srcf)
			return err
		}
		return nil
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/codegangsta/cli"
)

var appCommand = cli.Command{
	Name:   "apps",
	Usage:  "view and mangage applications",
	Action: appAction,
}

func appAction(context *cli.Context) {
	apps, err := registry.FetchApplications()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
	fmt.Fprint(w, "ID\tCONTAINERS\tPORTS\tCPUS\tMEMORY\n")

	for _, a := range apps {
		var (
			ports  = ""
			cpus   int
			memory int
		)

		for _, p := range a.Ports {
			if p.Host != 0 {
				ports += fmt.Sprint(p.Host)
			}
		}

		for _, c := range a.Containers {
			cpus += len(c.Cpus)
			memory += c.Memory
		}

		fmt.Fprintf(w, "%s\t%d\t%s\t%d\t%d\n", a.ID, len(a.Containers), ports, cpus, memory)
	}

	w.Flush()
}
<commit_msg>minor cli fix for ports<commit_after>package main

import (
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/codegangsta/cli"
)

var appCommand = cli.Command{
	Name:   "apps",
	Usage:  "view and mangage applications",
	Action: appAction,
}

func appAction(context *cli.Context) {
	apps, err := registry.FetchApplications()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
	fmt.Fprint(w, "ID\tCONTAINERS\tPORTS\tCPUS\tMEMORY\n")

	for _, a := range apps {
		var (
			ports  = ""
			cpus   int
			memory int
		)

		for _, p := range a.Ports {
			if p.Host != 0 {
				ports += fmt.Sprintf("%d ", p.Host)
			}
		}

		for _, c := range a.Containers {
			cpus += len(c.Cpus)
			memory += c.Memory
		}

		fmt.Fprintf(w, "%s\t%d\t%s\t%d\t%d\n", a.ID, len(a.Containers), ports, cpus, memory)
	}

	w.Flush()
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.

package cpuid

import (
	"fmt"
	"testing"
)

// There is no real way to test a CPU identifier, since results will
// obviously differ on each machine.
func TestCPUID(t *testing.T) {
	n := maxExtendedFunction()
	t.Logf("MAX:%x\n", n)
	t.Log("Name:", CPU.BrandName)
	t.Log("PhysicalCores:", CPU.PhysicalCores)
	t.Log("ThreadsPerCore:", CPU.ThreadsPerCore)
	t.Log("LogicalCores:", CPU.LogicalCores)
	t.Log("Family", CPU.Family, "Mt odel:", CPU.Model)
	t.Log("Features:", CPU.Features)
	t.Log("Cacheline bytes:", CPU.CacheLine)
	if CPU.SSE2() {
		t.Log("Yay - we have SSE 2")
	}
}

func Example() {
	// Print basic CPU information:
	fmt.Println("Name:", CPU.BrandName)
	fmt.Println("PhysicalCores:", CPU.PhysicalCores)
	fmt.Println("ThreadsPerCore:", CPU.ThreadsPerCore)
	fmt.Println("LogicalCores:", CPU.LogicalCores)
	fmt.Println("Family", CPU.Family, "Model:", CPU.Model)
	fmt.Println("Features:", CPU.Features)
	fmt.Println("Cacheline bytes:", CPU.CacheLine)

	// Test if we have a specific feature:
	if CPU.SSE() {
		fmt.Println("We have Streaming SIMD Extensions")
	}
}

// Generated here: http://play.golang.org/p/31X7Inkrpp

// TestCmov tests Cmov() function
func TestCmov(t *testing.T) {
	got := CPU.Cmov()
	expected := CPU.Features&CMOV == CMOV
	if got != expected {
		t.Fatalf("Cmov: expected %v, got %v", expected, got)
	}
	t.Log("CMOV Support:", got)
}

// TestAmd3dnow tests Amd3dnow() function
func TestAmd3dnow(t *testing.T) {
	got := CPU.Amd3dnow()
	expected := CPU.Features&AMD3DNOW == AMD3DNOW
	if got != expected {
		t.Fatalf("Amd3dnow: expected %v, got %v", expected, got)
	}
	t.Log("AMD3DNOW Support:", got)
}

// TestAmd3dnowExt tests Amd3dnowExt() function
func TestAmd3dnowExt(t *testing.T) {
	got := CPU.Amd3dnowExt()
	expected := CPU.Features&AMD3DNOWEXT == AMD3DNOWEXT
	if got != expected {
		t.Fatalf("Amd3dnowExt: expected %v, got %v", expected, got)
	}
	t.Log("AMD3DNOWEXT Support:", got)
}

// TestMMX tests MMX() function
func TestMMX(t *testing.T) {
	got := CPU.MMX()
	expected := CPU.Features&MMX == MMX
	if got != expected {
		t.Fatalf("MMX: expected %v, got %v", expected, got)
	}
	t.Log("MMX Support:", got)
}

// TestMMXext tests MMXext() function
func TestMMXext(t *testing.T) {
	got := CPU.MMXExt()
	expected := CPU.Features&MMXEXT == MMXEXT
	if got != expected {
		t.Fatalf("MMXExt: expected %v, got %v", expected, got)
	}
	t.Log("MMXEXT Support:", got)
}

// TestSSE tests SSE() function
func TestSSE(t *testing.T) {
	got := CPU.SSE()
	expected := CPU.Features&SSE == SSE
	if got != expected {
		t.Fatalf("SSE: expected %v, got %v", expected, got)
	}
	t.Log("SSE Support:", got)
}

// TestSSE2 tests SSE2() function
func TestSSE2(t *testing.T) {
	got := CPU.SSE2()
	expected := CPU.Features&SSE2 == SSE2
	if got != expected {
		t.Fatalf("SSE2: expected %v, got %v", expected, got)
	}
	t.Log("SSE2 Support:", got)
}

// TestSSE3 tests SSE3() function
func TestSSE3(t *testing.T) {
	got := CPU.SSE3()
	expected := CPU.Features&SSE3 == SSE3
	if got != expected {
		t.Fatalf("SSE3: expected %v, got %v", expected, got)
	}
	t.Log("SSE3 Support:", got)
}

// TestSSSE3 tests SSSE3() function
func TestSSSE3(t *testing.T) {
	got := CPU.SSSE3()
	expected := CPU.Features&SSSE3 == SSSE3
	if got != expected {
		t.Fatalf("SSSE3: expected %v, got %v", expected, got)
	}
	t.Log("SSSE3 Support:", got)
}

// TestSSE4 tests SSE4() function
func TestSSE4(t *testing.T) {
	got := CPU.SSE4()
	expected := CPU.Features&SSE4 == SSE4
	if got != expected {
		t.Fatalf("SSE4: expected %v, got %v", expected, got)
	}
	t.Log("SSE4 Support:", got)
}

// TestSSE42 tests SSE42() function
func TestSSE42(t *testing.T) {
	got := CPU.SSE42()
	expected := CPU.Features&SSE42 == SSE42
	if got != expected {
		t.Fatalf("SSE42: expected %v, got %v", expected, got)
	}
	t.Log("SSE42 Support:", got)
}

// TestAVX tests AVX() function
func TestAVX(t *testing.T) {
	got := CPU.AVX()
	expected := CPU.Features&AVX == AVX
	if got != expected {
		t.Fatalf("AVX: expected %v, got %v", expected, got)
	}
	t.Log("AVX Support:", got)
}

// TestAVX2 tests AVX2() function
func TestAVX2(t *testing.T) {
	got := CPU.AVX2()
	expected := CPU.Features&AVX2 == AVX2
	if got != expected {
		t.Fatalf("AVX2: expected %v, got %v", expected, got)
	}
	t.Log("AVX2 Support:", got)
}

// TestFMA3 tests FMA3() function
func TestFMA3(t *testing.T) {
	got := CPU.FMA3()
	expected := CPU.Features&FMA3 == FMA3
	if got != expected {
		t.Fatalf("FMA3: expected %v, got %v", expected, got)
	}
	t.Log("FMA3 Support:", got)
}

// TestFMA4 tests FMA4() function
func TestFMA4(t *testing.T) {
	got := CPU.FMA4()
	expected := CPU.Features&FMA4 == FMA4
	if got != expected {
		t.Fatalf("FMA4: expected %v, got %v", expected, got)
	}
	t.Log("FMA4 Support:", got)
}

// TestXOP tests XOP() function
func TestXOP(t *testing.T) {
	got := CPU.XOP()
	expected := CPU.Features&XOP == XOP
	if got != expected {
		t.Fatalf("XOP: expected %v, got %v", expected, got)
	}
	t.Log("XOP Support:", got)
}

// TestBMI1 tests BMI1() function
func TestBMI1(t *testing.T) {
	got := CPU.BMI1()
	expected := CPU.Features&BMI1 == BMI1
	if got != expected {
		t.Fatalf("BMI1: expected %v, got %v", expected, got)
	}
	t.Log("BMI1 Support:", got)
}

// TestBMI2 tests BMI2() function
func TestBMI2(t *testing.T) {
	got := CPU.BMI2()
	expected := CPU.Features&BMI2 == BMI2
	if got != expected {
		t.Fatalf("BMI2: expected %v, got %v", expected, got)
	}
	t.Log("BMI2 Support:", got)
}

// TestTBM tests TBM() function
func TestTBM(t *testing.T) {
	got := CPU.TBM()
	expected := CPU.Features&TBM == TBM
	if got != expected {
		t.Fatalf("TBM: expected %v, got %v", expected, got)
	}
	t.Log("TBM Support:", got)
}

// TestLzcnt tests Lzcnt() function
func TestLzcnt(t *testing.T) {
	got := CPU.Lzcnt()
	expected := CPU.Features&LZCNT == LZCNT
	if got != expected {
		t.Fatalf("Lzcnt: expected %v, got %v", expected, got)
	}
	t.Log("LZCNT Support:", got)
}

// TestAesNi tests AesNi() function
func TestAesNi(t *testing.T) {
	got := CPU.AesNi()
	expected := CPU.Features&AESNI == AESNI
	if got != expected {
		t.Fatalf("AesNi: expected %v, got %v", expected, got)
	}
	t.Log("AESNI Support:", got)
}

// TestClmul tests Clmul() function
func TestClmul(t *testing.T) {
	got := CPU.Clmul()
	expected := CPU.Features&CLMUL == CLMUL
	if got != expected {
		t.Fatalf("Clmul: expected %v, got %v", expected, got)
	}
	t.Log("CLMUL Support:", got)
}

// TestSSE2Slow tests SSE2Slow() function
func TestSSE2Slow(t *testing.T) {
	got := CPU.SSE2Slow()
	expected := CPU.Features&SSE2SLOW == SSE2SLOW
	if got != expected {
		t.Fatalf("SSE2Slow: expected %v, got %v", expected, got)
	}
	t.Log("SSE2SLOW Support:", got)
}

// TestSSE3Slow tests SSE3slow() function
func TestSSE3Slow(t *testing.T) {
	got := CPU.SSE3Slow()
	expected := CPU.Features&SSE3SLOW == SSE3SLOW
	if got != expected {
		t.Fatalf("SSE3slow: expected %v, got %v", expected, got)
	}
	t.Log("SSE3SLOW Support:", got)
}

// TestAtom tests Atom() function
func TestAtom(t *testing.T) {
	got := CPU.Atom()
	expected := CPU.Features&ATOM == ATOM
	if got != expected {
		t.Fatalf("Atom: expected %v, got %v", expected, got)
	}
	t.Log("ATOM Support:", got)
}
<commit_msg>Fix typo<commit_after>// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.

package cpuid

import (
	"fmt"
	"testing"
)

// There is no real way to test a CPU identifier, since results will
// obviously differ on each machine.
func TestCPUID(t *testing.T) {
	n := maxExtendedFunction()
	t.Logf("MAX:%x\n", n)
	t.Log("Name:", CPU.BrandName)
	t.Log("PhysicalCores:", CPU.PhysicalCores)
	t.Log("ThreadsPerCore:", CPU.ThreadsPerCore)
	t.Log("LogicalCores:", CPU.LogicalCores)
	t.Log("Family", CPU.Family, "Model:", CPU.Model)
	t.Log("Features:", CPU.Features)
	t.Log("Cacheline bytes:", CPU.CacheLine)
	if CPU.SSE2() {
		t.Log("Yay - we have SSE 2")
	}
}

func Example() {
	// Print basic CPU information:
	fmt.Println("Name:", CPU.BrandName)
	fmt.Println("PhysicalCores:", CPU.PhysicalCores)
	fmt.Println("ThreadsPerCore:", CPU.ThreadsPerCore)
	fmt.Println("LogicalCores:", CPU.LogicalCores)
	fmt.Println("Family", CPU.Family, "Model:", CPU.Model)
	fmt.Println("Features:", CPU.Features)
	fmt.Println("Cacheline bytes:", CPU.CacheLine)

	// Test if we have a specific feature:
	if CPU.SSE() {
		fmt.Println("We have Streaming SIMD Extensions")
	}
}

// Generated here: http://play.golang.org/p/31X7Inkrpp

// TestCmov tests Cmov() function
func TestCmov(t *testing.T) {
	got := CPU.Cmov()
	expected := CPU.Features&CMOV == CMOV
	if got != expected {
		t.Fatalf("Cmov: expected %v, got %v", expected, got)
	}
	t.Log("CMOV Support:", got)
}

// TestAmd3dnow tests Amd3dnow() function
func TestAmd3dnow(t *testing.T) {
	got := CPU.Amd3dnow()
	expected := CPU.Features&AMD3DNOW == AMD3DNOW
	if got != expected {
		t.Fatalf("Amd3dnow: expected %v, got %v", expected, got)
	}
	t.Log("AMD3DNOW Support:", got)
}

// TestAmd3dnowExt tests Amd3dnowExt() function
func TestAmd3dnowExt(t *testing.T) {
	got := CPU.Amd3dnowExt()
	expected := CPU.Features&AMD3DNOWEXT == AMD3DNOWEXT
	if got != expected {
		t.Fatalf("Amd3dnowExt: expected %v, got %v", expected, got)
	}
	t.Log("AMD3DNOWEXT Support:", got)
}

// TestMMX tests MMX() function
func TestMMX(t *testing.T) {
	got := CPU.MMX()
	expected := CPU.Features&MMX == MMX
	if got != expected {
		t.Fatalf("MMX: expected %v, got %v", expected, got)
	}
	t.Log("MMX Support:", got)
}

// TestMMXext tests MMXext() function
func TestMMXext(t *testing.T) {
	got := CPU.MMXExt()
	expected := CPU.Features&MMXEXT == MMXEXT
	if got != expected {
		t.Fatalf("MMXExt: expected %v, got %v", expected, got)
	}
	t.Log("MMXEXT Support:", got)
}

// TestSSE tests SSE() function
func TestSSE(t *testing.T) {
	got := CPU.SSE()
	expected := CPU.Features&SSE == SSE
	if got != expected {
		t.Fatalf("SSE: expected %v, got %v", expected, got)
	}
	t.Log("SSE Support:", got)
}

// TestSSE2 tests SSE2() function
func TestSSE2(t *testing.T) {
	got := CPU.SSE2()
	expected := CPU.Features&SSE2 == SSE2
	if got != expected {
		t.Fatalf("SSE2: expected %v, got %v", expected, got)
	}
	t.Log("SSE2 Support:", got)
}

// TestSSE3 tests SSE3() function
func TestSSE3(t *testing.T) {
	got := CPU.SSE3()
	expected := CPU.Features&SSE3 == SSE3
	if got != expected {
		t.Fatalf("SSE3: expected %v, got %v", expected, got)
	}
	t.Log("SSE3 Support:", got)
}

// TestSSSE3 tests SSSE3() function
func TestSSSE3(t *testing.T) {
	got := CPU.SSSE3()
	expected := CPU.Features&SSSE3 == SSSE3
	if got != expected {
		t.Fatalf("SSSE3: expected %v, got %v", expected, got)
	}
	t.Log("SSSE3 Support:", got)
}

// TestSSE4 tests SSE4() function
func TestSSE4(t *testing.T) {
	got := CPU.SSE4()
	expected := CPU.Features&SSE4 == SSE4
	if got != expected {
		t.Fatalf("SSE4: expected %v, got %v", expected, got)
	}
	t.Log("SSE4 Support:", got)
}

// TestSSE42 tests SSE42() function
func TestSSE42(t *testing.T) {
	got := CPU.SSE42()
	expected := CPU.Features&SSE42 == SSE42
	if got != expected {
		t.Fatalf("SSE42: expected %v, got %v", expected, got)
	}
	t.Log("SSE42 Support:", got)
}

// TestAVX tests AVX() function
func TestAVX(t *testing.T) {
	got := CPU.AVX()
	expected := CPU.Features&AVX == AVX
	if got != expected {
		t.Fatalf("AVX: expected %v, got %v", expected, got)
	}
	t.Log("AVX Support:", got)
}

// TestAVX2 tests AVX2() function
func TestAVX2(t *testing.T) {
	got := CPU.AVX2()
	expected := CPU.Features&AVX2 == AVX2
	if got != expected {
		t.Fatalf("AVX2: expected %v, got %v", expected, got)
	}
	t.Log("AVX2 Support:", got)
}

// TestFMA3 tests FMA3() function
func TestFMA3(t *testing.T) {
	got := CPU.FMA3()
	expected := CPU.Features&FMA3 == FMA3
	if got != expected {
		t.Fatalf("FMA3: expected %v, got %v", expected, got)
	}
	t.Log("FMA3 Support:", got)
}

// TestFMA4 tests FMA4() function
func TestFMA4(t *testing.T) {
	got := CPU.FMA4()
	expected := CPU.Features&FMA4 == FMA4
	if got != expected {
		t.Fatalf("FMA4: expected %v, got %v", expected, got)
	}
	t.Log("FMA4 Support:", got)
}

// TestXOP tests XOP() function
func TestXOP(t *testing.T) {
	got := CPU.XOP()
	expected := CPU.Features&XOP == XOP
	if got != expected {
		t.Fatalf("XOP: expected %v, got %v", expected, got)
	}
	t.Log("XOP Support:", got)
}

// TestBMI1 tests BMI1() function
func TestBMI1(t *testing.T) {
	got := CPU.BMI1()
	expected := CPU.Features&BMI1 == BMI1
	if got != expected {
		t.Fatalf("BMI1: expected %v, got %v", expected, got)
	}
	t.Log("BMI1 Support:", got)
}

// TestBMI2 tests BMI2() function
func TestBMI2(t *testing.T) {
	got := CPU.BMI2()
	expected := CPU.Features&BMI2 == BMI2
	if got != expected {
		t.Fatalf("BMI2: expected %v, got %v", expected, got)
	}
	t.Log("BMI2 Support:", got)
}

// TestTBM tests TBM() function
func TestTBM(t *testing.T) {
	got := CPU.TBM()
	expected := CPU.Features&TBM == TBM
	if got != expected {
		t.Fatalf("TBM: expected %v, got %v", expected, got)
	}
	t.Log("TBM Support:", got)
}

// TestLzcnt tests Lzcnt() function
func TestLzcnt(t *testing.T) {
	got := CPU.Lzcnt()
	expected := CPU.Features&LZCNT == LZCNT
	if got != expected {
		t.Fatalf("Lzcnt: expected %v, got %v", expected, got)
	}
	t.Log("LZCNT Support:", got)
}

// TestAesNi tests AesNi() function
func TestAesNi(t *testing.T) {
	got := CPU.AesNi()
	expected := CPU.Features&AESNI == AESNI
	if got != expected {
		t.Fatalf("AesNi: expected %v, got %v", expected, got)
	}
	t.Log("AESNI Support:", got)
}

// TestClmul tests Clmul() function
func TestClmul(t *testing.T) {
	got := CPU.Clmul()
	expected := CPU.Features&CLMUL == CLMUL
	if got != expected {
		t.Fatalf("Clmul: expected %v, got %v", expected, got)
	}
	t.Log("CLMUL Support:", got)
}

// TestSSE2Slow tests SSE2Slow() function
func TestSSE2Slow(t *testing.T) {
	got := CPU.SSE2Slow()
	expected := CPU.Features&SSE2SLOW == SSE2SLOW
	if got != expected {
		t.Fatalf("SSE2Slow: expected %v, got %v", expected, got)
	}
	t.Log("SSE2SLOW Support:", got)
}

// TestSSE3Slow tests SSE3slow() function
func TestSSE3Slow(t *testing.T) {
	got := CPU.SSE3Slow()
	expected := CPU.Features&SSE3SLOW == SSE3SLOW
	if got != expected {
		t.Fatalf("SSE3slow: expected %v, got %v", expected, got)
	}
	t.Log("SSE3SLOW Support:", got)
}

// TestAtom tests Atom() function
func TestAtom(t *testing.T) {
	got := CPU.Atom()
	expected := CPU.Features&ATOM == ATOM
	if got != expected {
		t.Fatalf("Atom: expected %v, got %v", expected, got)
	}
	t.Log("ATOM Support:", got)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// apiserver is the main api server and master for the cluster.
// it is responsible for serving the cluster management API.
package main

import (
	"flag"
	"math/rand"
	"net"
	"net/http"
	"strconv"
	"time"

	"code.google.com/p/goprotobuf/proto"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/registry"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	log "github.com/golang/glog"
	"github.com/mesosphere/kubernetes-mesos/framework"
	"github.com/mesosphere/mesos-go/mesos"
)

var (
	port                        = flag.Uint("port", 8080, "The port to listen on.  Default 8080.")
	address                     = flag.String("address", "127.0.0.1", "The address on the local server to listen to. Default 127.0.0.1")
	apiPrefix                   = flag.String("api_prefix", "/api/v1beta1", "The prefix for API requests on the server. Default '/api/v1beta1'")
	cloudProvider               = flag.String("cloud_provider", "", "The provider for cloud services.  Empty string for no provider.")
	minionRegexp                = flag.String("minion_regexp", "", "If non empty, and -cloud_provider is specified, a regular expression for matching minion VMs")
	minionPort                  = flag.Uint("minion_port", 10250, "The port at which kubelet will be listening on the minions.")
	mesosMaster                 = flag.String("mesos_master", "localhost:5050", "Location of leading Mesos master")
	executorURI                 = flag.String("executor_uri", "", "URI of dir that contains the executor executable")
	etcdServerList, machineList util.StringList
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "Servers for the etcd (http://ip:port), comma separated")
	flag.Var(&machineList, "machines", "List of machines to schedule onto, comma separated.")
}

type kubernatesMaster struct {
	podRegistry        registry.PodRegistry
	controllerRegistry registry.ControllerRegistry
	serviceRegistry    registry.ServiceRegistry
	minionRegistry     registry.MinionRegistry
	storage            map[string]apiserver.RESTStorage
	client             *client.Client
}

// Copied from cmd/apiserver.go
func main() {
	flag.Parse()
	util.InitLogs()
	defer util.FlushLogs()

	if len(machineList) == 0 {
		log.Fatal("No machines specified!")
	}

	var cloud cloudprovider.Interface
	switch *cloudProvider {
	case "gce":
		var err error
		cloud, err = cloudprovider.NewGCECloud()
		if err != nil {
			log.Fatal("Couldn't connect to GCE cloud: %#v", err)
		}
	default:
		if len(*cloudProvider) > 0 {
			log.Infof("Unknown cloud provider: %s", *cloudProvider)
		} else {
			log.Info("No cloud provider specified.")
		}
	}

	// TODO(yifan): Let mesos handle pod info getter.
	podInfoGetter := &client.HTTPPodInfoGetter{
		Client: http.DefaultClient,
		Port:   *minionPort,
	}

	client := client.New("http://"+net.JoinHostPort(*address, strconv.Itoa(int(*port))), nil)

	// Create mesos scheduler driver.
	executor := &mesos.ExecutorInfo{
		ExecutorId: &mesos.ExecutorID{Value: proto.String("KubeleteExecutorID")},
		Command: &mesos.CommandInfo{
			Value: proto.String("./executor_runner"),
			Uris: []*mesos.CommandInfo_URI{
				&mesos.CommandInfo_URI{Value: executorURI},
			},
		},
		Name:   proto.String("Executor for kubelete"),
		Source: proto.String("for test"),
	}

	mesosPodScheduler := framework.New(executor, framework.FCFSScheduleFunc)
	driver := &mesos.MesosSchedulerDriver{
		Master: *mesosMaster,
		Framework: mesos.FrameworkInfo{
			Name: proto.String("KubernetesFramework"),
			User: proto.String(""),
		},
		Scheduler: mesosPodScheduler,
	}
	mesosPodScheduler.Driver = driver

	driver.Init()
	defer driver.Destroy()
	go driver.Start()

	m := newKubernatesMaster(mesosPodScheduler, &master.Config{
		Client:        client,
		Cloud:         cloud,
		Minions:       machineList,
		PodInfoGetter: podInfoGetter,
	})
	log.Fatal(m.run(net.JoinHostPort(*address, strconv.Itoa(int(*port))), *apiPrefix))
}

func newKubernatesMaster(mesosPodScheduler *framework.KubernetesFramework, c *master.Config) *kubernatesMaster {
	m := &kubernatesMaster{
		podRegistry:        mesosPodScheduler,
		controllerRegistry: registry.MakeMemoryRegistry(),
		serviceRegistry:    registry.MakeMemoryRegistry(),
		minionRegistry:     registry.MakeMinionRegistry(c.Minions),
		client:             c.Client,
	}
	m.init(c.Cloud, c.PodInfoGetter)
	return m
}

func (m *kubernatesMaster) init(cloud cloudprovider.Interface, podInfoGetter client.PodInfoGetter) {
	podCache := master.NewPodCache(podInfoGetter, m.podRegistry, time.Second*30)
	go podCache.Loop()
	random := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
	s := scheduler.NewRandomFitScheduler(m.podRegistry, random)
	m.storage = map[string]apiserver.RESTStorage{
		"pods": registry.MakePodRegistryStorage(m.podRegistry, podInfoGetter, s, m.minionRegistry, cloud, podCache),
		"replicationControllers": registry.NewControllerRegistryStorage(m.controllerRegistry, m.podRegistry),
		"services":               registry.MakeServiceRegistryStorage(m.serviceRegistry, cloud, m.minionRegistry),
		"minions":                registry.MakeMinionRegistryStorage(m.minionRegistry),
	}
}

// Run begins serving the Kubernetes API. It never returns.
func (m *kubernatesMaster) run(myAddress, apiPrefix string) error {
	endpoints := registry.MakeEndpointController(m.serviceRegistry, m.client)
	go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10)

	s := &http.Server{
		Addr:           myAddress,
		Handler:        apiserver.New(m.storage, apiPrefix),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	return s.ListenAndServe()
}
<commit_msg>fixed runner.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// apiserver is the main api server and master for the cluster.
// it is responsible for serving the cluster management API.
package main

import (
	"flag"
	"net"
	"net/http"
	"strconv"
	"time"

	"code.google.com/p/goprotobuf/proto"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/registry"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	log "github.com/golang/glog"
	"github.com/mesosphere/kubernetes-mesos/framework"
	"github.com/mesosphere/mesos-go/mesos"
)

var (
	port                        = flag.Uint("port", 8080, "The port to listen on.  Default 8080.")
	address                     = flag.String("address", "127.0.0.1", "The address on the local server to listen to. Default 127.0.0.1")
	apiPrefix                   = flag.String("api_prefix", "/api/v1beta1", "The prefix for API requests on the server. Default '/api/v1beta1'")
	cloudProvider               = flag.String("cloud_provider", "", "The provider for cloud services.  Empty string for no provider.")
	minionRegexp                = flag.String("minion_regexp", "", "If non empty, and -cloud_provider is specified, a regular expression for matching minion VMs")
	minionPort                  = flag.Uint("minion_port", 10250, "The port at which kubelet will be listening on the minions.")
	mesosMaster                 = flag.String("mesos_master", "localhost:5050", "Location of leading Mesos master")
	executorURI                 = flag.String("executor_uri", "", "URI of dir that contains the executor executable")
	etcdServerList, machineList util.StringList
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "Servers for the etcd (http://ip:port), comma separated")
	flag.Var(&machineList, "machines", "List of machines to schedule onto, comma separated.")
}

type kubernatesMaster struct {
	podRegistry        registry.PodRegistry
	controllerRegistry registry.ControllerRegistry
	serviceRegistry    registry.ServiceRegistry
	minionRegistry     registry.MinionRegistry
	storage            map[string]apiserver.RESTStorage
	client             *client.Client
}

// Copied from cmd/apiserver.go
func main() {
	flag.Parse()
	util.InitLogs()
	defer util.FlushLogs()

	if len(machineList) == 0 {
		log.Fatal("No machines specified!")
	}

	var cloud cloudprovider.Interface
	switch *cloudProvider {
	case "gce":
		var err error
		cloud, err = cloudprovider.NewGCECloud()
		if err != nil {
			log.Fatal("Couldn't connect to GCE cloud: %#v", err)
		}
	default:
		if len(*cloudProvider) > 0 {
			log.Infof("Unknown cloud provider: %s", *cloudProvider)
		} else {
			log.Info("No cloud provider specified.")
		}
	}

	// TODO(yifan): Let mesos handle pod info getter.
	podInfoGetter := &client.HTTPPodInfoGetter{
		Client: http.DefaultClient,
		Port:   *minionPort,
	}

	client := client.New("http://"+net.JoinHostPort(*address, strconv.Itoa(int(*port))), nil)

	// Create mesos scheduler driver.
	executor := &mesos.ExecutorInfo{
		ExecutorId: &mesos.ExecutorID{Value: proto.String("KubeleteExecutorID")},
		Command: &mesos.CommandInfo{
			Value: proto.String("./executor_runner"),
			Uris: []*mesos.CommandInfo_URI{
				&mesos.CommandInfo_URI{Value: executorURI},
			},
		},
		Name:   proto.String("Executor for kubelete"),
		Source: proto.String("for test"),
	}

	mesosPodScheduler := framework.New(executor, framework.FCFSScheduleFunc)
	driver := &mesos.MesosSchedulerDriver{
		Master: *mesosMaster,
		Framework: mesos.FrameworkInfo{
			Name: proto.String("KubernetesFramework"),
			User: proto.String(""),
		},
		Scheduler: mesosPodScheduler,
	}
	mesosPodScheduler.Driver = driver

	driver.Init()
	defer driver.Destroy()
	go driver.Start()

	m := newKubernatesMaster(mesosPodScheduler, &master.Config{
		Client:        client,
		Cloud:         cloud,
		Minions:       machineList,
		PodInfoGetter: podInfoGetter,
	})
	log.Fatal(m.run(net.JoinHostPort(*address, strconv.Itoa(int(*port))), *apiPrefix))
}

func newKubernatesMaster(framework *framework.KubernetesFramework, c *master.Config) *kubernatesMaster {
	m := &kubernatesMaster{
		podRegistry:        framework,
		controllerRegistry: registry.MakeMemoryRegistry(),
		serviceRegistry:    registry.MakeMemoryRegistry(),
		minionRegistry:     registry.MakeMinionRegistry(c.Minions),
		client:             c.Client,
	}
	m.init(framework, c.Cloud, c.PodInfoGetter)
	return m
}

func (m *kubernatesMaster) init(scheduler scheduler.Scheduler, cloud cloudprovider.Interface, podInfoGetter client.PodInfoGetter) {
	podCache := master.NewPodCache(podInfoGetter, m.podRegistry, time.Second*30)
	go podCache.Loop()
	m.storage = map[string]apiserver.RESTStorage{
		"pods": registry.MakePodRegistryStorage(m.podRegistry, podInfoGetter, scheduler, m.minionRegistry, cloud, podCache),
		"replicationControllers": registry.NewControllerRegistryStorage(m.controllerRegistry, m.podRegistry),
		"services":               registry.MakeServiceRegistryStorage(m.serviceRegistry, cloud, m.minionRegistry),
		"minions":                registry.MakeMinionRegistryStorage(m.minionRegistry),
	}
}

// Run begins serving the Kubernetes API. It never returns.
func (m *kubernatesMaster) run(myAddress, apiPrefix string) error {
	endpoints := registry.MakeEndpointController(m.serviceRegistry, m.client)
	go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10)

	s := &http.Server{
		Addr:           myAddress,
		Handler:        apiserver.New(m.storage, apiPrefix),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	return s.ListenAndServe()
}
<|endoftext|>


<commit_before>package dockerignore

import (
	"bufio"
	"fmt"
	"io"
	"path/filepath"
	"strings"
)

// ReadAll reads a .dockerignore file and returns the list of file patterns
// to ignore. Note this will trim whitespace from each line as well
// as use GO's "clean" func to get the shortest/cleanest path for each.
func ReadAll(reader io.ReadCloser) ([]string, error) {
	if reader == nil {
		return nil, nil
	}
	defer reader.Close()
	scanner := bufio.NewScanner(reader)
	var excludes []string

	for scanner.Scan() {
		pattern := strings.TrimSpace(scanner.Text())
		if pattern == "" {
			continue
		}
		pattern = filepath.Clean(pattern)
		pattern = filepath.ToSlash(pattern)
		excludes = append(excludes, pattern)
	}
	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("Error reading .dockerignore: %v", err)
	}
	return excludes, nil
}
<commit_msg>Add support for comment in .dockerignore<commit_after>package dockerignore

import (
	"bufio"
	"fmt"
	"io"
	"path/filepath"
	"strings"
)

// ReadAll reads a .dockerignore file and returns the list of file patterns
// to ignore. Note this will trim whitespace from each line as well
// as use GO's "clean" func to get the shortest/cleanest path for each.
func ReadAll(reader io.ReadCloser) ([]string, error) {
	if reader == nil {
		return nil, nil
	}
	defer reader.Close()
	scanner := bufio.NewScanner(reader)
	var excludes []string

	for scanner.Scan() {
		// Lines starting with # (comments) are ignored before processing
		pattern := scanner.Text()
		if strings.HasPrefix(pattern, "#") {
			continue
		}
		pattern = strings.TrimSpace(pattern)
		if pattern == "" {
			continue
		}
		pattern = filepath.Clean(pattern)
		pattern = filepath.ToSlash(pattern)
		excludes = append(excludes, pattern)
	}
	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("Error reading .dockerignore: %v", err)
	}
	return excludes, nil
}
<|endoftext|>