<commit_before>package main

import (
	"errors"
	"fmt"
	"strconv"
	"encoding/json"
	"encoding/base64"
	"github.com/hyperledger/fabric/core/chaincode/shim"
	"github.com/hyperledger/fabric/core/util"
)

type Chaincode struct { }

type ChaincodeFunctions struct {
	stub shim.ChaincodeStubInterface
}

func main() {
	err := shim.Start(new(Chaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t Chaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	_ = stub.PutState("dealStatus", []byte("draft")) // Possible Values [draft, open, closed, allocated]
	_ = stub.PutState("orderbook", []byte("{}"))
	return nil, nil
}

func (t Chaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "addOrder" {
		investor := args[0]
		ioi, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AddOrder(investor, ioi)
	} else if function == "allocateOrder" {
		investor := args[0]
		alloc, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AllocateOrder(investor, alloc)
	} else if function == "updateDealStatus" {
		status := args[0]
		return fns.UpdateDealStatus(status)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation: " + function)
}

func (t Chaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "getRole" {
		return fns.GetRole()
	} else if function == "getOrder" {
		investor := args[0]
		return fns.GetOrder(investor)
	} else if function == "getOrderbook" {
		return fns.GetOrderbook()
	} else if function == "echo" {
		address := args[0]
		f := "echo"
		arg := args[1]
		return fns.Echo(address, f, arg)
	} else if function == "getDealStatus" {
		return fns.GetDealStatus()
	}
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query: " + function)
}

type Order struct {
	Investor	string 		`json:"investor"`
	Ioi 		float64 	`json:"ioi"`
	Alloc 		float64		`json:"alloc"`
}

// Public Functions

func (c ChaincodeFunctions) GetRole() ([]byte, error) {
    role, err := c.stub.ReadCertAttribute("company")
    if err != nil { return nil, errors.New("Couldn't get attribute 'role'. Error: " + err.Error()) }
    str := base64.StdEncoding.EncodeToString(role)
	return []byte(str), nil
}

func (c ChaincodeFunctions) GetDealStatus() ([]byte, error)  {
	dealStatus, _ := c.stub.GetState("dealStatus")
	return dealStatus, nil
}

func (c ChaincodeFunctions) UpdateDealStatus(dealStatus string) ([]byte, error)  {
	_ = c.stub.PutState("dealStatus", []byte(dealStatus))
	c.stub.SetEvent("Book Status Change", []byte("{\"status\":\"" + dealStatus + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) AddOrder(investor string, ioi float64) ([]byte, error)  {
	dealStatus, _ := c.GetDealStatus()
	if(string(dealStatus) != "open") {
		c.stub.SetEvent("Permission Denied", []byte("{\"reason\":\"book is not open\"}"))
		return nil, errors.New("Orders cannot be placed unless deal status is 'Open'")
	}
	order := Order{Investor: investor, Ioi: ioi, Alloc: 0.0}
	c.saveOrderToBlockChain(order)
	c.stub.SetEvent("Order Added", []byte("{\"investor\":\"" + investor + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) GetOrder(investor string) ([]byte, error) {
	orderJson := c.getOrderAsJsonFromBlockchain(investor)
	return []byte(orderJson), nil
}

func (c ChaincodeFunctions) GetOrderbook() ([]byte, error) {
	orderbookJson, _ := c.stub.GetState("orderbook")
	return []byte(orderbookJson), nil
}

func (c ChaincodeFunctions) AllocateOrder(investor string, alloc float64) ([]byte, error) {
	order := c.getOrderFromBlockChain(investor)
	order.Alloc = alloc
	c.saveOrderToBlockChain(order)
	c.stub.SetEvent("Order Allocated", []byte("{\"investor\":\"" + investor + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) Echo(address string, f string, arg string) ([]byte, error) {
	invokeArgs := util.ToChaincodeArgs(f, arg)
	return c.stub.QueryChaincode(address, invokeArgs)
}

// Private Functions

func (c ChaincodeFunctions) getOrderAsJsonFromBlockchain(investor string) string {
	order := c.getOrderFromBlockChain(investor)
	orderJson, _ := json.Marshal(order)
	return string(orderJson)
}

func (c ChaincodeFunctions) getOrderFromBlockChain(investor string) Order {
	orderbook := c.getOrderbookFromBlockChain()
	return orderbook[investor]
}

func (c ChaincodeFunctions) saveOrderToBlockChain(order Order) {
	orderbook := c.getOrderbookFromBlockChain()
	orderbook[order.Investor] = order
	c.saveOrderbookToBlockChain(orderbook)
}

func (c ChaincodeFunctions) getOrderbookFromBlockChain() map[string]Order {
	orderbookJson, _ := c.stub.GetState("orderbook")
	var orderbook map[string]Order
	_ = json.Unmarshal(orderbookJson, &orderbook)
	return orderbook
}

func (c ChaincodeFunctions) saveOrderbookToBlockChain(orderbook map[string]Order) {
	orderbookJson, _ := json.Marshal(orderbook)
	_ = c.stub.PutState("orderbook", []byte(orderbookJson))
}<commit_msg>batch check-in<commit_after>package main

import (
	"errors"
	"fmt"
	"strconv"
	"encoding/json"
	"encoding/base64"
	"github.com/hyperledger/fabric/core/chaincode/shim"
	"github.com/hyperledger/fabric/core/util"
)

type Chaincode struct { }

type ChaincodeFunctions struct {
	stub shim.ChaincodeStubInterface
}

func main() {
	err := shim.Start(new(Chaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t Chaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	_ = stub.PutState("dealStatus", []byte("draft")) // Possible Values [draft, open, closed, allocated]
	_ = stub.PutState("orderbook", []byte("{}"))
	return nil, nil
}

func (t Chaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "addOrder" {
		investor := args[0]
		ioi, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AddOrder(investor, ioi)
	} else if function == "allocateOrder" {
		investor := args[0]
		alloc, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AllocateOrder(investor, alloc)
	} else if function == "updateDealStatus" {
		status := args[0]
		return fns.UpdateDealStatus(status)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation: " + function)
}

func (t Chaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "getRole" {
		return fns.GetRole()
	} else if function == "getCompany" {
		return fns.GetCompany()
	} else if function == "getOrder" {
		investor := args[0]
		return fns.GetOrder(investor)
	} else if function == "getOrderbook" {
		return fns.GetOrderbook()
	} else if function == "echo" {
		address := args[0]
		f := "echo"
		arg := args[1]
		return fns.Echo(address, f, arg)
	} else if function == "getDealStatus" {
		return fns.GetDealStatus()
	}
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query: " + function)
}

type Order struct {
	Investor	string 		`json:"investor"`
	Ioi 		float64 	`json:"ioi"`
	Alloc 		float64		`json:"alloc"`
}

// Public Functions

func (c ChaincodeFunctions) GetRole() ([]byte, error) {
    role, err := c.stub.ReadCertAttribute("role")
    if err != nil { return nil, errors.New("Couldn't get attribute 'role'. Error: " + err.Error()) }
    str := base64.StdEncoding.EncodeToString(role)
	return []byte(str), nil
}

func (c ChaincodeFunctions) GetCompany() ([]byte, error) {
    role, err := c.stub.ReadCertAttribute("company")
    if err != nil { return nil, errors.New("Couldn't get attribute 'company'. Error: " + err.Error()) }
    str := base64.StdEncoding.EncodeToString(role)
	return []byte(str), nil
}

func (c ChaincodeFunctions) GetDealStatus() ([]byte, error)  {
	dealStatus, _ := c.stub.GetState("dealStatus")
	return dealStatus, nil
}

func (c ChaincodeFunctions) UpdateDealStatus(dealStatus string) ([]byte, error)  {
	_ = c.stub.PutState("dealStatus", []byte(dealStatus))
	c.stub.SetEvent("Book Status Change", []byte("{\"status\":\"" + dealStatus + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) AddOrder(investor string, ioi float64) ([]byte, error)  {
	dealStatus, _ := c.GetDealStatus()
	if(string(dealStatus) != "open") {
		c.stub.SetEvent("Permission Denied", []byte("{\"reason\":\"book is not open\"}"))
		return nil, errors.New("Orders cannot be placed unless deal status is 'Open'")
	}
	order := Order{Investor: investor, Ioi: ioi, Alloc: 0.0}
	c.saveOrderToBlockChain(order)
	c.stub.SetEvent("Order Added", []byte("{\"investor\":\"" + investor + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) GetOrder(investor string) ([]byte, error) {
	orderJson := c.getOrderAsJsonFromBlockchain(investor)
	return []byte(orderJson), nil
}

func (c ChaincodeFunctions) GetOrderbook() ([]byte, error) {
	orderbookJson, _ := c.stub.GetState("orderbook")
	return []byte(orderbookJson), nil
}

func (c ChaincodeFunctions) AllocateOrder(investor string, alloc float64) ([]byte, error) {
	order := c.getOrderFromBlockChain(investor)
	order.Alloc = alloc
	c.saveOrderToBlockChain(order)
	c.stub.SetEvent("Order Allocated", []byte("{\"investor\":\"" + investor + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) Echo(address string, f string, arg string) ([]byte, error) {
	invokeArgs := util.ToChaincodeArgs(f, arg)
	return c.stub.QueryChaincode(address, invokeArgs)
}

// Private Functions

func (c ChaincodeFunctions) getOrderAsJsonFromBlockchain(investor string) string {
	order := c.getOrderFromBlockChain(investor)
	orderJson, _ := json.Marshal(order)
	return string(orderJson)
}

func (c ChaincodeFunctions) getOrderFromBlockChain(investor string) Order {
	orderbook := c.getOrderbookFromBlockChain()
	return orderbook[investor]
}

func (c ChaincodeFunctions) saveOrderToBlockChain(order Order) {
	orderbook := c.getOrderbookFromBlockChain()
	orderbook[order.Investor] = order
	c.saveOrderbookToBlockChain(orderbook)
}

func (c ChaincodeFunctions) getOrderbookFromBlockChain() map[string]Order {
	orderbookJson, _ := c.stub.GetState("orderbook")
	var orderbook map[string]Order
	_ = json.Unmarshal(orderbookJson, &orderbook)
	return orderbook
}

func (c ChaincodeFunctions) saveOrderbookToBlockChain(orderbook map[string]Order) {
	orderbookJson, _ := json.Marshal(orderbook)
	_ = c.stub.PutState("orderbook", []byte(orderbookJson))
}<|endoftext|>


<commit_before>package adapters

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/mxmCherry/openrtb"
	"github.com/prebid/prebid-server/pbs"
	"golang.org/x/net/context/ctxhttp"
)

type LifestreetAdapter struct {
	http         *HTTPAdapter
	URI          string
	usersyncInfo *pbs.UsersyncInfo
}

/* Name - export adapter name */
func (a *LifestreetAdapter) Name() string {
	return "Lifestreet"
}

// used for cookies and such
func (a *LifestreetAdapter) FamilyName() string {
	return "lifestreet"
}

func (a *LifestreetAdapter) GetUsersyncInfo() *pbs.UsersyncInfo {
	return a.usersyncInfo
}

func (a *LifestreetAdapter) SkipNoCookies() bool {
	return false
}

// parameters for Lifestreet adapter.
type lifestreetParams struct {
	SlotTag string `json:"slot_tag"`
}

func (a *LifestreetAdapter) callOne(ctx context.Context, req *pbs.PBSRequest, reqJSON bytes.Buffer) (result callOneResult, err error) {
	httpReq, err := http.NewRequest("POST", a.URI, &reqJSON)
	httpReq.Header.Add("Content-Type", "application/json;charset=utf-8")
	httpReq.Header.Add("Accept", "application/json")

	lsmResp, e := ctxhttp.Do(ctx, a.http.Client, httpReq)
	if e != nil {
		err = e
		return
	}

	defer lsmResp.Body.Close()
	body, _ := ioutil.ReadAll(lsmResp.Body)
	result.responseBody = string(body)

	result.statusCode = lsmResp.StatusCode

	if lsmResp.StatusCode == 204 {
		return
	}

	if lsmResp.StatusCode != 200 {
		err = fmt.Errorf("HTTP status %d; body: %s", lsmResp.StatusCode, result.responseBody)
		return
	}

	var bidResp openrtb.BidResponse
	err = json.Unmarshal(body, &bidResp)
	if err != nil {
		return
	}
	if len(bidResp.SeatBid) == 0 || len(bidResp.SeatBid[0].Bid) == 0 {
		return
	}
	bid := bidResp.SeatBid[0].Bid[0]

	result.bid = &pbs.PBSBid{
		AdUnitCode:  bid.ImpID,
		Price:       bid.Price,
		Adm:         bid.AdM,
		Creative_id: bid.CrID,
		Width:       bid.W,
		Height:      bid.H,
		DealId:      bid.DealID,
		NURL:        bid.NURL,
	}
	return
}

func (a *LifestreetAdapter) MakeOpenRtbBidRequest(req *pbs.PBSRequest, bidder *pbs.PBSBidder, slotTag string, mtype pbs.MediaType, unitInd int) (openrtb.BidRequest, error) {
	lsReq, err := makeOpenRTBGeneric(req, bidder, a.FamilyName(), []pbs.MediaType{mtype}, true)

	if err != nil {
		return openrtb.BidRequest{}, err
	}

	if lsReq.Imp != nil && len(lsReq.Imp) > 0 {
		lsReq.Imp = lsReq.Imp[unitInd : unitInd+1]

		lsReq.Imp[0].Banner.Format = nil
		lsReq.Imp[0].TagID = slotTag

		return lsReq, nil
	} else {
		return lsReq, errors.New("No supported impressions")
	}
}

func (a *LifestreetAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) {
	requests := make([]bytes.Buffer, len(bidder.AdUnits)*2)
	reqIndex := 0
	for i, unit := range bidder.AdUnits {
		var params lifestreetParams
		err := json.Unmarshal(unit.Params, &params)
		if err != nil {
			return nil, err
		}
		if params.SlotTag == "" {
			return nil, errors.New("Missing slot_tag param")
		}
		s := strings.Split(params.SlotTag, ".")
		if len(s) != 2 {
			return nil, fmt.Errorf("Invalid slot_tag param '%s'", params.SlotTag)
		}

		// BANNER
		lsReqB, err := a.MakeOpenRtbBidRequest(req, bidder, params.SlotTag, pbs.MEDIA_TYPE_BANNER, i)
		if err == nil {
			err = json.NewEncoder(&requests[reqIndex]).Encode(lsReqB)
			reqIndex = reqIndex + 1
			if err != nil {
				return nil, err
			}
		}

		// VIDEO
		lsReqV, err := a.MakeOpenRtbBidRequest(req, bidder, params.SlotTag, pbs.MEDIA_TYPE_VIDEO, i)
		if err == nil {
			err = json.NewEncoder(&requests[reqIndex]).Encode(lsReqV)
			reqIndex = reqIndex + 1
			if err != nil {
				return nil, err
			}
		}
	}

	ch := make(chan callOneResult)
	for i, _ := range bidder.AdUnits {
		go func(bidder *pbs.PBSBidder, reqJSON bytes.Buffer) {
			result, err := a.callOne(ctx, req, reqJSON)
			result.Error = err
			if result.bid != nil {
				result.bid.BidderCode = bidder.BidderCode
				result.bid.BidID = bidder.LookupBidID(result.bid.AdUnitCode)
				if result.bid.BidID == "" {
					result.Error = fmt.Errorf("Unknown ad unit code '%s'", result.bid.AdUnitCode)
					result.bid = nil
				}
			}
			ch <- result
		}(bidder, requests[i])
	}

	var err error

	bids := make(pbs.PBSBidSlice, 0)
	for i := 0; i < len(bidder.AdUnits); i++ {
		result := <-ch
		if result.bid != nil {
			bids = append(bids, result.bid)
		}
		if req.IsDebug {
			debug := &pbs.BidderDebug{
				RequestURI:   a.URI,
				RequestBody:  requests[i].String(),
				StatusCode:   result.statusCode,
				ResponseBody: result.responseBody,
			}
			bidder.Debug = append(bidder.Debug, debug)
		}
		if result.Error != nil {
			err = result.Error
		}
	}

	if len(bids) == 0 {
		return nil, err
	}
	return bids, nil
}

func NewLifestreetAdapter(config *HTTPAdapterConfig, externalURL string) *LifestreetAdapter {
	a := NewHTTPAdapter(config)

	redirect_uri := fmt.Sprintf("%s/setuid?bidder=lifestreet&uid=$$visitor_cookie$$", externalURL)
	usersyncURL := "//ads.lfstmedia.com/idsync/137062?synced=1&ttl=1s&rurl="

	info := &pbs.UsersyncInfo{
		URL:         fmt.Sprintf("%s%s", usersyncURL, url.QueryEscape(redirect_uri)),
		Type:        "redirect",
		SupportCORS: false,
	}

	return &LifestreetAdapter{
		http:         a,
		URI:          "https://prebid.s2s.lfstmedia.com/adrequest",
		usersyncInfo: info,
	}
}
<commit_msg>making small change to avoid segfault error (#156)<commit_after>package adapters

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/mxmCherry/openrtb"
	"github.com/prebid/prebid-server/pbs"
	"golang.org/x/net/context/ctxhttp"
)

type LifestreetAdapter struct {
	http         *HTTPAdapter
	URI          string
	usersyncInfo *pbs.UsersyncInfo
}

/* Name - export adapter name */
func (a *LifestreetAdapter) Name() string {
	return "Lifestreet"
}

// used for cookies and such
func (a *LifestreetAdapter) FamilyName() string {
	return "lifestreet"
}

func (a *LifestreetAdapter) GetUsersyncInfo() *pbs.UsersyncInfo {
	return a.usersyncInfo
}

func (a *LifestreetAdapter) SkipNoCookies() bool {
	return false
}

// parameters for Lifestreet adapter.
type lifestreetParams struct {
	SlotTag string `json:"slot_tag"`
}

func (a *LifestreetAdapter) callOne(ctx context.Context, req *pbs.PBSRequest, reqJSON bytes.Buffer) (result callOneResult, err error) {
	httpReq, err := http.NewRequest("POST", a.URI, &reqJSON)
	httpReq.Header.Add("Content-Type", "application/json;charset=utf-8")
	httpReq.Header.Add("Accept", "application/json")

	lsmResp, e := ctxhttp.Do(ctx, a.http.Client, httpReq)
	if e != nil {
		err = e
		return
	}

	defer lsmResp.Body.Close()
	body, _ := ioutil.ReadAll(lsmResp.Body)
	result.responseBody = string(body)

	result.statusCode = lsmResp.StatusCode

	if lsmResp.StatusCode == 204 {
		return
	}

	if lsmResp.StatusCode != 200 {
		err = fmt.Errorf("HTTP status %d; body: %s", lsmResp.StatusCode, result.responseBody)
		return
	}

	var bidResp openrtb.BidResponse
	err = json.Unmarshal(body, &bidResp)
	if err != nil {
		return
	}
	if len(bidResp.SeatBid) == 0 || len(bidResp.SeatBid[0].Bid) == 0 {
		return
	}
	bid := bidResp.SeatBid[0].Bid[0]

	result.bid = &pbs.PBSBid{
		AdUnitCode:  bid.ImpID,
		Price:       bid.Price,
		Adm:         bid.AdM,
		Creative_id: bid.CrID,
		Width:       bid.W,
		Height:      bid.H,
		DealId:      bid.DealID,
		NURL:        bid.NURL,
	}
	return
}

func (a *LifestreetAdapter) MakeOpenRtbBidRequest(req *pbs.PBSRequest, bidder *pbs.PBSBidder, slotTag string, mtype pbs.MediaType, unitInd int) (openrtb.BidRequest, error) {
	lsReq, err := makeOpenRTBGeneric(req, bidder, a.FamilyName(), []pbs.MediaType{mtype}, true)

	if err != nil {
		return openrtb.BidRequest{}, err
	}

	if lsReq.Imp != nil && len(lsReq.Imp) > 0 {
		lsReq.Imp = lsReq.Imp[unitInd : unitInd+1]

		if lsReq.Imp[0].Banner != nil {
			lsReq.Imp[0].Banner.Format = nil
		}
		lsReq.Imp[0].TagID = slotTag

		return lsReq, nil
	} else {
		return lsReq, errors.New("No supported impressions")
	}
}

func (a *LifestreetAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) {
	requests := make([]bytes.Buffer, len(bidder.AdUnits)*2)
	reqIndex := 0
	for i, unit := range bidder.AdUnits {
		var params lifestreetParams
		err := json.Unmarshal(unit.Params, &params)
		if err != nil {
			return nil, err
		}
		if params.SlotTag == "" {
			return nil, errors.New("Missing slot_tag param")
		}
		s := strings.Split(params.SlotTag, ".")
		if len(s) != 2 {
			return nil, fmt.Errorf("Invalid slot_tag param '%s'", params.SlotTag)
		}

		// BANNER
		lsReqB, err := a.MakeOpenRtbBidRequest(req, bidder, params.SlotTag, pbs.MEDIA_TYPE_BANNER, i)
		if err == nil {
			err = json.NewEncoder(&requests[reqIndex]).Encode(lsReqB)
			reqIndex = reqIndex + 1
			if err != nil {
				return nil, err
			}
		}

		// VIDEO
		lsReqV, err := a.MakeOpenRtbBidRequest(req, bidder, params.SlotTag, pbs.MEDIA_TYPE_VIDEO, i)
		if err == nil {
			err = json.NewEncoder(&requests[reqIndex]).Encode(lsReqV)
			reqIndex = reqIndex + 1
			if err != nil {
				return nil, err
			}
		}
	}

	ch := make(chan callOneResult)
	for i, _ := range bidder.AdUnits {
		go func(bidder *pbs.PBSBidder, reqJSON bytes.Buffer) {
			result, err := a.callOne(ctx, req, reqJSON)
			result.Error = err
			if result.bid != nil {
				result.bid.BidderCode = bidder.BidderCode
				result.bid.BidID = bidder.LookupBidID(result.bid.AdUnitCode)
				if result.bid.BidID == "" {
					result.Error = fmt.Errorf("Unknown ad unit code '%s'", result.bid.AdUnitCode)
					result.bid = nil
				}
			}
			ch <- result
		}(bidder, requests[i])
	}

	var err error

	bids := make(pbs.PBSBidSlice, 0)
	for i := 0; i < len(bidder.AdUnits); i++ {
		result := <-ch
		if result.bid != nil {
			bids = append(bids, result.bid)
		}
		if req.IsDebug {
			debug := &pbs.BidderDebug{
				RequestURI:   a.URI,
				RequestBody:  requests[i].String(),
				StatusCode:   result.statusCode,
				ResponseBody: result.responseBody,
			}
			bidder.Debug = append(bidder.Debug, debug)
		}
		if result.Error != nil {
			err = result.Error
		}
	}

	if len(bids) == 0 {
		return nil, err
	}
	return bids, nil
}

func NewLifestreetAdapter(config *HTTPAdapterConfig, externalURL string) *LifestreetAdapter {
	a := NewHTTPAdapter(config)

	redirect_uri := fmt.Sprintf("%s/setuid?bidder=lifestreet&uid=$$visitor_cookie$$", externalURL)
	usersyncURL := "//ads.lfstmedia.com/idsync/137062?synced=1&ttl=1s&rurl="

	info := &pbs.UsersyncInfo{
		URL:         fmt.Sprintf("%s%s", usersyncURL, url.QueryEscape(redirect_uri)),
		Type:        "redirect",
		SupportCORS: false,
	}

	return &LifestreetAdapter{
		http:         a,
		URI:          "https://prebid.s2s.lfstmedia.com/adrequest",
		usersyncInfo: info,
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	toml "github.com/BurntSushi/toml"
	"github.com/direnv/direnv/v2/xdg"
)

// Config represents the direnv configuration and state.
type Config struct {
	Env             Env
	WorkDir         string // Current directory
	ConfDir         string
	CacheDir        string
	DataDir         string
	SelfPath        string
	BashPath        string
	RCFile          string
	TomlPath        string
	DisableStdin    bool
	StrictEnv       bool
	SkipDotenv      bool
	WarnTimeout     time.Duration
	WhitelistPrefix []string
	WhitelistExact  map[string]bool
}

type tomlDuration struct {
	time.Duration
}

func (d *tomlDuration) UnmarshalText(text []byte) error {
	var err error
	d.Duration, err = time.ParseDuration(string(text))
	return err
}

type tomlConfig struct {
	*tomlGlobal               // For backward-compatibility
	Global      *tomlGlobal   `toml:"global"`
	Whitelist   tomlWhitelist `toml:"whitelist"`
}

type tomlGlobal struct {
	BashPath     string       `toml:"bash_path"`
	DisableStdin bool         `toml:"disable_stdin"`
	StrictEnv    bool         `toml:"strict_env"`
	SkipDotenv   bool         `toml:"skip_dotenv"`
	WarnTimeout  tomlDuration `toml:"warn_timeout"`
}

type tomlWhitelist struct {
	Prefix []string
	Exact  []string
}

// LoadConfig opens up the direnv configuration from the Env.
func LoadConfig(env Env) (config *Config, err error) {
	config = &Config{
		Env: env,
	}

	config.ConfDir = env[DIRENV_CONFIG]
	if config.ConfDir == "" {
		config.ConfDir = xdg.ConfigDir(env, "direnv")
	}
	if config.ConfDir == "" {
		err = fmt.Errorf("couldn't find a configuration directory for direnv")
		return
	}

	var exePath string
	if exePath, err = os.Executable(); err != nil {
		err = fmt.Errorf("LoadConfig() os.Executable() failed: %w", err)
		return
	}
	// Fix for mingsys
	exePath = strings.Replace(exePath, "\\", "/", -1)
	config.SelfPath = exePath

	if config.WorkDir, err = os.Getwd(); err != nil {
		err = fmt.Errorf("LoadConfig() Getwd failed: %w", err)
		return
	}

	config.RCFile = env[DIRENV_FILE]

	config.WhitelistPrefix = make([]string, 0)
	config.WhitelistExact = make(map[string]bool)

	// Load the TOML config
	config.TomlPath = filepath.Join(config.ConfDir, "direnv.toml")
	if _, statErr := os.Stat(config.TomlPath); statErr != nil {
		config.TomlPath = filepath.Join(config.ConfDir, "config.toml")
		if _, statErr := os.Stat(config.TomlPath); statErr != nil {
			config.TomlPath = ""
		}
	}

	if config.TomlPath != "" {
		// Declare global once and then share it between the top-level and Global
		// keys. The goal here is to let the decoder fill global regardless of if
		// the values are in the [global] section or not. The reason we do that is
		var global tomlGlobal
		tomlConf := tomlConfig{
			tomlGlobal: &global,
			Global:     &global,
		}
		if _, err = toml.DecodeFile(config.TomlPath, &tomlConf); err != nil {
			err = fmt.Errorf("LoadConfig() failed to parse %s: %w", config.TomlPath, err)
			return
		}

		config.WhitelistPrefix = append(config.WhitelistPrefix, tomlConf.Whitelist.Prefix...)

		for _, path := range tomlConf.Whitelist.Exact {
			if !(strings.HasSuffix(path, "/.envrc") || strings.HasSuffix(path, "/.env")) {
				path = filepath.Join(path, ".envrc")
			}

			config.WhitelistExact[path] = true
		}

		config.BashPath = tomlConf.BashPath
		config.DisableStdin = tomlConf.DisableStdin
		config.StrictEnv = tomlConf.StrictEnv
		config.SkipDotenv = tomlConf.SkipDotenv
		config.WarnTimeout = tomlConf.WarnTimeout.Duration
	}

	if config.WarnTimeout == 0 {
		timeout, err := time.ParseDuration(env.Fetch("DIRENV_WARN_TIMEOUT", "5s"))
		if err != nil {
			logError("invalid DIRENV_WARN_TIMEOUT: " + err.Error())
			timeout = 5 * time.Second
		}
		config.WarnTimeout = timeout
	}

	if config.BashPath == "" {
		if env[DIRENV_BASH] != "" {
			config.BashPath = env[DIRENV_BASH]
		} else if bashPath != "" {
			config.BashPath = bashPath
		} else if config.BashPath, err = exec.LookPath("bash"); err != nil {
			err = fmt.Errorf("can't find bash: %w", err)
			return
		}
	}

	if config.CacheDir == "" {
		config.CacheDir = xdg.CacheDir(env, "direnv")
	}
	if config.CacheDir == "" {
		err = fmt.Errorf("couldn't find a cache directory for direnv")
		return
	}

	if config.DataDir == "" {
		config.DataDir = xdg.DataDir(env, "direnv")
	}
	if config.DataDir == "" {
		err = fmt.Errorf("couldn't find a data directory for direnv")
		return
	}

	return
}

// AllowDir is the folder where all the "allow" files are stored.
func (config *Config) AllowDir() string {
	return filepath.Join(config.DataDir, "allow")
}

// LoadedRC returns a RC file if any has been loaded
func (config *Config) LoadedRC() *RC {
	if config.RCFile == "" {
		logDebug("RCFile is blank - loadedRC is nil")
		return nil
	}
	rcPath := config.RCFile

	timesString := config.Env[DIRENV_WATCHES]

	return RCFromEnv(rcPath, timesString, config)
}

// EnvFromRC loads an RC from a specified path and returns the new environment
func (config *Config) EnvFromRC(path string, previousEnv Env) (Env, error) {
	rc, err := RCFromPath(path, config)
	if err != nil {
		return nil, err
	}
	return rc.Load(previousEnv)
}

// FindRC looks for a RC file in the config environment
func (config *Config) FindRC() (*RC, error) {
	return FindRC(config.WorkDir, config)
}

// Revert undoes the recorded changes (if any) to the supplied environment,
// returning a new environment
func (config *Config) Revert(env Env) (Env, error) {
	if config.Env[DIRENV_DIFF] == "" {
		return env.Copy(), nil
	}
	diff, err := LoadEnvDiff(config.Env[DIRENV_DIFF])
	if err == nil {
		return diff.Reverse().Patch(env), nil
	}
	return nil, err
}
<commit_msg>Re-add accidentally deleted comment line (#881)<commit_after>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	toml "github.com/BurntSushi/toml"
	"github.com/direnv/direnv/v2/xdg"
)

// Config represents the direnv configuration and state.
type Config struct {
	Env             Env
	WorkDir         string // Current directory
	ConfDir         string
	CacheDir        string
	DataDir         string
	SelfPath        string
	BashPath        string
	RCFile          string
	TomlPath        string
	DisableStdin    bool
	StrictEnv       bool
	SkipDotenv      bool
	WarnTimeout     time.Duration
	WhitelistPrefix []string
	WhitelistExact  map[string]bool
}

type tomlDuration struct {
	time.Duration
}

func (d *tomlDuration) UnmarshalText(text []byte) error {
	var err error
	d.Duration, err = time.ParseDuration(string(text))
	return err
}

type tomlConfig struct {
	*tomlGlobal               // For backward-compatibility
	Global      *tomlGlobal   `toml:"global"`
	Whitelist   tomlWhitelist `toml:"whitelist"`
}

type tomlGlobal struct {
	BashPath     string       `toml:"bash_path"`
	DisableStdin bool         `toml:"disable_stdin"`
	StrictEnv    bool         `toml:"strict_env"`
	SkipDotenv   bool         `toml:"skip_dotenv"`
	WarnTimeout  tomlDuration `toml:"warn_timeout"`
}

type tomlWhitelist struct {
	Prefix []string
	Exact  []string
}

// LoadConfig opens up the direnv configuration from the Env.
func LoadConfig(env Env) (config *Config, err error) {
	config = &Config{
		Env: env,
	}

	config.ConfDir = env[DIRENV_CONFIG]
	if config.ConfDir == "" {
		config.ConfDir = xdg.ConfigDir(env, "direnv")
	}
	if config.ConfDir == "" {
		err = fmt.Errorf("couldn't find a configuration directory for direnv")
		return
	}

	var exePath string
	if exePath, err = os.Executable(); err != nil {
		err = fmt.Errorf("LoadConfig() os.Executable() failed: %w", err)
		return
	}
	// Fix for mingsys
	exePath = strings.Replace(exePath, "\\", "/", -1)
	config.SelfPath = exePath

	if config.WorkDir, err = os.Getwd(); err != nil {
		err = fmt.Errorf("LoadConfig() Getwd failed: %w", err)
		return
	}

	config.RCFile = env[DIRENV_FILE]

	config.WhitelistPrefix = make([]string, 0)
	config.WhitelistExact = make(map[string]bool)

	// Load the TOML config
	config.TomlPath = filepath.Join(config.ConfDir, "direnv.toml")
	if _, statErr := os.Stat(config.TomlPath); statErr != nil {
		config.TomlPath = filepath.Join(config.ConfDir, "config.toml")
		if _, statErr := os.Stat(config.TomlPath); statErr != nil {
			config.TomlPath = ""
		}
	}

	if config.TomlPath != "" {
		// Declare global once and then share it between the top-level and Global
		// keys. The goal here is to let the decoder fill global regardless of if
		// the values are in the [global] section or not. The reason we do that is
		// to keep backward-compatibility with the old top-level notation.
		var global tomlGlobal
		tomlConf := tomlConfig{
			tomlGlobal: &global,
			Global:     &global,
		}
		if _, err = toml.DecodeFile(config.TomlPath, &tomlConf); err != nil {
			err = fmt.Errorf("LoadConfig() failed to parse %s: %w", config.TomlPath, err)
			return
		}

		config.WhitelistPrefix = append(config.WhitelistPrefix, tomlConf.Whitelist.Prefix...)

		for _, path := range tomlConf.Whitelist.Exact {
			if !(strings.HasSuffix(path, "/.envrc") || strings.HasSuffix(path, "/.env")) {
				path = filepath.Join(path, ".envrc")
			}

			config.WhitelistExact[path] = true
		}

		config.BashPath = tomlConf.BashPath
		config.DisableStdin = tomlConf.DisableStdin
		config.StrictEnv = tomlConf.StrictEnv
		config.SkipDotenv = tomlConf.SkipDotenv
		config.WarnTimeout = tomlConf.WarnTimeout.Duration
	}

	if config.WarnTimeout == 0 {
		timeout, err := time.ParseDuration(env.Fetch("DIRENV_WARN_TIMEOUT", "5s"))
		if err != nil {
			logError("invalid DIRENV_WARN_TIMEOUT: " + err.Error())
			timeout = 5 * time.Second
		}
		config.WarnTimeout = timeout
	}

	if config.BashPath == "" {
		if env[DIRENV_BASH] != "" {
			config.BashPath = env[DIRENV_BASH]
		} else if bashPath != "" {
			config.BashPath = bashPath
		} else if config.BashPath, err = exec.LookPath("bash"); err != nil {
			err = fmt.Errorf("can't find bash: %w", err)
			return
		}
	}

	if config.CacheDir == "" {
		config.CacheDir = xdg.CacheDir(env, "direnv")
	}
	if config.CacheDir == "" {
		err = fmt.Errorf("couldn't find a cache directory for direnv")
		return
	}

	if config.DataDir == "" {
		config.DataDir = xdg.DataDir(env, "direnv")
	}
	if config.DataDir == "" {
		err = fmt.Errorf("couldn't find a data directory for direnv")
		return
	}

	return
}

// AllowDir is the folder where all the "allow" files are stored.
func (config *Config) AllowDir() string {
	return filepath.Join(config.DataDir, "allow")
}

// LoadedRC returns a RC file if any has been loaded
func (config *Config) LoadedRC() *RC {
	if config.RCFile == "" {
		logDebug("RCFile is blank - loadedRC is nil")
		return nil
	}
	rcPath := config.RCFile

	timesString := config.Env[DIRENV_WATCHES]

	return RCFromEnv(rcPath, timesString, config)
}

// EnvFromRC loads an RC from a specified path and returns the new environment
func (config *Config) EnvFromRC(path string, previousEnv Env) (Env, error) {
	rc, err := RCFromPath(path, config)
	if err != nil {
		return nil, err
	}
	return rc.Load(previousEnv)
}

// FindRC looks for a RC file in the config environment
func (config *Config) FindRC() (*RC, error) {
	return FindRC(config.WorkDir, config)
}

// Revert undoes the recorded changes (if any) to the supplied environment,
// returning a new environment
func (config *Config) Revert(env Env) (Env, error) {
	if config.Env[DIRENV_DIFF] == "" {
		return env.Copy(), nil
	}
	diff, err := LoadEnvDiff(config.Env[DIRENV_DIFF])
	if err == nil {
		return diff.Reverse().Patch(env), nil
	}
	return nil, err
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/pkg/errors"
)

// A Solution is returned by a solver run. It is mostly just a Lock, with some
// additional methods that report information about the solve run.
type Solution interface {
	Lock
	// The name of the ProjectAnalyzer used in generating this solution.
	AnalyzerName() string
	// The version of the ProjectAnalyzer used in generating this solution.
	AnalyzerVersion() int
	// The name of the Solver used in generating this solution.
	SolverName() string
	// The version of the Solver used in generating this solution.
	SolverVersion() int
	Attempts() int
}

type solution struct {
	// A list of the projects selected by the solver.
	p []LockedProject

	// The number of solutions that were attempted
	att int

	// The hash digest of the input opts
	hd []byte

	// The analyzer info
	analyzerInfo ProjectAnalyzerInfo

	// The solver used in producing this solution
	solv Solver
}

// WriteDepTree takes a basedir and a Lock, and exports all the projects
// listed in the lock to the appropriate target location within the basedir.
//
// If the goal is to populate a vendor directory, basedir should be the absolute
// path to that vendor directory, not its parent (a project root, typically).
//
// It requires a SourceManager to do the work, and takes a flag indicating
// whether or not to strip vendor directories contained in the exported
// dependencies.
func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log.Logger) error {
	if l == nil {
		return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
	}

	err := os.MkdirAll(basedir, 0777)
	if err != nil {
		return err
	}

	lps := l.Projects()

	type resp struct {
		i   int
		err error
	}
	respCh := make(chan resp, len(lps))
	writeCh := make(chan int, len(lps))
	cancel := make(chan struct{})

	// Queue work.
	for i := range lps {
		writeCh <- i
	}
	close(writeCh)
	// Launch writers.
	writers := runtime.GOMAXPROCS(-1)
	if len(lps) < writers {
		writers = len(lps)
	}
	var wg sync.WaitGroup
	wg.Add(writers)
	for i := 0; i < writers; i++ {
		go func() {
			defer wg.Done()

			for i := range writeCh {
				select {
				case <-cancel:
					return
				default:
				}

				p := lps[i]
				to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))

				if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
					respCh <- resp{i, errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot)}
					continue
				}

				if sv {
					select {
					case <-cancel:
						return
					default:
					}

					if err := filepath.Walk(to, stripVendor); err != nil {
						respCh <- resp{i, errors.Wrapf(err, "failed to strip vendor from %s", p.Ident().ProjectRoot)}
						continue
					}
				}

				respCh <- resp{i, nil}
			}
		}()
	}
	// Monitor writers
	go func() {
		wg.Wait()
		close(respCh)
	}()

	// Log results and collect errors
	var errs []error
	var cnt int
	for resp := range respCh {
		cnt++
		msg := "Wrote"
		if resp.err != nil {
			if len(errs) == 0 {
				close(cancel)
			}
			errs = append(errs, resp.err)
			msg = "Failed to write"
		}
		p := lps[resp.i]
		logger.Printf("(%d/%d) %s %s@%s\n", cnt, len(lps), msg, p.Ident(), p.Version())
	}

	if len(errs) > 0 {
		logger.Println("Failed to write dep tree. The following errors occurred:")
		for i, err := range errs {
			logger.Printf("(%d/%d) %s\n", i+1, len(errs), err)
		}

		removeAll(basedir)

		return errors.New("failed to write dep tree")
	}
	return nil
}

func (r solution) Projects() []LockedProject {
	return r.p
}

func (r solution) Attempts() int {
	return r.att
}

func (r solution) InputHash() []byte {
	return r.hd
}

func (r solution) AnalyzerName() string {
	return r.analyzerInfo.Name
}

func (r solution) AnalyzerVersion() int {
	return r.analyzerInfo.Version
}

func (r solution) SolverName() string {
	return r.solv.Name()
}

func (r solution) SolverVersion() int {
	return r.solv.Version()
}
<commit_msg>gps: change WriteDepTree concurrent writers from GOMAXPROCS to 16<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/pkg/errors"
)

// A Solution is returned by a solver run. It is mostly just a Lock, with some
// additional methods that report information about the solve run.
type Solution interface {
	Lock
	// The name of the ProjectAnalyzer used in generating this solution.
	AnalyzerName() string
	// The version of the ProjectAnalyzer used in generating this solution.
	AnalyzerVersion() int
	// The name of the Solver used in generating this solution.
	SolverName() string
	// The version of the Solver used in generating this solution.
	SolverVersion() int
	Attempts() int
}

type solution struct {
	// A list of the projects selected by the solver.
	p []LockedProject

	// The number of solutions that were attempted
	att int

	// The hash digest of the input opts
	hd []byte

	// The analyzer info
	analyzerInfo ProjectAnalyzerInfo

	// The solver used in producing this solution
	solv Solver
}

const concurrentWriters = 16

// WriteDepTree takes a basedir and a Lock, and exports all the projects
// listed in the lock to the appropriate target location within the basedir.
//
// If the goal is to populate a vendor directory, basedir should be the absolute
// path to that vendor directory, not its parent (a project root, typically).
//
// It requires a SourceManager to do the work, and takes a flag indicating
// whether or not to strip vendor directories contained in the exported
// dependencies.
func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log.Logger) error {
	if l == nil {
		return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
	}

	err := os.MkdirAll(basedir, 0777)
	if err != nil {
		return err
	}

	lps := l.Projects()

	type resp struct {
		i   int
		err error
	}
	respCh := make(chan resp, len(lps))
	writeCh := make(chan int, len(lps))
	cancel := make(chan struct{})

	// Queue work.
	for i := range lps {
		writeCh <- i
	}
	close(writeCh)
	// Launch writers.
	writers := concurrentWriters
	if len(lps) < writers {
		writers = len(lps)
	}
	var wg sync.WaitGroup
	wg.Add(writers)
	for i := 0; i < writers; i++ {
		go func() {
			defer wg.Done()

			for i := range writeCh {
				select {
				case <-cancel:
					return
				default:
				}

				p := lps[i]
				to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))

				if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
					respCh <- resp{i, errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot)}
					continue
				}

				if sv {
					select {
					case <-cancel:
						return
					default:
					}

					if err := filepath.Walk(to, stripVendor); err != nil {
						respCh <- resp{i, errors.Wrapf(err, "failed to strip vendor from %s", p.Ident().ProjectRoot)}
						continue
					}
				}

				respCh <- resp{i, nil}
			}
		}()
	}
	// Monitor writers
	go func() {
		wg.Wait()
		close(respCh)
	}()

	// Log results and collect errors
	var errs []error
	var cnt int
	for resp := range respCh {
		cnt++
		msg := "Wrote"
		if resp.err != nil {
			if len(errs) == 0 {
				close(cancel)
			}
			errs = append(errs, resp.err)
			msg = "Failed to write"
		}
		p := lps[resp.i]
		logger.Printf("(%d/%d) %s %s@%s\n", cnt, len(lps), msg, p.Ident(), p.Version())
	}

	if len(errs) > 0 {
		logger.Println("Failed to write dep tree. The following errors occurred:")
		for i, err := range errs {
			logger.Printf("(%d/%d) %s\n", i+1, len(errs), err)
		}

		removeAll(basedir)

		return errors.New("failed to write dep tree")
	}
	return nil
}

func (r solution) Projects() []LockedProject {
	return r.p
}

func (r solution) Attempts() int {
	return r.att
}

func (r solution) InputHash() []byte {
	return r.hd
}

func (r solution) AnalyzerName() string {
	return r.analyzerInfo.Name
}

func (r solution) AnalyzerVersion() int {
	return r.analyzerInfo.Version
}

func (r solution) SolverName() string {
	return r.solv.Name()
}

func (r solution) SolverVersion() int {
	return r.solv.Version()
}
<|endoftext|>


<commit_before>package k8s

import (
	"context"
	"os"
	"time"

	"github.com/golang/glog"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
)

// newLeaderElector creates a new LeaderElection and returns the Elector.
func newLeaderElector(client kubernetes.Interface, callbacks leaderelection.LeaderCallbacks, namespace string, lockName string) (*leaderelection.LeaderElector, error) {
	podName := os.Getenv("POD_NAME")

	broadcaster := record.NewBroadcaster()
	hostname, _ := os.Hostname()

	source := v1.EventSource{Component: "nginx-ingress-leader-elector", Host: hostname}
	recorder := broadcaster.NewRecorder(scheme.Scheme, source)

	lock := resourcelock.ConfigMapLock{
		ConfigMapMeta: metav1.ObjectMeta{Namespace: namespace, Name: lockName},
		Client:        client.CoreV1(),
		LockConfig: resourcelock.ResourceLockConfig{
			Identity:      podName,
			EventRecorder: recorder,
		},
	}

	ttl := 30 * time.Second
	return leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
		Lock:          &lock,
		LeaseDuration: ttl,
		RenewDeadline: ttl / 2,
		RetryPeriod:   ttl / 4,
		Callbacks:     callbacks,
	})
}

// createLeaderHandler builds the handler funcs for leader handling
func createLeaderHandler(lbc *LoadBalancerController) leaderelection.LeaderCallbacks {
	return leaderelection.LeaderCallbacks{
		OnStartedLeading: func(ctx context.Context) {
			glog.V(3).Info("started leading")
			if lbc.reportIngressStatus {
				ingresses := lbc.configuration.GetResourcesWithFilter(resourceFilter{Ingresses: true})

				glog.V(3).Infof("Updating status for %v Ingresses", len(ingresses))

				err := lbc.statusUpdater.UpdateExternalEndpointsForResources(ingresses)
				if err != nil {
					glog.V(3).Infof("error updating status when starting leading: %v", err)
				}
			}

			if lbc.areCustomResourcesEnabled {
				glog.V(3).Info("updating VirtualServer and VirtualServerRoutes status")

				err := lbc.updateVirtualServersStatusFromEvents()
				if err != nil {
					glog.V(3).Infof("error updating VirtualServers status when starting leading: %v", err)
				}

				err = lbc.updateVirtualServerRoutesStatusFromEvents()
				if err != nil {
					glog.V(3).Infof("error updating VirtualServerRoutes status when starting leading: %v", err)
				}

				err = lbc.updatePoliciesStatus()
				if err != nil {
					glog.V(3).Infof("error updating Policies status when starting leading: %v", err)
				}

				err = lbc.updateTransportServersStatusFromEvents()
				if err != nil {
					glog.V(3).Infof("error updating TransportServers status when starting leading: %v", err)
				}
			}
		},
		OnStoppedLeading: func() {
			glog.V(3).Info("stopped leading")
		},
	}
}
<commit_msg>Remove deprecated ConfigMapLock<commit_after>package k8s

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"

	v1 "k8s.io/api/core/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
)

// newLeaderElector creates a new LeaderElection and returns the Elector.
func newLeaderElector(client kubernetes.Interface, callbacks leaderelection.LeaderCallbacks, namespace string, lockName string) (*leaderelection.LeaderElector, error) {
	podName := os.Getenv("POD_NAME")

	broadcaster := record.NewBroadcaster()
	hostname, _ := os.Hostname()

	source := v1.EventSource{Component: "nginx-ingress-leader-elector", Host: hostname}
	recorder := broadcaster.NewRecorder(scheme.Scheme, source)

	lc := resourcelock.ResourceLockConfig{
		Identity:      podName,
		EventRecorder: recorder,
	}

	lock, err := resourcelock.New(resourcelock.ConfigMapsLeasesResourceLock,
		namespace,
		lockName,
		client.CoreV1(),
		client.CoordinationV1(),
		lc,
	)
	if err != nil {
		return nil, fmt.Errorf("error creating leader election lock: %w", err)
	}

	ttl := 30 * time.Second
	return leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
		Lock:          lock,
		LeaseDuration: ttl,
		RenewDeadline: ttl / 2,
		RetryPeriod:   ttl / 4,
		Callbacks:     callbacks,
	})
}

// createLeaderHandler builds the handler funcs for leader handling
func createLeaderHandler(lbc *LoadBalancerController) leaderelection.LeaderCallbacks {
	return leaderelection.LeaderCallbacks{
		OnStartedLeading: func(ctx context.Context) {
			glog.V(3).Info("started leading")
			if lbc.reportIngressStatus {
				ingresses := lbc.configuration.GetResourcesWithFilter(resourceFilter{Ingresses: true})

				glog.V(3).Infof("Updating status for %v Ingresses", len(ingresses))

				err := lbc.statusUpdater.UpdateExternalEndpointsForResources(ingresses)
				if err != nil {
					glog.V(3).Infof("error updating status when starting leading: %v", err)
				}
			}

			if lbc.areCustomResourcesEnabled {
				glog.V(3).Info("updating VirtualServer and VirtualServerRoutes status")

				err := lbc.updateVirtualServersStatusFromEvents()
				if err != nil {
					glog.V(3).Infof("error updating VirtualServers status when starting leading: %v", err)
				}

				err = lbc.updateVirtualServerRoutesStatusFromEvents()
				if err != nil {
					glog.V(3).Infof("error updating VirtualServerRoutes status when starting leading: %v", err)
				}

				err = lbc.updatePoliciesStatus()
				if err != nil {
					glog.V(3).Infof("error updating Policies status when starting leading: %v", err)
				}

				err = lbc.updateTransportServersStatusFromEvents()
				if err != nil {
					glog.V(3).Infof("error updating TransportServers status when starting leading: %v", err)
				}
			}
		},
		OnStoppedLeading: func() {
			glog.V(3).Info("stopped leading")
		},
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package descriptor

import (
	"bytes"
	"fmt"
	"net"
	"strings"

	"github.com/pkg/errors"

	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/utils/addrs"
	interfaces "github.com/ligato/vpp-agent/api/models/vpp/interfaces"
	l3 "github.com/ligato/vpp-agent/api/models/vpp/l3"
	"github.com/ligato/vpp-agent/pkg/models"
	kvs "github.com/ligato/vpp-agent/plugins/kvscheduler/api"
	ifdescriptor "github.com/ligato/vpp-agent/plugins/vpp/ifplugin/descriptor"
	"github.com/ligato/vpp-agent/plugins/vpp/l3plugin/descriptor/adapter"
	"github.com/ligato/vpp-agent/plugins/vpp/l3plugin/vppcalls"
)

const (
	// RouteDescriptorName is the name of the descriptor for static routes.
	RouteDescriptorName = "vpp-route"

	// dependency labels
	routeOutInterfaceDep = "interface-exists"
	vrfTableDep          = "vrf-table-exists"
	viaVrfTableDep       = "via-vrf-table-exists"

	// static route weight by default
	defaultWeight = 1
)

// RouteDescriptor teaches KVScheduler how to configure VPP routes.
type RouteDescriptor struct {
	log          logging.Logger
	routeHandler vppcalls.RouteVppAPI
}

// NewRouteDescriptor creates a new instance of the Route descriptor.
func NewRouteDescriptor(
	routeHandler vppcalls.RouteVppAPI, log logging.PluginLogger) *kvs.KVDescriptor {

	ctx := &RouteDescriptor{
		routeHandler: routeHandler,
		log:          log.NewLogger("static-route-descriptor"),
	}

	typedDescr := &adapter.RouteDescriptor{
		Name:                 RouteDescriptorName,
		NBKeyPrefix:          l3.ModelRoute.KeyPrefix(),
		ValueTypeName:        l3.ModelRoute.ProtoName(),
		KeySelector:          l3.ModelRoute.IsKeyValid,
		ValueComparator:      ctx.EquivalentRoutes,
		Validate:             ctx.Validate,
		Create:               ctx.Create,
		Delete:               ctx.Delete,
		Retrieve:             ctx.Retrieve,
		Dependencies:         ctx.Dependencies,
		RetrieveDependencies: []string{ifdescriptor.InterfaceDescriptorName},
	}
	return adapter.NewRouteDescriptor(typedDescr)
}

// EquivalentRoutes is case-insensitive comparison function for l3.Route.
func (d *RouteDescriptor) EquivalentRoutes(key string, oldRoute, newRoute *l3.Route) bool {
	if oldRoute.GetType() != newRoute.GetType() ||
		oldRoute.GetVrfId() != newRoute.GetVrfId() ||
		oldRoute.GetViaVrfId() != newRoute.GetViaVrfId() ||
		oldRoute.GetOutgoingInterface() != newRoute.GetOutgoingInterface() ||
		getWeight(oldRoute) != getWeight(newRoute) ||
		oldRoute.GetPreference() != newRoute.GetPreference() {
		return false
	}

	// compare dst networks
	if !equalNetworks(oldRoute.DstNetwork, newRoute.DstNetwork) {
		return false
	}

	// compare gw addresses (next hop)
	if !equalAddrs(getGwAddr(oldRoute), getGwAddr(newRoute)) {
		return false
	}

	return true
}

// Validate validates VPP static route configuration.
func (d *RouteDescriptor) Validate(key string, route *l3.Route) (err error) {
	// validation destination network
	_, ipNet, err := net.ParseCIDR(route.DstNetwork)
	if err != nil {
		return kvs.NewInvalidValueError(err, "dst_network")
	}

	// validate IP network implied by the IP and prefix length
	if strings.ToLower(ipNet.String()) != strings.ToLower(route.DstNetwork) {
		e := fmt.Errorf("DstNetwork (%s) must represent IP network (%s)", route.DstNetwork, ipNet.String())
		return kvs.NewInvalidValueError(e, "dst_network")
	}

	// TODO: validate mix of IP versions?

	return nil
}

// Create adds VPP static route.
func (d *RouteDescriptor) Create(key string, route *l3.Route) (metadata interface{}, err error) {
	err = d.routeHandler.VppAddRoute(route)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Delete removes VPP static route.
func (d *RouteDescriptor) Delete(key string, route *l3.Route, metadata interface{}) error {
	err := d.routeHandler.VppDelRoute(route)
	if err != nil {
		return err
	}

	return nil
}

// Retrieve returns all routes associated with interfaces managed by this agent.
func (d *RouteDescriptor) Retrieve(correlate []adapter.RouteKVWithMetadata) (
	retrieved []adapter.RouteKVWithMetadata, err error,
) {
	// Retrieve VPP route configuration
	routes, err := d.routeHandler.DumpRoutes()
	if err != nil {
		return nil, errors.Errorf("failed to dump VPP routes: %v", err)
	}

	for _, route := range routes {
		retrieved = append(retrieved, adapter.RouteKVWithMetadata{
			Key:    models.Key(route.Route),
			Value:  route.Route,
			Origin: kvs.UnknownOrigin,
		})
	}

	return retrieved, nil
}

// Dependencies lists dependencies for a VPP route.
func (d *RouteDescriptor) Dependencies(key string, route *l3.Route) []kvs.Dependency {
	var dependencies []kvs.Dependency
	// the outgoing interface must exist and be UP
	if route.OutgoingInterface != "" {
		dependencies = append(dependencies, kvs.Dependency{
			Label: routeOutInterfaceDep,
			Key:   interfaces.InterfaceKey(route.OutgoingInterface),
		})
	}

	// non-zero VRFs
	var protocol l3.VrfTable_Protocol
	_, isIPv6, _ := addrs.ParseIPWithPrefix(route.DstNetwork)
	if isIPv6 {
		protocol = l3.VrfTable_IPV6
	}
	if route.VrfId != 0 {
		dependencies = append(dependencies, kvs.Dependency{
			Label: vrfTableDep,
			Key:   l3.VrfTableKey(route.VrfId, protocol),
		})
	}
	if route.Type == l3.Route_INTER_VRF && route.ViaVrfId != 0 {
		dependencies = append(dependencies, kvs.Dependency{
			Label: viaVrfTableDep,
			Key:   l3.VrfTableKey(route.ViaVrfId, protocol),
		})
	}

	// TODO: perhaps check GW routability
	return dependencies
}

// equalAddrs compares two IP addresses for equality.
func equalAddrs(addr1, addr2 string) bool {
	a1 := net.ParseIP(addr1)
	a2 := net.ParseIP(addr2)
	if a1 == nil || a2 == nil {
		// if parsing fails, compare as strings
		return strings.ToLower(addr1) == strings.ToLower(addr2)
	}
	return a1.Equal(a2)
}

// getGwAddr returns the GW address chosen in the given route, handling the cases
// when it is left undefined.
func getGwAddr(route *l3.Route) string {
	if route.GetNextHopAddr() != "" {
		return route.GetNextHopAddr()
	}
	// return zero address
	_, dstIPNet, err := net.ParseCIDR(route.GetDstNetwork())
	if err != nil {
		return ""
	}
	if dstIPNet.IP.To4() == nil {
		return net.IPv6zero.String()
	}
	return net.IPv4zero.String()
}

// getWeight returns static route weight, handling the cases when it is left undefined.
func getWeight(route *l3.Route) uint32 {
	if route.Weight == 0 {
		return defaultWeight
	}
	return route.Weight
}

// equalNetworks compares two IP networks for equality.
func equalNetworks(net1, net2 string) bool {
	_, n1, err1 := net.ParseCIDR(net1)
	_, n2, err2 := net.ParseCIDR(net2)
	if err1 != nil || err2 != nil {
		// if parsing fails, compare as strings
		return strings.ToLower(net1) == strings.ToLower(net2)
	}
	return n1.IP.Equal(n2.IP) && bytes.Equal(n1.Mask, n2.Mask)
}
<commit_msg>Define VRF as retrieve dependency for routes (#1429)<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package descriptor

import (
	"bytes"
	"fmt"
	"net"
	"strings"

	"github.com/pkg/errors"

	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/utils/addrs"
	interfaces "github.com/ligato/vpp-agent/api/models/vpp/interfaces"
	l3 "github.com/ligato/vpp-agent/api/models/vpp/l3"
	"github.com/ligato/vpp-agent/pkg/models"
	kvs "github.com/ligato/vpp-agent/plugins/kvscheduler/api"
	ifdescriptor "github.com/ligato/vpp-agent/plugins/vpp/ifplugin/descriptor"
	"github.com/ligato/vpp-agent/plugins/vpp/l3plugin/descriptor/adapter"
	"github.com/ligato/vpp-agent/plugins/vpp/l3plugin/vppcalls"
)

const (
	// RouteDescriptorName is the name of the descriptor for static routes.
	RouteDescriptorName = "vpp-route"

	// dependency labels
	routeOutInterfaceDep = "interface-exists"
	vrfTableDep          = "vrf-table-exists"
	viaVrfTableDep       = "via-vrf-table-exists"

	// static route weight by default
	defaultWeight = 1
)

// RouteDescriptor teaches KVScheduler how to configure VPP routes.
type RouteDescriptor struct {
	log          logging.Logger
	routeHandler vppcalls.RouteVppAPI
}

// NewRouteDescriptor creates a new instance of the Route descriptor.
func NewRouteDescriptor(
	routeHandler vppcalls.RouteVppAPI, log logging.PluginLogger) *kvs.KVDescriptor {

	ctx := &RouteDescriptor{
		routeHandler: routeHandler,
		log:          log.NewLogger("static-route-descriptor"),
	}

	typedDescr := &adapter.RouteDescriptor{
		Name:                 RouteDescriptorName,
		NBKeyPrefix:          l3.ModelRoute.KeyPrefix(),
		ValueTypeName:        l3.ModelRoute.ProtoName(),
		KeySelector:          l3.ModelRoute.IsKeyValid,
		ValueComparator:      ctx.EquivalentRoutes,
		Validate:             ctx.Validate,
		Create:               ctx.Create,
		Delete:               ctx.Delete,
		Retrieve:             ctx.Retrieve,
		Dependencies:         ctx.Dependencies,
		RetrieveDependencies: []string{ifdescriptor.InterfaceDescriptorName, VrfTableDescriptorName},
	}
	return adapter.NewRouteDescriptor(typedDescr)
}

// EquivalentRoutes is case-insensitive comparison function for l3.Route.
func (d *RouteDescriptor) EquivalentRoutes(key string, oldRoute, newRoute *l3.Route) bool {
	if oldRoute.GetType() != newRoute.GetType() ||
		oldRoute.GetVrfId() != newRoute.GetVrfId() ||
		oldRoute.GetViaVrfId() != newRoute.GetViaVrfId() ||
		oldRoute.GetOutgoingInterface() != newRoute.GetOutgoingInterface() ||
		getWeight(oldRoute) != getWeight(newRoute) ||
		oldRoute.GetPreference() != newRoute.GetPreference() {
		return false
	}

	// compare dst networks
	if !equalNetworks(oldRoute.DstNetwork, newRoute.DstNetwork) {
		return false
	}

	// compare gw addresses (next hop)
	if !equalAddrs(getGwAddr(oldRoute), getGwAddr(newRoute)) {
		return false
	}

	return true
}

// Validate validates VPP static route configuration.
func (d *RouteDescriptor) Validate(key string, route *l3.Route) (err error) {
	// validation destination network
	_, ipNet, err := net.ParseCIDR(route.DstNetwork)
	if err != nil {
		return kvs.NewInvalidValueError(err, "dst_network")
	}

	// validate IP network implied by the IP and prefix length
	if strings.ToLower(ipNet.String()) != strings.ToLower(route.DstNetwork) {
		e := fmt.Errorf("DstNetwork (%s) must represent IP network (%s)", route.DstNetwork, ipNet.String())
		return kvs.NewInvalidValueError(e, "dst_network")
	}

	// TODO: validate mix of IP versions?

	return nil
}

// Create adds VPP static route.
func (d *RouteDescriptor) Create(key string, route *l3.Route) (metadata interface{}, err error) {
	err = d.routeHandler.VppAddRoute(route)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Delete removes VPP static route.
func (d *RouteDescriptor) Delete(key string, route *l3.Route, metadata interface{}) error {
	err := d.routeHandler.VppDelRoute(route)
	if err != nil {
		return err
	}

	return nil
}

// Retrieve returns all routes associated with interfaces managed by this agent.
func (d *RouteDescriptor) Retrieve(correlate []adapter.RouteKVWithMetadata) (
	retrieved []adapter.RouteKVWithMetadata, err error,
) {
	// Retrieve VPP route configuration
	routes, err := d.routeHandler.DumpRoutes()
	if err != nil {
		return nil, errors.Errorf("failed to dump VPP routes: %v", err)
	}

	for _, route := range routes {
		retrieved = append(retrieved, adapter.RouteKVWithMetadata{
			Key:    models.Key(route.Route),
			Value:  route.Route,
			Origin: kvs.UnknownOrigin,
		})
	}

	return retrieved, nil
}

// Dependencies lists dependencies for a VPP route.
func (d *RouteDescriptor) Dependencies(key string, route *l3.Route) []kvs.Dependency {
	var dependencies []kvs.Dependency
	// the outgoing interface must exist and be UP
	if route.OutgoingInterface != "" {
		dependencies = append(dependencies, kvs.Dependency{
			Label: routeOutInterfaceDep,
			Key:   interfaces.InterfaceKey(route.OutgoingInterface),
		})
	}

	// non-zero VRFs
	var protocol l3.VrfTable_Protocol
	_, isIPv6, _ := addrs.ParseIPWithPrefix(route.DstNetwork)
	if isIPv6 {
		protocol = l3.VrfTable_IPV6
	}
	if route.VrfId != 0 {
		dependencies = append(dependencies, kvs.Dependency{
			Label: vrfTableDep,
			Key:   l3.VrfTableKey(route.VrfId, protocol),
		})
	}
	if route.Type == l3.Route_INTER_VRF && route.ViaVrfId != 0 {
		dependencies = append(dependencies, kvs.Dependency{
			Label: viaVrfTableDep,
			Key:   l3.VrfTableKey(route.ViaVrfId, protocol),
		})
	}

	// TODO: perhaps check GW routability
	return dependencies
}

// equalAddrs compares two IP addresses for equality.
func equalAddrs(addr1, addr2 string) bool {
	a1 := net.ParseIP(addr1)
	a2 := net.ParseIP(addr2)
	if a1 == nil || a2 == nil {
		// if parsing fails, compare as strings
		return strings.ToLower(addr1) == strings.ToLower(addr2)
	}
	return a1.Equal(a2)
}

// getGwAddr returns the GW address chosen in the given route, handling the cases
// when it is left undefined.
func getGwAddr(route *l3.Route) string {
	if route.GetNextHopAddr() != "" {
		return route.GetNextHopAddr()
	}
	// return zero address
	_, dstIPNet, err := net.ParseCIDR(route.GetDstNetwork())
	if err != nil {
		return ""
	}
	if dstIPNet.IP.To4() == nil {
		return net.IPv6zero.String()
	}
	return net.IPv4zero.String()
}

// getWeight returns static route weight, handling the cases when it is left undefined.
func getWeight(route *l3.Route) uint32 {
	if route.Weight == 0 {
		return defaultWeight
	}
	return route.Weight
}

// equalNetworks compares two IP networks for equality.
func equalNetworks(net1, net2 string) bool {
	_, n1, err1 := net.ParseCIDR(net1)
	_, n2, err2 := net.ParseCIDR(net2)
	if err1 != nil || err2 != nil {
		// if parsing fails, compare as strings
		return strings.ToLower(net1) == strings.ToLower(net2)
	}
	return n1.IP.Equal(n2.IP) && bytes.Equal(n1.Mask, n2.Mask)
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"flag"
	"time"
)

// CiliumTestConfigType holds all of the configurable elements of the testsuite
type CiliumTestConfigType struct {
	Reprovision bool
	// HoldEnvironment leaves the test infrastructure in place on failure
	HoldEnvironment bool
	// PassCLIEnvironment passes through the environment invoking the gingko
	// tests. When false all subcommands are executed with an empty environment,
	// including PATH.
	PassCLIEnvironment  bool
	SSHConfig           string
	ShowCommands        bool
	TestScope           string
	SkipLogGathering    bool
	CiliumImage         string
	CiliumOperatorImage string
	ProvisionK8s        bool
	Timeout             time.Duration
	Kubeconfig          string
	Registry            string
	Benchmarks          bool
}

// CiliumTestConfig holds the global configuration of commandline flags
// in the ginkgo-based testing environment.
var CiliumTestConfig = CiliumTestConfigType{}

// ParseFlags parses commandline flags relevant to testing.
func (c *CiliumTestConfigType) ParseFlags() {
	flag.BoolVar(&c.Reprovision, "cilium.provision", true,
		"Provision Vagrant boxes and Cilium before running test")
	flag.BoolVar(&c.HoldEnvironment, "cilium.holdEnvironment", false,
		"On failure, hold the environment in its current state")
	flag.BoolVar(&c.PassCLIEnvironment, "cilium.passCLIEnvironment", false,
		"Pass the environment invoking ginkgo, including PATH, to subcommands")
	flag.BoolVar(&c.SkipLogGathering, "cilium.skipLogs", false,
		"skip gathering logs if a test fails")
	flag.StringVar(&c.SSHConfig, "cilium.SSHConfig", "",
		"Specify a custom command to fetch SSH configuration (eg: 'vagrant ssh-config')")
	flag.BoolVar(&c.ShowCommands, "cilium.showCommands", false,
		"Output which commands are ran to stdout")
	flag.StringVar(&c.TestScope, "cilium.testScope", "",
		"Specifies scope of test to be ran (k8s, Nightly, runtime)")
	flag.StringVar(&c.CiliumImage, "cilium.image", "",
		"Specifies which image of cilium to use during tests")
	flag.StringVar(&c.CiliumOperatorImage, "cilium.operator-image", "",
		"Specifies which image of cilium-operator to use during tests")
	flag.BoolVar(&c.ProvisionK8s, "cilium.provision-k8s", true,
		"Specifies whether Kubernetes should be deployed and installed via kubeadm or not")
	flag.DurationVar(&c.Timeout, "cilium.timeout", 24*time.Hour,
		"Specifies timeout for test run")
	flag.StringVar(&c.Kubeconfig, "cilium.kubeconfig", "",
		"Kubeconfig to be used for k8s tests")
	flag.StringVar(&c.Registry, "cilium.registry", "k8s1:5000", "docker registry hostname for Cilium image")
	flag.BoolVar(&c.Benchmarks, "cilium.benchmarks", false,
		"Specifies benchmark tests should be run which may increase test time")
}
<commit_msg>CI: Local CI runs don't set CILIUM_REGISTRY<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"flag"
	"time"
)

// CiliumTestConfigType holds all of the configurable elements of the testsuite
type CiliumTestConfigType struct {
	Reprovision bool
	// HoldEnvironment leaves the test infrastructure in place on failure
	HoldEnvironment bool
	// PassCLIEnvironment passes through the environment invoking the gingko
	// tests. When false all subcommands are executed with an empty environment,
	// including PATH.
	PassCLIEnvironment  bool
	SSHConfig           string
	ShowCommands        bool
	TestScope           string
	SkipLogGathering    bool
	CiliumImage         string
	CiliumOperatorImage string
	ProvisionK8s        bool
	Timeout             time.Duration
	Kubeconfig          string
	Registry            string
	Benchmarks          bool
}

// CiliumTestConfig holds the global configuration of commandline flags
// in the ginkgo-based testing environment.
var CiliumTestConfig = CiliumTestConfigType{}

// ParseFlags parses commandline flags relevant to testing.
func (c *CiliumTestConfigType) ParseFlags() {
	flag.BoolVar(&c.Reprovision, "cilium.provision", true,
		"Provision Vagrant boxes and Cilium before running test")
	flag.BoolVar(&c.HoldEnvironment, "cilium.holdEnvironment", false,
		"On failure, hold the environment in its current state")
	flag.BoolVar(&c.PassCLIEnvironment, "cilium.passCLIEnvironment", false,
		"Pass the environment invoking ginkgo, including PATH, to subcommands")
	flag.BoolVar(&c.SkipLogGathering, "cilium.skipLogs", false,
		"skip gathering logs if a test fails")
	flag.StringVar(&c.SSHConfig, "cilium.SSHConfig", "",
		"Specify a custom command to fetch SSH configuration (eg: 'vagrant ssh-config')")
	flag.BoolVar(&c.ShowCommands, "cilium.showCommands", false,
		"Output which commands are ran to stdout")
	flag.StringVar(&c.TestScope, "cilium.testScope", "",
		"Specifies scope of test to be ran (k8s, Nightly, runtime)")
	flag.StringVar(&c.CiliumImage, "cilium.image", "",
		"Specifies which image of cilium to use during tests")
	flag.StringVar(&c.CiliumOperatorImage, "cilium.operator-image", "",
		"Specifies which image of cilium-operator to use during tests")
	flag.BoolVar(&c.ProvisionK8s, "cilium.provision-k8s", true,
		"Specifies whether Kubernetes should be deployed and installed via kubeadm or not")
	flag.DurationVar(&c.Timeout, "cilium.timeout", 24*time.Hour,
		"Specifies timeout for test run")
	flag.StringVar(&c.Kubeconfig, "cilium.kubeconfig", "",
		"Kubeconfig to be used for k8s tests")
	flag.StringVar(&c.Registry, "cilium.registry", "", "docker registry hostname for Cilium image")
	flag.BoolVar(&c.Benchmarks, "cilium.benchmarks", false,
		"Specifies benchmark tests should be run which may increase test time")
}
<|endoftext|>


<commit_before>package dpn

import (
	"archive/tar"
	"crypto/sha256"
	"fmt"
	"github.com/APTrust/bagins"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

// BAGIT_TAGS contains a list of tags required in the bagit file.
var BAGIT_TAGS = []string {
	"BagIt-Version",
	"Tag-File-Character-Encoding",
}

// BAG_INFO_TAGS contains a list of tags required in the bag-info file.
var BAG_INFO_TAGS = []string {
	"Source-Organization",
	"Organization-Address",
	"Contact-Name",
	"Contact-Phone",
	"Contact-Email",
	"Bagging-Date",
	"Bag-Size",
	"Bag-Group-Identifier",
	"Bag-Count",
}

// DPN_INFO_TAGS contains a list tags required in the dpn-info file.
var DPN_INFO_TAGS = []string {
	"DPN-Object-ID",
	"Local-ID",
	"Ingest-Node-Name",
	"Ingest-Node-Address",
	"Ingest-Node-Contact-Name",
	"Ingest-Node-Contact-Email",
	"Version-Number",
	"First-Version-Object-ID",
	// "Brightening-Object-ID",
	"Rights-Object-ID",
	"Bag-Type",
}

// TAGS_FOR_FILE maps a tag file to the list of tags it should contain.
var TAGS_FOR_FILE = map[string][]string {
	"bagit.txt": BAGIT_TAGS,
	"bag-info.txt": BAG_INFO_TAGS,
	filepath.Join("dpn-tags", "dpn-info.txt"): DPN_INFO_TAGS,
}

// TagFiles() returns a list of tag files we should check while
// performing validation.
func TagFiles() ([]string) {
	tagFiles := make([]string, 0)
	for key := range TAGS_FOR_FILE {
		tagFiles = append(tagFiles, key)
	}
	return tagFiles
}

// ValidationResult stores information about whether a DPN
// bag is valid.
type ValidationResult struct {
	// TarFilePath is the path to the tarred bag we'll be validating.
	TarFilePath          string

	// UntarredPath is the path to the untarred version of this bag.
	UntarredPath         string

	// The NSQ message we're currently working on. This will be nil
	// outside of production. In production, we need to touch the
	// message periodically to keep it from timing out, especially
	// on very large bags.
	NsqMessage           *nsq.Message    `json:"-"`

	// TagManifestChecksum is the sha256 digest (calculated with a nonce)
	// that we need to send back to the originating node as a receipt
	// when we're fulfilling replication requests. Outside of fulfilling
	// replication requests, we don't need to even calculate this value.
	TagManifestChecksum  string

	// ErrorMessages contains a list of everything that's wrong with the
	// bag. If this list is empty, the bag is valid.
	ErrorMessages        []string

	// Warning messages about non-fatal issues we might want to look into.
	Warnings             []string
}

func NewValidationResult(pathToFile string, nsqMessage *nsq.Message) (*ValidationResult, error) {
	absPath, err := filepath.Abs(pathToFile)
	if err != nil {
		return nil, fmt.Errorf("Cannot determine absolute path from '%s': %v",
			pathToFile, err)
	}
	var validator *ValidationResult
	if _, err := os.Stat(absPath); os.IsNotExist(err) {
		return nil, fmt.Errorf("File does not exist at %s", absPath)
	}
	if strings.HasSuffix(absPath, ".tar") {
		validator = &ValidationResult{
			TarFilePath: absPath,
			NsqMessage: nsqMessage,
		}
	} else {
		validator = &ValidationResult{
			UntarredPath: absPath,
			NsqMessage: nsqMessage,
		}
	}
	return validator, nil
}

// IsValid() returns true if the bag is valid.
func (validator *ValidationResult) IsValid() (bool) {
	return len(validator.ErrorMessages) == 0
}

// AddError adds a message to the list of validation errors.
func (validator *ValidationResult) AddError(message string) () {
	validator.ErrorMessages = append(validator.ErrorMessages, message)
}

// AddWarning adds a message to the list of validation errors.
func (validator *ValidationResult) AddWarning(message string) () {
	validator.Warnings = append(validator.Warnings, message)
}

// PathToFileInBag returns the path the to file within a bag.
// If your bag is untarred to /mnt/data/my_bag and you call
// this function with param 'dpn-tags/dpn-info.txt', you'll
// get /mnt/data/my_bag/dpn-tags/dpn-info.txt
func (validator *ValidationResult) PathToFileInBag(relativePath string) (string) {
	return filepath.Join(validator.UntarredPath, relativePath)
}

// Run all validation checks on the bag.
func (validator *ValidationResult) ValidateBag()  {
	if validator.BagNameValid() == false {
		validator.AddError("Bag name is not valid. It should be a UUID.")
		return
	}
	if validator.NsqMessage != nil {
		validator.NsqMessage.Touch()
	}
	if validator.TarFilePath != "" && validator.UntarredPath == "" {
		if validator.untar() == false {
			return
		}
	}
	// Untar can take a long time on large bags.
	// Let NSQ know we're still working on it.
	if validator.NsqMessage != nil {
		validator.NsqMessage.Touch()
	}

	if validator.tagManifestPresent() == false {
		validator.AddError("Tag manifest file 'tagmanifest-sha256.txt' is missing.")
		return
	}

	// OK, the name is good, we untarred it and the tag manifest is valid.
	// Now do the heavy work... and there can be a lot to do on bags
	// over 100GB in size.
	bag, err := bagins.ReadBag(validator.UntarredPath, TagFiles(), "manifest-sha256.txt")
	if err != nil {
		validator.AddError(fmt.Sprintf("Error unpacking bag: %v", err))
		return
	}

	fileNames, err := bag.ListFiles()
	if err != nil {
		validator.AddError(fmt.Sprintf("Could not list bag files: %v ", err))
	}

	dataDirPrefix := "data/"
	if runtime.GOOS == "windows" {
		dataDirPrefix = "data\\"
	}

	hasBagit := false
	hasDPNInfo := false
	hasManifest := false
	hasDataFiles := false
	for _, fileName := range fileNames {
		if fileName == "bagit.txt" {
			hasBagit = true
		} else if fileName == filepath.Join("dpn-tags", "dpn-info.txt") {
			hasDPNInfo = true
		} else if fileName == "manifest-sha256.txt" {
			hasManifest = true
		} else if strings.HasPrefix(fileName, dataDirPrefix) {
			hasDataFiles = true
		}
	}
	if !hasBagit {
		validator.AddError("Bag is missing bagit.txt file.")
	}
	if !hasDPNInfo {
		validator.AddError("Bag is missing dpn-info.txt file.")
	}
	if !hasManifest {
		validator.AddError("Bag is missing manifest-md5.txt file.")
	}
	if !hasDataFiles {
		validator.AddError("Bag's data directory is missing or empty.")
	}

	// Make sure the tag files have the required tags.
	// They can be empty, but they have to be present.
	validator.checkRequiredTags(bag)

	// Run all the checksums on all files.
	checksumErrors := bag.Manifest.RunChecksums()

	// Running the checksums takes a long time on
	// large bags, so let NSQ know we're still working.
	if validator.NsqMessage != nil {
		validator.NsqMessage.Touch()
	}

	for _, err := range checksumErrors {
		validator.AddError(err.Error())
	}
}

// Extract all of the tags from tag files "bagit.txt", "bag-info.txt",
// and "dpn-info.txt" and make sure the required tags are present.
// There may be other tag files, but since they're optional, we don't
// have to check their content.
func (validator *ValidationResult) checkRequiredTags(bag *bagins.Bag) {
	for _, file := range TagFiles() {
		tagFile, err := bag.TagFile(file)
		if err != nil {
			validator.AddError(fmt.Sprintf("Error reading tags from file '%s': %v", file, err))
			return
		}
		tagFields := tagFile.Data.Fields()
		requiredTags := TAGS_FOR_FILE[file]
		tagsFound := make(map[string]bool)
		for _, tagField := range tagFields {
			tagsFound[tagField.Label()] = true
		}
		for _, tagName := range requiredTags {
			if _, ok := tagsFound[tagName]; !ok {
				validator.AddError(fmt.Sprintf("Required tag '%s' is missing from %s", tagName, file))
			}
		}
	}
}

func (validator *ValidationResult) BagNameValid() (bool) {
	bagPath := validator.TarFilePath
	if bagPath == "" {
		bagPath = validator.UntarredPath
	}
	basename := strings.Replace(filepath.Base(bagPath), ".tar", "", 1)
	return bagman.LooksLikeUUID(basename)
}

// If the tag manifest is present, bagins will validate it.
// We have to make sure it's here, and bagins will do the rest.
func (validator *ValidationResult) tagManifestPresent() (bool) {
	fullPath := filepath.Join(validator.UntarredPath, "tagmanifest-sha256.txt")
	return bagman.FileExists(fullPath)
}

func (validator *ValidationResult) CalculateTagManifestDigest(nonce string)  {
	filePath := validator.PathToFileInBag("tagmanifest-sha256.txt")
	src, err := os.Open(filePath)
	if err != nil {
		validator.AddError(fmt.Sprintf("Error reading tag manifest: %v", err))
	}
	defer src.Close()

	shaHash := sha256.New()
	_, err = io.Copy(shaHash, src)
	if err != nil {
		validator.AddError(fmt.Sprintf("Error calculating checksum on tag manifest: %v", err))
	}
	if nonce == "" {
		validator.TagManifestChecksum = fmt.Sprintf("%x", shaHash.Sum(nil))
	} else {
		validator.TagManifestChecksum = fmt.Sprintf("%x", shaHash.Sum([]byte(nonce)))
	}
}

func (validator *ValidationResult) untar() (bool) {
	absInputFile, err := filepath.Abs(validator.TarFilePath)
	if err != nil {
		validator.AddError(fmt.Sprintf("Before untarring, could not determine "+
			"absolute path to tar file: %v", err))
		return false
	}

	// Open the tar file for reading.
	file, err := os.Open(validator.TarFilePath)
	if file != nil {
		defer file.Close()
	}
	if err != nil {
		validator.AddError(fmt.Sprintf("Could not open file %s for untarring: %v",
			validator.TarFilePath, err))
		return false
	}

	// Untar the file and record the validators.
	tarReader := tar.NewReader(file)

	for {
		header, err := tarReader.Next()
		if err != nil && err.Error() == "EOF" {
			break // end of archive
		}
		if err != nil {
			validator.AddError(fmt.Sprintf(
				"Error reading tar file header: %v. " +
					"Either this is not a tar file, or the file is corrupt.", err))
			return false
		}

		// Set the untarred path, which will usually be the depositor's
		// bag identifier.
		if validator.UntarredPath == "" {
			nameParts := strings.Split(header.Name, string(os.PathSeparator))
			validator.UntarredPath = filepath.Join(filepath.Dir(absInputFile), nameParts[0])
		}

		outputPath := filepath.Join(filepath.Dir(absInputFile), header.Name)

		// Make sure the directory that we're about to write into exists.
		err = os.MkdirAll(filepath.Dir(outputPath), 0755)
		if err != nil {
			validator.AddError(fmt.Sprintf("Could not create destination file '%s' "+
				"while unpacking tar archive: %v", outputPath, err))
			return false
		}

		// Copy the file, if it's an actual file. Otherwise, ignore it and record
		// a warning. The bag library does not deal with items like symlinks.
		if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA {
			validator.saveFile(outputPath, tarReader)
		} else if header.Typeflag != tar.TypeDir {
			validator.AddWarning(
				fmt.Sprintf("Ignoring item %s of type %c because it's neither a file nor a directory",
					header.Name, header.Typeflag))
		}
	}
	return true
}

func (validator *ValidationResult) saveFile (destination string, tarReader *tar.Reader) {
	outputWriter, err := os.OpenFile(destination, os.O_CREATE|os.O_WRONLY, 0644)
	if outputWriter != nil {
		defer outputWriter.Close()
	}
	if err != nil {
		validator.AddError(fmt.Sprintf("Error opening file '%s': %v", destination, err))
		return
	}
	_, err = io.Copy(outputWriter, tarReader)
	if err != nil {
		validator.AddError(fmt.Sprintf("Error copying file to '%s': %v", destination, err))
	}
}

func (validator *ValidationResult) DeleteUntarredBag () {
	fmt.Println(validator.UntarredPath)
	//os.RemoveAll(validator.UntarredPath)
}
<commit_msg>PT #114015663: Clean up untarred DPN bag after validation<commit_after>package dpn

import (
	"archive/tar"
	"crypto/sha256"
	"fmt"
	"github.com/APTrust/bagins"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

// BAGIT_TAGS contains a list of tags required in the bagit file.
var BAGIT_TAGS = []string {
	"BagIt-Version",
	"Tag-File-Character-Encoding",
}

// BAG_INFO_TAGS contains a list of tags required in the bag-info file.
var BAG_INFO_TAGS = []string {
	"Source-Organization",
	"Organization-Address",
	"Contact-Name",
	"Contact-Phone",
	"Contact-Email",
	"Bagging-Date",
	"Bag-Size",
	"Bag-Group-Identifier",
	"Bag-Count",
}

// DPN_INFO_TAGS contains a list tags required in the dpn-info file.
var DPN_INFO_TAGS = []string {
	"DPN-Object-ID",
	"Local-ID",
	"Ingest-Node-Name",
	"Ingest-Node-Address",
	"Ingest-Node-Contact-Name",
	"Ingest-Node-Contact-Email",
	"Version-Number",
	"First-Version-Object-ID",
	// "Brightening-Object-ID",
	"Rights-Object-ID",
	"Bag-Type",
}

// TAGS_FOR_FILE maps a tag file to the list of tags it should contain.
var TAGS_FOR_FILE = map[string][]string {
	"bagit.txt": BAGIT_TAGS,
	"bag-info.txt": BAG_INFO_TAGS,
	filepath.Join("dpn-tags", "dpn-info.txt"): DPN_INFO_TAGS,
}

// TagFiles() returns a list of tag files we should check while
// performing validation.
func TagFiles() ([]string) {
	tagFiles := make([]string, 0)
	for key := range TAGS_FOR_FILE {
		tagFiles = append(tagFiles, key)
	}
	return tagFiles
}

// ValidationResult stores information about whether a DPN
// bag is valid.
type ValidationResult struct {
	// TarFilePath is the path to the tarred bag we'll be validating.
	TarFilePath          string

	// UntarredPath is the path to the untarred version of this bag.
	UntarredPath         string

	// The NSQ message we're currently working on. This will be nil
	// outside of production. In production, we need to touch the
	// message periodically to keep it from timing out, especially
	// on very large bags.
	NsqMessage           *nsq.Message    `json:"-"`

	// TagManifestChecksum is the sha256 digest (calculated with a nonce)
	// that we need to send back to the originating node as a receipt
	// when we're fulfilling replication requests. Outside of fulfilling
	// replication requests, we don't need to even calculate this value.
	TagManifestChecksum  string

	// ErrorMessages contains a list of everything that's wrong with the
	// bag. If this list is empty, the bag is valid.
	ErrorMessages        []string

	// Warning messages about non-fatal issues we might want to look into.
	Warnings             []string
}

func NewValidationResult(pathToFile string, nsqMessage *nsq.Message) (*ValidationResult, error) {
	absPath, err := filepath.Abs(pathToFile)
	if err != nil {
		return nil, fmt.Errorf("Cannot determine absolute path from '%s': %v",
			pathToFile, err)
	}
	var validator *ValidationResult
	if _, err := os.Stat(absPath); os.IsNotExist(err) {
		return nil, fmt.Errorf("File does not exist at %s", absPath)
	}
	if strings.HasSuffix(absPath, ".tar") {
		validator = &ValidationResult{
			TarFilePath: absPath,
			NsqMessage: nsqMessage,
		}
	} else {
		validator = &ValidationResult{
			UntarredPath: absPath,
			NsqMessage: nsqMessage,
		}
	}
	return validator, nil
}

// IsValid() returns true if the bag is valid.
func (validator *ValidationResult) IsValid() (bool) {
	return len(validator.ErrorMessages) == 0
}

// AddError adds a message to the list of validation errors.
func (validator *ValidationResult) AddError(message string) () {
	validator.ErrorMessages = append(validator.ErrorMessages, message)
}

// AddWarning adds a message to the list of validation errors.
func (validator *ValidationResult) AddWarning(message string) () {
	validator.Warnings = append(validator.Warnings, message)
}

// PathToFileInBag returns the path the to file within a bag.
// If your bag is untarred to /mnt/data/my_bag and you call
// this function with param 'dpn-tags/dpn-info.txt', you'll
// get /mnt/data/my_bag/dpn-tags/dpn-info.txt
func (validator *ValidationResult) PathToFileInBag(relativePath string) (string) {
	return filepath.Join(validator.UntarredPath, relativePath)
}

// Run all validation checks on the bag.
func (validator *ValidationResult) ValidateBag()  {
	if validator.BagNameValid() == false {
		validator.AddError("Bag name is not valid. It should be a UUID.")
		return
	}
	if validator.NsqMessage != nil {
		validator.NsqMessage.Touch()
	}
	if validator.TarFilePath != "" && validator.UntarredPath == "" {
		if validator.untar() == false {
			return
		}
	}
	// Untar can take a long time on large bags.
	// Let NSQ know we're still working on it.
	if validator.NsqMessage != nil {
		validator.NsqMessage.Touch()
	}

	if validator.tagManifestPresent() == false {
		validator.AddError("Tag manifest file 'tagmanifest-sha256.txt' is missing.")
		return
	}

	// OK, the name is good, we untarred it and the tag manifest is valid.
	// Now do the heavy work... and there can be a lot to do on bags
	// over 100GB in size.
	bag, err := bagins.ReadBag(validator.UntarredPath, TagFiles(), "manifest-sha256.txt")
	if err != nil {
		validator.AddError(fmt.Sprintf("Error unpacking bag: %v", err))
		return
	}

	fileNames, err := bag.ListFiles()
	if err != nil {
		validator.AddError(fmt.Sprintf("Could not list bag files: %v ", err))
	}

	dataDirPrefix := "data/"
	if runtime.GOOS == "windows" {
		dataDirPrefix = "data\\"
	}

	hasBagit := false
	hasDPNInfo := false
	hasManifest := false
	hasDataFiles := false
	for _, fileName := range fileNames {
		if fileName == "bagit.txt" {
			hasBagit = true
		} else if fileName == filepath.Join("dpn-tags", "dpn-info.txt") {
			hasDPNInfo = true
		} else if fileName == "manifest-sha256.txt" {
			hasManifest = true
		} else if strings.HasPrefix(fileName, dataDirPrefix) {
			hasDataFiles = true
		}
	}
	if !hasBagit {
		validator.AddError("Bag is missing bagit.txt file.")
	}
	if !hasDPNInfo {
		validator.AddError("Bag is missing dpn-info.txt file.")
	}
	if !hasManifest {
		validator.AddError("Bag is missing manifest-md5.txt file.")
	}
	if !hasDataFiles {
		validator.AddError("Bag's data directory is missing or empty.")
	}

	// Make sure the tag files have the required tags.
	// They can be empty, but they have to be present.
	validator.checkRequiredTags(bag)

	// Run all the checksums on all files.
	checksumErrors := bag.Manifest.RunChecksums()

	// Running the checksums takes a long time on
	// large bags, so let NSQ know we're still working.
	if validator.NsqMessage != nil {
		validator.NsqMessage.Touch()
	}

	for _, err := range checksumErrors {
		validator.AddError(err.Error())
	}
}

// Extract all of the tags from tag files "bagit.txt", "bag-info.txt",
// and "dpn-info.txt" and make sure the required tags are present.
// There may be other tag files, but since they're optional, we don't
// have to check their content.
func (validator *ValidationResult) checkRequiredTags(bag *bagins.Bag) {
	for _, file := range TagFiles() {
		tagFile, err := bag.TagFile(file)
		if err != nil {
			validator.AddError(fmt.Sprintf("Error reading tags from file '%s': %v", file, err))
			return
		}
		tagFields := tagFile.Data.Fields()
		requiredTags := TAGS_FOR_FILE[file]
		tagsFound := make(map[string]bool)
		for _, tagField := range tagFields {
			tagsFound[tagField.Label()] = true
		}
		for _, tagName := range requiredTags {
			if _, ok := tagsFound[tagName]; !ok {
				validator.AddError(fmt.Sprintf("Required tag '%s' is missing from %s", tagName, file))
			}
		}
	}
}

func (validator *ValidationResult) BagNameValid() (bool) {
	bagPath := validator.TarFilePath
	if bagPath == "" {
		bagPath = validator.UntarredPath
	}
	basename := strings.Replace(filepath.Base(bagPath), ".tar", "", 1)
	return bagman.LooksLikeUUID(basename)
}

// If the tag manifest is present, bagins will validate it.
// We have to make sure it's here, and bagins will do the rest.
func (validator *ValidationResult) tagManifestPresent() (bool) {
	fullPath := filepath.Join(validator.UntarredPath, "tagmanifest-sha256.txt")
	return bagman.FileExists(fullPath)
}

func (validator *ValidationResult) CalculateTagManifestDigest(nonce string)  {
	filePath := validator.PathToFileInBag("tagmanifest-sha256.txt")
	src, err := os.Open(filePath)
	if err != nil {
		validator.AddError(fmt.Sprintf("Error reading tag manifest: %v", err))
	}
	defer src.Close()

	shaHash := sha256.New()
	_, err = io.Copy(shaHash, src)
	if err != nil {
		validator.AddError(fmt.Sprintf("Error calculating checksum on tag manifest: %v", err))
	}
	if nonce == "" {
		validator.TagManifestChecksum = fmt.Sprintf("%x", shaHash.Sum(nil))
	} else {
		validator.TagManifestChecksum = fmt.Sprintf("%x", shaHash.Sum([]byte(nonce)))
	}
}

func (validator *ValidationResult) untar() (bool) {
	absInputFile, err := filepath.Abs(validator.TarFilePath)
	if err != nil {
		validator.AddError(fmt.Sprintf("Before untarring, could not determine "+
			"absolute path to tar file: %v", err))
		return false
	}

	// Open the tar file for reading.
	file, err := os.Open(validator.TarFilePath)
	if file != nil {
		defer file.Close()
	}
	if err != nil {
		validator.AddError(fmt.Sprintf("Could not open file %s for untarring: %v",
			validator.TarFilePath, err))
		return false
	}

	// Untar the file and record the validators.
	tarReader := tar.NewReader(file)

	for {
		header, err := tarReader.Next()
		if err != nil && err.Error() == "EOF" {
			break // end of archive
		}
		if err != nil {
			validator.AddError(fmt.Sprintf(
				"Error reading tar file header: %v. " +
					"Either this is not a tar file, or the file is corrupt.", err))
			return false
		}

		// Set the untarred path, which will usually be the depositor's
		// bag identifier.
		if validator.UntarredPath == "" {
			nameParts := strings.Split(header.Name, string(os.PathSeparator))
			validator.UntarredPath = filepath.Join(filepath.Dir(absInputFile), nameParts[0])
		}

		outputPath := filepath.Join(filepath.Dir(absInputFile), header.Name)

		// Make sure the directory that we're about to write into exists.
		err = os.MkdirAll(filepath.Dir(outputPath), 0755)
		if err != nil {
			validator.AddError(fmt.Sprintf("Could not create destination file '%s' "+
				"while unpacking tar archive: %v", outputPath, err))
			return false
		}

		// Copy the file, if it's an actual file. Otherwise, ignore it and record
		// a warning. The bag library does not deal with items like symlinks.
		if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA {
			validator.saveFile(outputPath, tarReader)
		} else if header.Typeflag != tar.TypeDir {
			validator.AddWarning(
				fmt.Sprintf("Ignoring item %s of type %c because it's neither a file nor a directory",
					header.Name, header.Typeflag))
		}
	}
	return true
}

func (validator *ValidationResult) saveFile (destination string, tarReader *tar.Reader) {
	outputWriter, err := os.OpenFile(destination, os.O_CREATE|os.O_WRONLY, 0644)
	if outputWriter != nil {
		defer outputWriter.Close()
	}
	if err != nil {
		validator.AddError(fmt.Sprintf("Error opening file '%s': %v", destination, err))
		return
	}
	_, err = io.Copy(outputWriter, tarReader)
	if err != nil {
		validator.AddError(fmt.Sprintf("Error copying file to '%s': %v", destination, err))
	}
}

// We had to untar the bag to validate it, but once validation
// is done, all we need is the tarred bag, which we'll send to
// storage. Delete the untarred dir if we're not in test mode.
// We know we're in test mode if there's no validator.NsqMessage.
func (validator *ValidationResult) DeleteUntarredBag () {
	if validator.NsqMessage != nil && validator.UntarredPath != "" {
		//fmt.Println(validator.UntarredPath)
		os.RemoveAll(validator.UntarredPath)
	}
}
<|endoftext|>