<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"errors"
	"log"
	"time"
)

const (
	minimumRefreshPeriod    = 5 * time.Minute
	notificationRetryPeriod = 1 * time.Minute
)

// AggregationRule creates and manages the scope for received events.
type AggregationRule struct {
	Filters Filters

	RepeatRate time.Duration
}

type AggregationInstances []*AggregationInstance

type AggregationInstance struct {
	Rule  *AggregationRule
	Event *Event

	// When was this AggregationInstance created?
	Created time.Time
	// When was the last refresh received into this AggregationInstance?
	LastRefreshed time.Time

	// When was the last successful notification sent out for this
	// AggregationInstance?
	lastNotificationSent time.Time
	// Timer used to trigger a notification retry/resend.
	notificationResendTimer *time.Timer
	// Timer used to trigger the deletion of the AggregationInstance after it
	// hasn't been refreshed for too long.
	expiryTimer *time.Timer
}

func (r *AggregationRule) Handles(e *Event) bool {
	return r.Filters.Handles(e)
}

func (r *AggregationInstance) Ingest(e *Event) {
	r.Event = e
	r.LastRefreshed = time.Now()

	r.expiryTimer.Reset(minimumRefreshPeriod)
}

func (r *AggregationInstance) SendNotification(s SummaryReceiver) {
	if time.Since(r.lastNotificationSent) < r.Rule.RepeatRate {
		return
	}

	err := s.Receive(&EventSummary{
		Rule:  r.Rule,
		Event: r.Event,
	})
	if err != nil {
		log.Printf("Error while sending notification: %s, retrying in %v", err, notificationRetryPeriod)
		r.resendNotificationAfter(notificationRetryPeriod, s)
		return
	}

	r.resendNotificationAfter(r.Rule.RepeatRate, s)
	r.lastNotificationSent = time.Now()
}

func (r *AggregationInstance) resendNotificationAfter(d time.Duration, s SummaryReceiver) {
	// BUG: we can't just call SendNotification whenever the timer ends without
	// any synchronisation. The timer should instead feed into a channel which is
	// served by the main Dispatch() loop.
	r.notificationResendTimer = time.AfterFunc(d, func() {
		r.SendNotification(s)
	})
}

func (r *AggregationInstance) Close() {
	if r.notificationResendTimer != nil {
		r.notificationResendTimer.Stop()
	}
	if r.expiryTimer != nil {
		r.expiryTimer.Stop()
	}
}

type AggregationRules []*AggregationRule

type Aggregator struct {
	Rules      AggregationRules
	Aggregates map[EventFingerprint]*AggregationInstance

	aggRequests             chan *aggregateEventsRequest
	getAggregatesRequests   chan *getAggregatesRequest
	removeAggregateRequests chan EventFingerprint
	rulesRequests           chan *aggregatorResetRulesRequest
	closed                  chan bool
}

func NewAggregator() *Aggregator {
	return &Aggregator{
		Aggregates: make(map[EventFingerprint]*AggregationInstance),

		aggRequests:             make(chan *aggregateEventsRequest),
		getAggregatesRequests:   make(chan *getAggregatesRequest),
		removeAggregateRequests: make(chan EventFingerprint),
		rulesRequests:           make(chan *aggregatorResetRulesRequest),
		closed:                  make(chan bool),
	}
}

func (a *Aggregator) Close() {
	close(a.rulesRequests)
	close(a.aggRequests)
	close(a.getAggregatesRequests)
	close(a.removeAggregateRequests)

	<-a.closed
	close(a.closed)
}

type aggregateEventsResponse struct {
	Err error
}

type aggregateEventsRequest struct {
	Events Events

	Response chan *aggregateEventsResponse
}

type getAggregatesResponse struct {
	Aggregates AggregationInstances
}

type getAggregatesRequest struct {
	Response chan getAggregatesResponse
}

func (a *Aggregator) aggregate(req *aggregateEventsRequest, s SummaryReceiver) {
	if len(a.Rules) == 0 {
		req.Response <- &aggregateEventsResponse{
			Err: errors.New("No aggregation rules"),
		}
		close(req.Response)
		return
	}
	log.Println("aggregating", *req)
	for _, element := range req.Events {
		for _, r := range a.Rules {
			log.Println("Checking rule", r, r.Handles(element))
			if r.Handles(element) {
				fp := element.Fingerprint()
				aggregation, ok := a.Aggregates[fp]
				if !ok {
					expTimer := time.AfterFunc(minimumRefreshPeriod, func() {
						a.removeAggregateRequests <- fp
					})

					aggregation = &AggregationInstance{
						Rule:        r,
						Created:     time.Now(),
						expiryTimer: expTimer,
					}

					a.Aggregates[fp] = aggregation
				}

				aggregation.Ingest(element)
				aggregation.SendNotification(s)
				break
			}
		}
	}

	req.Response <- new(aggregateEventsResponse)
	close(req.Response)
}

type aggregatorResetRulesResponse struct{}

type aggregatorResetRulesRequest struct {
	Rules AggregationRules

	Response chan *aggregatorResetRulesResponse
}

func (a *Aggregator) replaceRules(r *aggregatorResetRulesRequest) {
	log.Println("Replacing", len(r.Rules), "aggregator rules...")
	a.Rules = r.Rules

	r.Response <- new(aggregatorResetRulesResponse)
	close(r.Response)
}

func (a *Aggregator) AlertAggregates() AggregationInstances {
	req := &getAggregatesRequest{
		Response: make(chan getAggregatesResponse),
	}

	a.getAggregatesRequests <- req

	result := <-req.Response

	return result.Aggregates
}

func (a *Aggregator) aggregates() AggregationInstances {
	aggs := make(AggregationInstances, 0, len(a.Aggregates))
	for _, agg := range a.Aggregates {
		aggs = append(aggs, agg)
	}
	return aggs
}

func (a *Aggregator) Receive(e Events) error {
	req := &aggregateEventsRequest{
		Events:   e,
		Response: make(chan *aggregateEventsResponse),
	}

	a.aggRequests <- req

	result := <-req.Response

	return result.Err
}

func (a *Aggregator) SetRules(r AggregationRules) error {
	req := &aggregatorResetRulesRequest{
		Rules:    r,
		Response: make(chan *aggregatorResetRulesResponse),
	}

	a.rulesRequests <- req

	_ = <-req.Response

	return nil
}

func (a *Aggregator) Dispatch(s SummaryReceiver) {
	t := time.NewTicker(time.Second)
	defer t.Stop()

	closed := 0

	for closed < 2 {
		select {
		case req, open := <-a.aggRequests:
			a.aggregate(req, s)

			if !open {
				closed++
			}

		case rules, open := <-a.rulesRequests:
			a.replaceRules(rules)

			if !open {
				closed++
			}

		case req := <-a.getAggregatesRequests:
			aggs := a.aggregates()
			req.Response <- getAggregatesResponse{
				Aggregates: aggs,
			}
			close(req.Response)

		case fp := <-a.removeAggregateRequests:
			log.Println("Deleting expired aggregation instance", a)
			a.Aggregates[fp].Close()
			delete(a.Aggregates, fp)
		}
	}

	a.closed <- true
}
<commit_msg>Synchronize Close(), fix race conditions.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"errors"
	"log"
	"time"
)

const (
	minimumRefreshPeriod    = 5 * time.Minute
	notificationRetryPeriod = 1 * time.Minute
)

// AggregationRule creates and manages the scope for received events.
type AggregationRule struct {
	Filters Filters

	RepeatRate time.Duration
}

type AggregationInstances []*AggregationInstance

type AggregationInstance struct {
	Rule  *AggregationRule
	Event *Event

	// When was this AggregationInstance created?
	Created time.Time
	// When was the last refresh received into this AggregationInstance?
	LastRefreshed time.Time

	// When was the last successful notification sent out for this
	// AggregationInstance?
	lastNotificationSent time.Time
	// Timer used to trigger a notification retry/resend.
	notificationResendTimer *time.Timer
	// Timer used to trigger the deletion of the AggregationInstance after it
	// hasn't been refreshed for too long.
	expiryTimer *time.Timer
}

func (r *AggregationRule) Handles(e *Event) bool {
	return r.Filters.Handles(e)
}

func (r *AggregationInstance) Ingest(e *Event) {
	r.Event = e
	r.LastRefreshed = time.Now()

	r.expiryTimer.Reset(minimumRefreshPeriod)
}

func (r *AggregationInstance) SendNotification(s SummaryReceiver) {
	if time.Since(r.lastNotificationSent) < r.Rule.RepeatRate {
		return
	}

	err := s.Receive(&EventSummary{
		Rule:  r.Rule,
		Event: r.Event,
	})
	if err != nil {
		log.Printf("Error while sending notification: %s, retrying in %v", err, notificationRetryPeriod)
		r.resendNotificationAfter(notificationRetryPeriod, s)
		return
	}

	r.resendNotificationAfter(r.Rule.RepeatRate, s)
	r.lastNotificationSent = time.Now()
}

func (r *AggregationInstance) resendNotificationAfter(d time.Duration, s SummaryReceiver) {
	// BUG: we can't just call SendNotification whenever the timer ends without
	// any synchronisation. The timer should instead feed into a channel which is
	// served by the main Dispatch() loop.
	r.notificationResendTimer = time.AfterFunc(d, func() {
		r.SendNotification(s)
	})
}

func (r *AggregationInstance) Close() {
	if r.notificationResendTimer != nil {
		r.notificationResendTimer.Stop()
	}
	if r.expiryTimer != nil {
		r.expiryTimer.Stop()
	}
}

type AggregationRules []*AggregationRule

type Aggregator struct {
	Rules      AggregationRules
	Aggregates map[EventFingerprint]*AggregationInstance

	aggRequests             chan *aggregateEventsRequest
	getAggregatesRequests   chan *getAggregatesRequest
	removeAggregateRequests chan EventFingerprint
	rulesRequests           chan *aggregatorResetRulesRequest
	closeRequests           chan *closeRequest
}

func NewAggregator() *Aggregator {
	return &Aggregator{
		Aggregates: make(map[EventFingerprint]*AggregationInstance),

		aggRequests:             make(chan *aggregateEventsRequest),
		getAggregatesRequests:   make(chan *getAggregatesRequest),
		removeAggregateRequests: make(chan EventFingerprint),
		rulesRequests:           make(chan *aggregatorResetRulesRequest),
		closeRequests:           make(chan *closeRequest),
	}
}

func (a *Aggregator) Close() {
	req := &closeRequest{
		done: make(chan bool),
	}
	a.closeRequests <- req
	<-req.done
}

func (a *Aggregator) closeInternal() {
	close(a.rulesRequests)
	close(a.aggRequests)
	close(a.getAggregatesRequests)
	close(a.removeAggregateRequests)
	close(a.closeRequests)
}

type aggregateEventsResponse struct {
	Err error
}

type aggregateEventsRequest struct {
	Events Events

	Response chan *aggregateEventsResponse
}

type getAggregatesResponse struct {
	Aggregates AggregationInstances
}

type getAggregatesRequest struct {
	Response chan getAggregatesResponse
}

type closeRequest struct {
	done chan bool
}

func (a *Aggregator) aggregate(req *aggregateEventsRequest, s SummaryReceiver) {
	if len(a.Rules) == 0 {
		req.Response <- &aggregateEventsResponse{
			Err: errors.New("No aggregation rules"),
		}
		close(req.Response)
		return
	}
	log.Println("aggregating", *req)
	for _, element := range req.Events {
		for _, r := range a.Rules {
			log.Println("Checking rule", r, r.Handles(element))
			if r.Handles(element) {
				fp := element.Fingerprint()
				aggregation, ok := a.Aggregates[fp]
				if !ok {
					expTimer := time.AfterFunc(minimumRefreshPeriod, func() {
						a.removeAggregateRequests <- fp
					})

					aggregation = &AggregationInstance{
						Rule:        r,
						Created:     time.Now(),
						expiryTimer: expTimer,
					}

					a.Aggregates[fp] = aggregation
				}

				aggregation.Ingest(element)
				aggregation.SendNotification(s)
				break
			}
		}
	}

	req.Response <- new(aggregateEventsResponse)
	close(req.Response)
}

type aggregatorResetRulesResponse struct{}

type aggregatorResetRulesRequest struct {
	Rules AggregationRules

	Response chan *aggregatorResetRulesResponse
}

func (a *Aggregator) replaceRules(r *aggregatorResetRulesRequest) {
	log.Println("Replacing", len(r.Rules), "aggregator rules...")
	a.Rules = r.Rules

	r.Response <- new(aggregatorResetRulesResponse)
	close(r.Response)
}

func (a *Aggregator) AlertAggregates() AggregationInstances {
	req := &getAggregatesRequest{
		Response: make(chan getAggregatesResponse),
	}

	a.getAggregatesRequests <- req

	result := <-req.Response

	return result.Aggregates
}

func (a *Aggregator) aggregates() AggregationInstances {
	aggs := make(AggregationInstances, 0, len(a.Aggregates))
	for _, agg := range a.Aggregates {
		aggs = append(aggs, agg)
	}
	return aggs
}

func (a *Aggregator) Receive(e Events) error {
	req := &aggregateEventsRequest{
		Events:   e,
		Response: make(chan *aggregateEventsResponse),
	}

	a.aggRequests <- req

	result := <-req.Response

	return result.Err
}

func (a *Aggregator) SetRules(r AggregationRules) error {
	req := &aggregatorResetRulesRequest{
		Rules:    r,
		Response: make(chan *aggregatorResetRulesResponse),
	}

	a.rulesRequests <- req

	_ = <-req.Response

	return nil
}

func (a *Aggregator) Dispatch(s SummaryReceiver) {
	t := time.NewTicker(time.Second)
	defer t.Stop()

	for {
		select {
		case req := <-a.aggRequests:
			a.aggregate(req, s)

		case rules := <-a.rulesRequests:
			a.replaceRules(rules)

		case req := <-a.getAggregatesRequests:
			aggs := a.aggregates()
			req.Response <- getAggregatesResponse{
				Aggregates: aggs,
			}
			close(req.Response)

		case fp := <-a.removeAggregateRequests:
			log.Println("Deleting expired aggregation instance", a)
			a.Aggregates[fp].Close()
			delete(a.Aggregates, fp)

		case req := <-a.closeRequests:
			a.closeInternal()
			req.done <- true
			return
		}
	}
}
<|endoftext|>


<commit_before>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	if n == nil { // FIXME remove this assertion. protect with constructor invariant
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(n *Node, k u.Key) []int {
	var out []int
	keybytes := []byte(k)
	for i, lnk := range n.Links {
		if bytes.Equal([]byte(lnk.Hash), keybytes) {
			out = append(out, i)
		}
	}
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		defer close(sig)

		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				// input data
				log.Error("Got back bad block!")
				break
			}
			is := FindLinks(root, blk.Key())
			for _, i := range is {
				nodes[i] = nd
			}

			for ; next < len(nodes) && nodes[next] != nil; next++ {
				sig <- nodes[next]
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
	}()

	return sig
}
<commit_msg>merkledag: LONG timeout on Get<commit_after>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	if n == nil { // FIXME remove this assertion. protect with constructor invariant
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Minute)
	// we shouldn't use an arbitrary timeout here.
	// since Get doesnt take in a context yet, we give a large upper bound.
	// think of an http request. we want it to go on as long as the client requests it.

	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(n *Node, k u.Key) []int {
	var out []int
	keybytes := []byte(k)
	for i, lnk := range n.Links {
		if bytes.Equal([]byte(lnk.Hash), keybytes) {
			out = append(out, i)
		}
	}
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		defer close(sig)

		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				// input data
				log.Error("Got back bad block!")
				break
			}
			is := FindLinks(root, blk.Key())
			for _, i := range is {
				nodes[i] = nd
			}

			for ; next < len(nodes) && nodes[next] != nil; next++ {
				sig <- nodes[next]
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
	}()

	return sig
}
<|endoftext|>


<commit_before>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileresource

import "fmt"

const (
	githubProtocol = "https://"
	githubRepo     = "raw.githubusercontent.com/kris-nova/kubicorn"
	githubBranch   = "master"
)

// getGitHubUrl will build a query-able URL from a bootstrap script that we can parse in at runtime.
// Example URL:
// https://raw.githubusercontent.com/kris-nova/kubicorn/master/bootstrap/amazon_k8s_centos_7_master.sh
func getGitHubUrl(bootstrapScript string) string {
	gitHubUrl := fmt.Sprintf("%s%s/%s/%s", githubProtocol, githubRepo, githubBranch, bootstrapScript)
	return gitHubUrl
}
<commit_msg>change repo location for bootstrap scripts<commit_after>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileresource

import "fmt"

const (
	githubProtocol = "https://"
	githubRepo     = "raw.githubusercontent.com/cschiewek/kubicorn"
	githubBranch   = "master"
)

// getGitHubUrl will build a query-able URL from a bootstrap script that we can parse in at runtime.
// Example URL:
// https://raw.githubusercontent.com/kris-nova/kubicorn/master/bootstrap/amazon_k8s_centos_7_master.sh
func getGitHubUrl(bootstrapScript string) string {
	gitHubUrl := fmt.Sprintf("%s%s/%s/%s", githubProtocol, githubRepo, githubBranch, bootstrapScript)
	return gitHubUrl
}
<|endoftext|>


<commit_before>/*
   PulseHA - HA Cluster Daemon
   Copyright (C) 2017-2018  Andrew Zak <andrew@pulseha.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package server

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/Syleron/PulseHA/proto"
	"github.com/Syleron/PulseHA/src/client"
	"github.com/Syleron/PulseHA/src/utils"
	"google.golang.org/grpc/connectivity"
	"math"
	"sync"
	"time"
)

/**
Member struct type
*/
type Member struct {
	// The hostname of the repented node
	Hostname string
	// The status of the local member
	Status proto.MemberStatus_Status
	// The last time a health check was received
	LastHCResponse time.Time
	// The latency between the active and the current passive member
	Latency string
	// Determines if the health check is being made.
	HCBusy bool
	// The client for the member that is used to send GRPC calls
	*client.Client
	// The mutex to lock the member object
	sync.Mutex
}

/**

 */
func (m *Member) Lock() {
	//_, _, no, _ := runtime.Caller(1)
	//log.Debugf("Member:Lock() Lock set line: %d by %s", no, MyCaller())
	m.Mutex.Lock()
}

/**

 */
func (m *Member) Unlock() {
	//_, _, no, _ := runtime.Caller(1)
	//log.Debugf("Member:Unlock() Unlock set line: %d by %s", no, MyCaller())
	m.Mutex.Unlock()
}

/*
	Getters and setters for Member which allow us to make them go routine safe
*/

/**

 */
func (m *Member) SetHCBusy(busy bool) {
	m.Lock()
	defer m.Unlock()
	m.HCBusy = busy
}

/**

 */
func (m *Member) GetHCBusy() bool {
	m.Lock()
	defer m.Unlock()
	return m.HCBusy
}

/**

 */
func (m *Member) SetLatency(latency string) {
	m.Lock()
	defer m.Unlock()
	m.Latency = latency
}

/**

 */
func (m *Member) GetLatency() string {
	m.Lock()
	defer m.Unlock()
	return m.Latency
}

/**
  Set the last time this member received a health check
*/
func (m *Member) SetLastHCResponse(time time.Time) {
	m.Lock()
	defer m.Unlock()
	m.LastHCResponse = time
}

/**
Get the last time this member received a health check
*/
func (m *Member) GetLastHCResponse() time.Time {
	m.Lock()
	defer m.Unlock()
	return m.LastHCResponse
}

/**
Get member hostname
*/
func (m *Member) GetHostname() string {
	m.Lock()
	defer m.Unlock()
	return m.Hostname
}

/**
Set member hostname
*/
func (m *Member) SetHostname(hostname string) {
	m.Lock()
	defer m.Unlock()
	m.Hostname = hostname
}

/**
Get member status
*/
func (m *Member) GetStatus() proto.MemberStatus_Status {
	//log.Debug("Member:getStatus() called by " + MyCaller())
	m.Lock()
	defer m.Unlock()
	return m.Status
}

/**
Set member status
*/
func (m *Member) SetStatus(status proto.MemberStatus_Status) {
	log.Debug("Member:setStatus() " + m.GetHostname() + " status set to " + status.String() + " called by " + MyCaller())
	m.Lock()
	defer m.Unlock()
	m.Status = status
}

/**
Set member Client GRPC
*/
func (m *Member) SetClient(client *client.Client) {
	m.Lock()
	defer m.Unlock()
	m.Client = client
}

/**
Note: Hostname is required for TLS as the certs are named after the hostname.
*/
func (m *Member) Connect() error {
	if (m.Connection == nil) || (m.Connection != nil && m.Connection.GetState() == connectivity.Shutdown) {
		nodeDetails, _ := nodeGetByName(m.Hostname)
		log.Debug("Member:Connect() Attempting to connect with node " + m.Hostname + " " + nodeDetails.IP + ":" + nodeDetails.Port)
		err := m.Client.Connect(nodeDetails.IP, nodeDetails.Port, m.Hostname, DB.Config.Pulse.TLS)
		if err != nil {
			log.Error("Member:Connect() " + err.Error())
			return err
		}
	}
	return nil
}

/**
Close the client connection
*/
func (m *Member) Close() {
	log.Debug("Member:Close() Connection closed")
	m.Client.Close()
}

/**
Active function - Send GRPC health check to current member
*/
func (m *Member) SendHealthCheck(data *proto.PulseHealthCheck) (interface{}, error) {
	if m.Connection == nil {
		return nil, errors.New("unable to send health check as member connection has not been initiated")
	}
	startTime := time.Now()
	r, err := m.Send(client.SendHealthCheck, data)
	// This is a record for the active appliance to know when it was last sent/received!
	m.SetLastHCResponse(time.Now())
	elapsed := fmt.Sprint(time.Since(startTime).Round(time.Millisecond))
	m.SetLatency(elapsed)
	return r, err
}

/**
Send health check via a go routine and mark the HC busy/not
*/
func (m *Member) RoutineHC(data *proto.PulseHealthCheck) {
	m.SetHCBusy(true)
	_, err := m.SendHealthCheck(data)
	if err != nil {
		m.Close()
		m.SetStatus(proto.MemberStatus_UNAVAILABLE) // This may not be required
	}
	m.SetHCBusy(false)
}

/*
	Make the node active (bring up its groups)
*/
func (m *Member) MakeActive() bool {
	log.Debugf("Member:makeActive() Making %s active", m.GetHostname())
	// Make ourself active if we are refering to ourself
	if m.GetHostname() == DB.Config.GetLocalNode() {
		MakeLocalActive()
		// Reset vars
		m.SetLatency("")
		m.SetLastHCResponse(time.Time{})
		m.SetStatus(proto.MemberStatus_ACTIVE)
		// Start performing health checks
		log.Debug("Member:PromoteMember() Starting client connections monitor")
		go utils.Scheduler(DB.MemberList.MonitorClientConns, 1*time.Second)
		log.Debug("Member:PromoteMember() Starting health check handler")
		go utils.Scheduler(DB.MemberList.AddHealthCheckHandler, 1*time.Second)
	} else {
		// TODO: Handle the closing of this connection
		m.Connect()
		_, err := m.Send(
			client.SendPromote,
			&proto.PulsePromote{
				Member: m.GetHostname(),
			})
		// Handle if we have an error
		if err != nil {
			log.Error(err)
			log.Errorf("Error making %s passive. Error: %s", m.GetHostname(), err.Error())
			return false
		}
	}
	return true
}

/**
Make the node passive (take down its groups)
*/
func (m *Member) MakePassive() bool {
	log.Debugf("Member:makePassive() Making %s passive", m.GetHostname())
	if m.GetHostname() == DB.Config.GetLocalNode() {
		// do this regardless to make sure we dont have any groups up
		MakeLocalPassive()
		// Update member variables
		m.SetLastHCResponse(time.Now())
		// check if we are already passive before starting a new scheduler
		if m.GetStatus() != proto.MemberStatus_PASSIVE {
			m.SetStatus(proto.MemberStatus_PASSIVE)
			// Start the scheduler
			log.Debug("Member:makePassive() Starting the monitor received health checks scheduler " + m.GetHostname())
			go utils.Scheduler(m.MonitorReceivedHCs, 10000*time.Millisecond)
		}
	} else {
		// TODO: Handle the closing of this connection
		m.Connect()
		_, err := m.Send(
			client.SendMakePassive,
			&proto.PulsePromote{
				Member: m.GetHostname(),
			})
		if err != nil {
			log.Error(err)
			log.Errorf("Error making %s passive. Error: %s", m.GetHostname(), err.Error())
			return false
		}
	}
	return true
}

/**
Used to bring up a single IP on member
Note: We need to know the group to work out what interface to
bring it up on.
*/
func (m *Member) BringUpIPs(ips []string, group string) bool {
	iface := DB.Config.GetGroupIface(m.Hostname, group)
	if m.Hostname == DB.Config.GetLocalNode() {
		log.Debug("member is local node bringing up IP's")
		BringUpIPs(iface, ips)
	} else {
		log.Debug("member is not local node making grpc call")
		_, err := m.Send(
			client.SendBringUpIP,
			&proto.PulseBringIP{
				Iface: iface,
				Ips:   ips,
			})
		if err != nil {
			log.Error(err)
			log.Errorf("Error making %s passive. Error: %s", m.GetHostname(), err.Error())
			return false
		}
	}
	return true
}

/**
Monitor the last time we received a health check and or failover
*/
func (m *Member) MonitorReceivedHCs() bool {
	// make sure we are still the active appliance
	member, err := DB.MemberList.GetLocalMember()
	if err != nil {
		log.Debug("Member:monitorReceivedHCs() Health check received monitor disabled as we are no longer in a cluster")
		return true
	}
	if member.GetStatus() == proto.MemberStatus_ACTIVE {
		log.Debug("Member:monitorReceivedHCs() Health check received monitor disabled as we are now active.")
		return true
	}
	// calculate elapsed time
	elapsed := math.Floor(float64(time.Since(m.GetLastHCResponse()).Seconds()))
	// determine if we might need to failover
	if int(elapsed) > 0 && int(elapsed)%4 == 0 {
		log.Warning("No health checks are being made.. Perhaps a failover is required?")
	}
	// has our threshold been met? Failover?
	//log.Info(elapsed)
	if int(elapsed) >= 10 {
		log.Debug("Member:monitorReceivedHCs() Performing Failover..")
		var addHCSuccess bool = false
		// TODO: Perform additional health checks plugin stuff HERE
		if !addHCSuccess {
			// Nothing has worked.. assume the master has failed. Fail over.
			member, err := DB.MemberList.GetNextActiveMember()
			// no new active appliance was found
			if err != nil {
				log.Warn("unable to find new active member.. we are now the active")
				// make ourself active as no new active can be found apparently
				m.MakeActive()
				return true
			}
			// If we are not the new member just return
			if member.GetHostname() != DB.Config.GetLocalNode() {
				log.Info("Waiting on " + member.GetHostname() + " to become active")
				m.SetLastHCResponse(time.Now())
				return false
			}
			// get our current active member
			_, activeMember := DB.MemberList.GetActiveMember()
			// If we have an active appliance mark it unavailable
			if activeMember != nil {
				activeMember.SetStatus(proto.MemberStatus_UNAVAILABLE)
			}
			// lets go active
			member.MakeActive()
			// Set the FO priority
			member.SetLastHCResponse(time.Time{})
			log.Info("Local node is now active")
			return true
		} else {
			m.SetLastHCResponse(time.Now())
		}
	}
	return false
}
<commit_msg>Added the suspicious state to help determine signify the possibility of a failed node.<commit_after>/*
   PulseHA - HA Cluster Daemon
   Copyright (C) 2017-2018  Andrew Zak <andrew@pulseha.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package server

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/Syleron/PulseHA/proto"
	"github.com/Syleron/PulseHA/src/client"
	"github.com/Syleron/PulseHA/src/utils"
	"google.golang.org/grpc/connectivity"
	"math"
	"sync"
	"time"
)

/**
Member struct type
*/
type Member struct {
	// The hostname of the repented node
	Hostname string
	// The status of the local member
	Status proto.MemberStatus_Status
	// The last time a health check was received
	LastHCResponse time.Time
	// The latency between the active and the current passive member
	Latency string
	// Determines if the health check is being made.
	HCBusy bool
	// The client for the member that is used to send GRPC calls
	*client.Client
	// The mutex to lock the member object
	sync.Mutex
}

/**

 */
func (m *Member) Lock() {
	//_, _, no, _ := runtime.Caller(1)
	//log.Debugf("Member:Lock() Lock set line: %d by %s", no, MyCaller())
	m.Mutex.Lock()
}

/**

 */
func (m *Member) Unlock() {
	//_, _, no, _ := runtime.Caller(1)
	//log.Debugf("Member:Unlock() Unlock set line: %d by %s", no, MyCaller())
	m.Mutex.Unlock()
}

/*
	Getters and setters for Member which allow us to make them go routine safe
*/

/**

 */
func (m *Member) SetHCBusy(busy bool) {
	m.Lock()
	defer m.Unlock()
	m.HCBusy = busy
}

/**

 */
func (m *Member) GetHCBusy() bool {
	m.Lock()
	defer m.Unlock()
	return m.HCBusy
}

/**

 */
func (m *Member) SetLatency(latency string) {
	m.Lock()
	defer m.Unlock()
	m.Latency = latency
}

/**

 */
func (m *Member) GetLatency() string {
	m.Lock()
	defer m.Unlock()
	return m.Latency
}

/**
  Set the last time this member received a health check
*/
func (m *Member) SetLastHCResponse(time time.Time) {
	m.Lock()
	defer m.Unlock()
	m.LastHCResponse = time
}

/**
Get the last time this member received a health check
*/
func (m *Member) GetLastHCResponse() time.Time {
	m.Lock()
	defer m.Unlock()
	return m.LastHCResponse
}

/**
Get member hostname
*/
func (m *Member) GetHostname() string {
	m.Lock()
	defer m.Unlock()
	return m.Hostname
}

/**
Set member hostname
*/
func (m *Member) SetHostname(hostname string) {
	m.Lock()
	defer m.Unlock()
	m.Hostname = hostname
}

/**
Get member status
*/
func (m *Member) GetStatus() proto.MemberStatus_Status {
	//log.Debug("Member:getStatus() called by " + MyCaller())
	m.Lock()
	defer m.Unlock()
	return m.Status
}

/**
Set member status
*/
func (m *Member) SetStatus(status proto.MemberStatus_Status) {
	log.Debug("Member:setStatus() " + m.GetHostname() + " status set to " + status.String() + " called by " + MyCaller())
	m.Lock()
	defer m.Unlock()
	m.Status = status
}

/**
Set member Client GRPC
*/
func (m *Member) SetClient(client *client.Client) {
	m.Lock()
	defer m.Unlock()
	m.Client = client
}

/**
Note: Hostname is required for TLS as the certs are named after the hostname.
*/
func (m *Member) Connect() error {
	if (m.Connection == nil) || (m.Connection != nil && m.Connection.GetState() == connectivity.Shutdown) {
		nodeDetails, _ := nodeGetByName(m.Hostname)
		log.Debug("Member:Connect() Attempting to connect with node " + m.Hostname + " " + nodeDetails.IP + ":" + nodeDetails.Port)
		err := m.Client.Connect(nodeDetails.IP, nodeDetails.Port, m.Hostname, DB.Config.Pulse.TLS)
		if err != nil {
			log.Error("Member:Connect() " + err.Error())
			return err
		}
	}
	return nil
}

/**
Close the client connection
*/
func (m *Member) Close() {
	log.Debug("Member:Close() Connection closed")
	m.Client.Close()
}

/**
Active function - Send GRPC health check to current member
*/
func (m *Member) SendHealthCheck(data *proto.PulseHealthCheck) (interface{}, error) {
	if m.Connection == nil {
		return nil, errors.New("unable to send health check as member connection has not been initiated")
	}
	startTime := time.Now()
	r, err := m.Send(client.SendHealthCheck, data)
	// This is a record for the active appliance to know when it was last sent/received!
	m.SetLastHCResponse(time.Now())
	elapsed := fmt.Sprint(time.Since(startTime).Round(time.Millisecond))
	m.SetLatency(elapsed)
	return r, err
}

/**
Send health check via a go routine and mark the HC busy/not
*/
func (m *Member) RoutineHC(data *proto.PulseHealthCheck) {
	m.SetHCBusy(true)
	_, err := m.SendHealthCheck(data)
	if err != nil {
		m.Close()
		m.SetStatus(proto.MemberStatus_UNAVAILABLE) // This may not be required
	}
	m.SetHCBusy(false)
}

/*
	Make the node active (bring up its groups)
*/
func (m *Member) MakeActive() bool {
	log.Debugf("Member:makeActive() Making %s active", m.GetHostname())
	// Make ourself active if we are refering to ourself
	if m.GetHostname() == DB.Config.GetLocalNode() {
		MakeLocalActive()
		// Reset vars
		m.SetLatency("")
		m.SetLastHCResponse(time.Time{})
		m.SetStatus(proto.MemberStatus_ACTIVE)
		// Start performing health checks
		log.Debug("Member:PromoteMember() Starting client connections monitor")
		go utils.Scheduler(DB.MemberList.MonitorClientConns, 1*time.Second)
		log.Debug("Member:PromoteMember() Starting health check handler")
		go utils.Scheduler(DB.MemberList.AddHealthCheckHandler, 1*time.Second)
	} else {
		// TODO: Handle the closing of this connection
		m.Connect()
		_, err := m.Send(
			client.SendPromote,
			&proto.PulsePromote{
				Member: m.GetHostname(),
			})
		// Handle if we have an error
		if err != nil {
			log.Error(err)
			log.Errorf("Error making %s passive. Error: %s", m.GetHostname(), err.Error())
			return false
		}
	}
	return true
}

/**
Make the node passive (take down its groups)
*/
func (m *Member) MakePassive() bool {
	log.Debugf("Member:makePassive() Making %s passive", m.GetHostname())
	if m.GetHostname() == DB.Config.GetLocalNode() {
		// do this regardless to make sure we dont have any groups up
		MakeLocalPassive()
		// Update member variables
		m.SetLastHCResponse(time.Now())
		// check if we are already passive before starting a new scheduler
		if m.GetStatus() != proto.MemberStatus_PASSIVE {
			m.SetStatus(proto.MemberStatus_PASSIVE)
			// Start the scheduler
			log.Debug("Member:makePassive() Starting the monitor received health checks scheduler " + m.GetHostname())
			go utils.Scheduler(m.MonitorReceivedHCs, 10000*time.Millisecond)
		}
	} else {
		// TODO: Handle the closing of this connection
		m.Connect()
		_, err := m.Send(
			client.SendMakePassive,
			&proto.PulsePromote{
				Member: m.GetHostname(),
			})
		if err != nil {
			log.Error(err)
			log.Errorf("Error making %s passive. Error: %s", m.GetHostname(), err.Error())
			return false
		}
	}
	return true
}

/**
Used to bring up a single IP on member
Note: We need to know the group to work out what interface to
bring it up on.
*/
func (m *Member) BringUpIPs(ips []string, group string) bool {
	iface := DB.Config.GetGroupIface(m.Hostname, group)
	if m.Hostname == DB.Config.GetLocalNode() {
		log.Debug("member is local node bringing up IP's")
		BringUpIPs(iface, ips)
	} else {
		log.Debug("member is not local node making grpc call")
		_, err := m.Send(
			client.SendBringUpIP,
			&proto.PulseBringIP{
				Iface: iface,
				Ips:   ips,
			})
		if err != nil {
			log.Error(err)
			log.Errorf("Error making %s passive. Error: %s", m.GetHostname(), err.Error())
			return false
		}
	}
	return true
}

/**
Monitor the last time we received a health check and or failover
*/
func (m *Member) MonitorReceivedHCs() bool {
	// make sure we are still the active appliance
	member, err := DB.MemberList.GetLocalMember()
	if err != nil {
		log.Debug("Member:monitorReceivedHCs() Health check received monitor disabled as we are no longer in a cluster")
		return true
	}
	if member.GetStatus() == proto.MemberStatus_ACTIVE {
		log.Debug("Member:monitorReceivedHCs() Health check received monitor disabled as we are now active.")
		return true
	}
	// calculate elapsed time
	elapsed := math.Floor(float64(time.Since(m.GetLastHCResponse()).Seconds()))
	// determine if we might need to failover
	if int(elapsed) > 0 && int(elapsed)%4 == 0 {
		_, member := DB.MemberList.GetActiveMember()
		if member != nil {
			member.SetStatus(proto.MemberStatus_SUSPICIOUS)
		}
		log.Debug("Member:MonitorReceivedHCs() No health checks are being made.. Perhaps a failover is required?")
	}
	// has our threshold been met? Failover?
	//log.Info(elapsed)
	if int(elapsed) >= 10 {
		log.Debug("Member:monitorReceivedHCs() Performing Failover..")
		var addHCSuccess bool = false
		// TODO: Perform additional health checks plugin stuff HERE
		if !addHCSuccess {
			// Nothing has worked.. assume the master has failed. Fail over.
			member, err := DB.MemberList.GetNextActiveMember()
			// no new active appliance was found
			if err != nil {
				log.Warn("unable to find new active member.. we are now the active")
				// make ourself active as no new active can be found apparently
				m.MakeActive()
				return true
			}
			// If we are not the new member just return
			if member.GetHostname() != DB.Config.GetLocalNode() {
				log.Info("Waiting on " + member.GetHostname() + " to become active")
				m.SetLastHCResponse(time.Now())
				return false
			}
			// get our current active member
			_, activeMember := DB.MemberList.GetActiveMember()
			// If we have an active appliance mark it unavailable
			if activeMember != nil {
				activeMember.SetStatus(proto.MemberStatus_UNAVAILABLE)
			}
			// lets go active
			member.MakeActive()
			// Set the FO priority
			member.SetLastHCResponse(time.Time{})
			log.Info("Local node is now active")
			return true
		} else {
			m.SetLastHCResponse(time.Now())
		}
	}
	return false
}
<|endoftext|>


<commit_before>package daemon

import (
	"fmt"
	"strings"

	"github.com/docker/docker/container"
	"github.com/docker/docker/errors"
	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/reference"
	"github.com/docker/engine-api/types"
)

type conflictType int

const (
	conflictDependentChild conflictType = (1 << iota)
	conflictRunningContainer
	conflictActiveReference
	conflictStoppedContainer
	conflictHard = conflictDependentChild | conflictRunningContainer
	conflictSoft = conflictActiveReference | conflictStoppedContainer
)

// ImageDelete deletes the image referenced by the given imageRef from this
// daemon. The given imageRef can be an image ID, ID prefix, or a repository
// reference (with an optional tag or digest, defaulting to the tag name
// "latest"). There is differing behavior depending on whether the given
// imageRef is a repository reference or not.
//
// If the given imageRef is a repository reference then that repository
// reference will be removed. However, if there exists any containers which
// were created using the same image reference then the repository reference
// cannot be removed unless either there are other repository references to the
// same image or force is true. Following removal of the repository reference,
// the referenced image itself will attempt to be deleted as described below
// but quietly, meaning any image delete conflicts will cause the image to not
// be deleted and the conflict will not be reported.
//
// There may be conflicts preventing deletion of an image and these conflicts
// are divided into two categories grouped by their severity:
//
// Hard Conflict:
// 	- a pull or build using the image.
// 	- any descendant image.
// 	- any running container using the image.
//
// Soft Conflict:
// 	- any stopped container using the image.
// 	- any repository tag or digest references to the image.
//
// The image cannot be removed if there are any hard conflicts and can be
// removed if there are soft conflicts only if force is true.
//
// If prune is true, ancestor images will each attempt to be deleted quietly,
// meaning any delete conflicts will cause the image to not be deleted and the
// conflict will not be reported.
//
// FIXME: remove ImageDelete's dependency on Daemon, then move to the graph
// package. This would require that we no longer need the daemon to determine
// whether images are being used by a stopped or running container.
func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDelete, error) {
	records := []types.ImageDelete{}

	imgID, err := daemon.GetImageID(imageRef)
	if err != nil {
		return nil, daemon.imageNotExistToErrcode(err)
	}

	repoRefs := daemon.referenceStore.References(imgID)

	var removedRepositoryRef bool
	if !isImageIDPrefix(imgID.String(), imageRef) {
		// A repository reference was given and should be removed
		// first. We can only remove this reference if either force is
		// true, there are multiple repository references to this
		// image, or there are no containers using the given reference.
		if !(force || len(repoRefs) > 1) {
			if container := daemon.getContainerUsingImage(imgID); container != nil {
				// If we removed the repository reference then
				// this image would remain "dangling" and since
				// we really want to avoid that the client must
				// explicitly force its removal.
				err := fmt.Errorf("conflict: unable to remove repository reference %q (must force) - container %s is using its referenced image %s", imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(imgID.String()))
				return nil, errors.NewRequestConflictError(err)
			}
		}

		parsedRef, err := reference.ParseNamed(imageRef)
		if err != nil {
			return nil, err
		}

		parsedRef, err = daemon.removeImageRef(parsedRef)
		if err != nil {
			return nil, err
		}

		untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}

		daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
		records = append(records, untaggedRecord)

		repoRefs = daemon.referenceStore.References(imgID)

		// If a tag reference was removed and the only remaining
		// references to the same repository are digest references,
		// then clean up those digest references.
		if _, isCanonical := parsedRef.(reference.Canonical); !isCanonical {
			foundRepoTagRef := false
			for _, repoRef := range repoRefs {
				if _, repoRefIsCanonical := repoRef.(reference.Canonical); !repoRefIsCanonical && parsedRef.Name() == repoRef.Name() {
					foundRepoTagRef = true
					break
				}
			}
			if !foundRepoTagRef {
				// Remove canonical references from same repository
				remainingRefs := []reference.Named{}
				for _, repoRef := range repoRefs {
					if _, repoRefIsCanonical := repoRef.(reference.Canonical); repoRefIsCanonical && parsedRef.Name() == repoRef.Name() {
						if _, err := daemon.removeImageRef(repoRef); err != nil {
							return records, err
						}

						untaggedRecord := types.ImageDelete{Untagged: repoRef.String()}
						records = append(records, untaggedRecord)
					} else {
						remainingRefs = append(remainingRefs, repoRef)

					}
				}
				repoRefs = remainingRefs
			}
		}

		// If it has remaining references then the untag finished the remove
		if len(repoRefs) > 0 {
			return records, nil
		}

		removedRepositoryRef = true
	} else {
		// If an ID reference was given AND there is at most one tag
		// reference to the image AND all references are within one
		// repository, then remove all references.
		if isSingleReference(repoRefs) {
			c := conflictHard
			if !force {
				c |= conflictSoft &^ conflictActiveReference
			}
			if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil {
				return nil, conflict
			}

			for _, repoRef := range repoRefs {
				parsedRef, err := daemon.removeImageRef(repoRef)
				if err != nil {
					return nil, err
				}

				untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}

				daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
				records = append(records, untaggedRecord)
			}
		}
	}

	return records, daemon.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef)
}

// isSingleReference returns true when all references are from one repository
// and there is at most one tag. Returns false for empty input.
func isSingleReference(repoRefs []reference.Named) bool {
	if len(repoRefs) <= 1 {
		return len(repoRefs) == 1
	}
	var singleRef reference.Named
	canonicalRefs := map[string]struct{}{}
	for _, repoRef := range repoRefs {
		if _, isCanonical := repoRef.(reference.Canonical); isCanonical {
			canonicalRefs[repoRef.Name()] = struct{}{}
		} else if singleRef == nil {
			singleRef = repoRef
		} else {
			return false
		}
	}
	if singleRef == nil {
		// Just use first canonical ref
		singleRef = repoRefs[0]
	}
	_, ok := canonicalRefs[singleRef.Name()]
	return len(canonicalRefs) == 1 && ok
}

// isImageIDPrefix returns whether the given possiblePrefix is a prefix of the
// given imageID.
func isImageIDPrefix(imageID, possiblePrefix string) bool {
	if strings.HasPrefix(imageID, possiblePrefix) {
		return true
	}

	if i := strings.IndexRune(imageID, ':'); i >= 0 {
		return strings.HasPrefix(imageID[i+1:], possiblePrefix)
	}

	return false
}

// getContainerUsingImage returns a container that was created using the given
// imageID. Returns nil if there is no such container.
func (daemon *Daemon) getContainerUsingImage(imageID image.ID) *container.Container {
	return daemon.containers.First(func(c *container.Container) bool {
		return c.ImageID == imageID
	})
}

// removeImageRef attempts to parse and remove the given image reference from
// this daemon's store of repository tag/digest references. The given
// repositoryRef must not be an image ID but a repository name followed by an
// optional tag or digest reference. If tag or digest is omitted, the default
// tag is used. Returns the resolved image reference and an error.
func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, error) {
	ref = reference.WithDefaultTag(ref)
	// Ignore the boolean value returned, as far as we're concerned, this
	// is an idempotent operation and it's okay if the reference didn't
	// exist in the first place.
	_, err := daemon.referenceStore.Delete(ref)

	return ref, err
}

// removeAllReferencesToImageID attempts to remove every reference to the given
// imgID from this daemon's store of repository tag/digest references. Returns
// on the first encountered error. Removed references are logged to this
// daemon's event service. An "Untagged" types.ImageDelete is added to the
// given list of records.
func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDelete) error {
	imageRefs := daemon.referenceStore.References(imgID)

	for _, imageRef := range imageRefs {
		parsedRef, err := daemon.removeImageRef(imageRef)
		if err != nil {
			return err
		}

		untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}

		daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
		*records = append(*records, untaggedRecord)
	}

	return nil
}

// ImageDeleteConflict holds a soft or hard conflict and an associated error.
// Implements the error interface.
type imageDeleteConflict struct {
	hard    bool
	used    bool
	imgID   image.ID
	message string
}

func (idc *imageDeleteConflict) Error() string {
	var forceMsg string
	if idc.hard {
		forceMsg = "cannot be forced"
	} else {
		forceMsg = "must be forced"
	}

	return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID.String()), forceMsg, idc.message)
}

// imageDeleteHelper attempts to delete the given image from this daemon. If
// the image has any hard delete conflicts (child images or running containers
// using the image) then it cannot be deleted. If the image has any soft delete
// conflicts (any tags/digests referencing the image or any stopped container
// using the image) then it can only be deleted if force is true. If the delete
// succeeds and prune is true, the parent images are also deleted if they do
// not have any soft or hard delete conflicts themselves. Any deleted images
// and untagged references are appended to the given records. If any error or
// conflict is encountered, it will be returned immediately without deleting
// the image. If quiet is true, any encountered conflicts will be ignored and
// the function will return nil immediately without deleting the image.
func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDelete, force, prune, quiet bool) error {
	// First, determine if this image has any conflicts. Ignore soft conflicts
	// if force is true.
	c := conflictHard
	if !force {
		c |= conflictSoft
	}
	if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil {
		if quiet && (!daemon.imageIsDangling(imgID) || conflict.used) {
			// Ignore conflicts UNLESS the image is "dangling" or not being used in
			// which case we want the user to know.
			return nil
		}

		// There was a conflict and it's either a hard conflict OR we are not
		// forcing deletion on soft conflicts.
		return conflict
	}

	parent, err := daemon.imageStore.GetParent(imgID)
	if err != nil {
		// There may be no parent
		parent = ""
	}

	// Delete all repository tag/digest references to this image.
	if err := daemon.removeAllReferencesToImageID(imgID, records); err != nil {
		return err
	}

	removedLayers, err := daemon.imageStore.Delete(imgID)
	if err != nil {
		return err
	}

	daemon.LogImageEvent(imgID.String(), imgID.String(), "delete")
	*records = append(*records, types.ImageDelete{Deleted: imgID.String()})
	for _, removedLayer := range removedLayers {
		*records = append(*records, types.ImageDelete{Deleted: removedLayer.ChainID.String()})
	}

	if !prune || parent == "" {
		return nil
	}

	// We need to prune the parent image. This means delete it if there are
	// no tags/digests referencing it and there are no containers using it (
	// either running or stopped).
	// Do not force prunings, but do so quietly (stopping on any encountered
	// conflicts).
	return daemon.imageDeleteHelper(parent, records, false, true, true)
}

// checkImageDeleteConflict determines whether there are any conflicts
// preventing deletion of the given image from this daemon. A hard conflict is
// any image which has the given image as a parent or any running container
// using the image. A soft conflict is any tags/digest referencing the given
// image or any stopped container using the image. If ignoreSoftConflicts is
// true, this function will not check for soft conflict conditions.
func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict {
	// Check if the image has any descendant images.
	if mask&conflictDependentChild != 0 && len(daemon.imageStore.Children(imgID)) > 0 {
		return &imageDeleteConflict{
			hard:    true,
			imgID:   imgID,
			message: "image has dependent child images",
		}
	}

	if mask&conflictRunningContainer != 0 {
		// Check if any running container is using the image.
		running := func(c *container.Container) bool {
			return c.IsRunning() && c.ImageID == imgID
		}
		if container := daemon.containers.First(running); container != nil {
			return &imageDeleteConflict{
				imgID:   imgID,
				hard:    true,
				used:    true,
				message: fmt.Sprintf("image is being used by running container %s", stringid.TruncateID(container.ID)),
			}
		}
	}

	// Check if any repository tags/digest reference this image.
	if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID)) > 0 {
		return &imageDeleteConflict{
			imgID:   imgID,
			message: "image is referenced in multiple repositories",
		}
	}

	if mask&conflictStoppedContainer != 0 {
		// Check if any stopped containers reference this image.
		stopped := func(c *container.Container) bool {
			return !c.IsRunning() && c.ImageID == imgID
		}
		if container := daemon.containers.First(stopped); container != nil {
			return &imageDeleteConflict{
				imgID:   imgID,
				used:    true,
				message: fmt.Sprintf("image is being used by stopped container %s", stringid.TruncateID(container.ID)),
			}
		}
	}

	return nil
}

// imageIsDangling returns whether the given image is "dangling" which means
// that there are no repository references to the given image and it has no
// child images.
func (daemon *Daemon) imageIsDangling(imgID image.ID) bool {
	return !(len(daemon.referenceStore.References(imgID)) > 0 || len(daemon.imageStore.Children(imgID)) > 0)
}
<commit_msg>Fix untag without force while container running<commit_after>package daemon

import (
	"fmt"
	"strings"

	"github.com/docker/docker/container"
	"github.com/docker/docker/errors"
	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/reference"
	"github.com/docker/engine-api/types"
)

type conflictType int

const (
	conflictDependentChild conflictType = (1 << iota)
	conflictRunningContainer
	conflictActiveReference
	conflictStoppedContainer
	conflictHard = conflictDependentChild | conflictRunningContainer
	conflictSoft = conflictActiveReference | conflictStoppedContainer
)

// ImageDelete deletes the image referenced by the given imageRef from this
// daemon. The given imageRef can be an image ID, ID prefix, or a repository
// reference (with an optional tag or digest, defaulting to the tag name
// "latest"). There is differing behavior depending on whether the given
// imageRef is a repository reference or not.
//
// If the given imageRef is a repository reference then that repository
// reference will be removed. However, if there exists any containers which
// were created using the same image reference then the repository reference
// cannot be removed unless either there are other repository references to the
// same image or force is true. Following removal of the repository reference,
// the referenced image itself will attempt to be deleted as described below
// but quietly, meaning any image delete conflicts will cause the image to not
// be deleted and the conflict will not be reported.
//
// There may be conflicts preventing deletion of an image and these conflicts
// are divided into two categories grouped by their severity:
//
// Hard Conflict:
// 	- a pull or build using the image.
// 	- any descendant image.
// 	- any running container using the image.
//
// Soft Conflict:
// 	- any stopped container using the image.
// 	- any repository tag or digest references to the image.
//
// The image cannot be removed if there are any hard conflicts and can be
// removed if there are soft conflicts only if force is true.
//
// If prune is true, ancestor images will each attempt to be deleted quietly,
// meaning any delete conflicts will cause the image to not be deleted and the
// conflict will not be reported.
//
// FIXME: remove ImageDelete's dependency on Daemon, then move to the graph
// package. This would require that we no longer need the daemon to determine
// whether images are being used by a stopped or running container.
func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDelete, error) {
	records := []types.ImageDelete{}

	imgID, err := daemon.GetImageID(imageRef)
	if err != nil {
		return nil, daemon.imageNotExistToErrcode(err)
	}

	repoRefs := daemon.referenceStore.References(imgID)

	var removedRepositoryRef bool
	if !isImageIDPrefix(imgID.String(), imageRef) {
		// A repository reference was given and should be removed
		// first. We can only remove this reference if either force is
		// true, there are multiple repository references to this
		// image, or there are no containers using the given reference.
		if !force && isSingleReference(repoRefs) {
			if container := daemon.getContainerUsingImage(imgID); container != nil {
				// If we removed the repository reference then
				// this image would remain "dangling" and since
				// we really want to avoid that the client must
				// explicitly force its removal.
				err := fmt.Errorf("conflict: unable to remove repository reference %q (must force) - container %s is using its referenced image %s", imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(imgID.String()))
				return nil, errors.NewRequestConflictError(err)
			}
		}

		parsedRef, err := reference.ParseNamed(imageRef)
		if err != nil {
			return nil, err
		}

		parsedRef, err = daemon.removeImageRef(parsedRef)
		if err != nil {
			return nil, err
		}

		untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}

		daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
		records = append(records, untaggedRecord)

		repoRefs = daemon.referenceStore.References(imgID)

		// If a tag reference was removed and the only remaining
		// references to the same repository are digest references,
		// then clean up those digest references.
		if _, isCanonical := parsedRef.(reference.Canonical); !isCanonical {
			foundRepoTagRef := false
			for _, repoRef := range repoRefs {
				if _, repoRefIsCanonical := repoRef.(reference.Canonical); !repoRefIsCanonical && parsedRef.Name() == repoRef.Name() {
					foundRepoTagRef = true
					break
				}
			}
			if !foundRepoTagRef {
				// Remove canonical references from same repository
				remainingRefs := []reference.Named{}
				for _, repoRef := range repoRefs {
					if _, repoRefIsCanonical := repoRef.(reference.Canonical); repoRefIsCanonical && parsedRef.Name() == repoRef.Name() {
						if _, err := daemon.removeImageRef(repoRef); err != nil {
							return records, err
						}

						untaggedRecord := types.ImageDelete{Untagged: repoRef.String()}
						records = append(records, untaggedRecord)
					} else {
						remainingRefs = append(remainingRefs, repoRef)

					}
				}
				repoRefs = remainingRefs
			}
		}

		// If it has remaining references then the untag finished the remove
		if len(repoRefs) > 0 {
			return records, nil
		}

		removedRepositoryRef = true
	} else {
		// If an ID reference was given AND there is at most one tag
		// reference to the image AND all references are within one
		// repository, then remove all references.
		if isSingleReference(repoRefs) {
			c := conflictHard
			if !force {
				c |= conflictSoft &^ conflictActiveReference
			}
			if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil {
				return nil, conflict
			}

			for _, repoRef := range repoRefs {
				parsedRef, err := daemon.removeImageRef(repoRef)
				if err != nil {
					return nil, err
				}

				untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}

				daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
				records = append(records, untaggedRecord)
			}
		}
	}

	return records, daemon.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef)
}

// isSingleReference returns true when all references are from one repository
// and there is at most one tag. Returns false for empty input.
func isSingleReference(repoRefs []reference.Named) bool {
	if len(repoRefs) <= 1 {
		return len(repoRefs) == 1
	}
	var singleRef reference.Named
	canonicalRefs := map[string]struct{}{}
	for _, repoRef := range repoRefs {
		if _, isCanonical := repoRef.(reference.Canonical); isCanonical {
			canonicalRefs[repoRef.Name()] = struct{}{}
		} else if singleRef == nil {
			singleRef = repoRef
		} else {
			return false
		}
	}
	if singleRef == nil {
		// Just use first canonical ref
		singleRef = repoRefs[0]
	}
	_, ok := canonicalRefs[singleRef.Name()]
	return len(canonicalRefs) == 1 && ok
}

// isImageIDPrefix returns whether the given possiblePrefix is a prefix of the
// given imageID.
func isImageIDPrefix(imageID, possiblePrefix string) bool {
	if strings.HasPrefix(imageID, possiblePrefix) {
		return true
	}

	if i := strings.IndexRune(imageID, ':'); i >= 0 {
		return strings.HasPrefix(imageID[i+1:], possiblePrefix)
	}

	return false
}

// getContainerUsingImage returns a container that was created using the given
// imageID. Returns nil if there is no such container.
func (daemon *Daemon) getContainerUsingImage(imageID image.ID) *container.Container {
	return daemon.containers.First(func(c *container.Container) bool {
		return c.ImageID == imageID
	})
}

// removeImageRef attempts to parse and remove the given image reference from
// this daemon's store of repository tag/digest references. The given
// repositoryRef must not be an image ID but a repository name followed by an
// optional tag or digest reference. If tag or digest is omitted, the default
// tag is used. Returns the resolved image reference and an error.
func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, error) {
	ref = reference.WithDefaultTag(ref)
	// Ignore the boolean value returned, as far as we're concerned, this
	// is an idempotent operation and it's okay if the reference didn't
	// exist in the first place.
	_, err := daemon.referenceStore.Delete(ref)

	return ref, err
}

// removeAllReferencesToImageID attempts to remove every reference to the given
// imgID from this daemon's store of repository tag/digest references. Returns
// on the first encountered error. Removed references are logged to this
// daemon's event service. An "Untagged" types.ImageDelete is added to the
// given list of records.
func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDelete) error {
	imageRefs := daemon.referenceStore.References(imgID)

	for _, imageRef := range imageRefs {
		parsedRef, err := daemon.removeImageRef(imageRef)
		if err != nil {
			return err
		}

		untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}

		daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
		*records = append(*records, untaggedRecord)
	}

	return nil
}

// ImageDeleteConflict holds a soft or hard conflict and an associated error.
// Implements the error interface.
type imageDeleteConflict struct {
	hard    bool
	used    bool
	imgID   image.ID
	message string
}

func (idc *imageDeleteConflict) Error() string {
	var forceMsg string
	if idc.hard {
		forceMsg = "cannot be forced"
	} else {
		forceMsg = "must be forced"
	}

	return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID.String()), forceMsg, idc.message)
}

// imageDeleteHelper attempts to delete the given image from this daemon. If
// the image has any hard delete conflicts (child images or running containers
// using the image) then it cannot be deleted. If the image has any soft delete
// conflicts (any tags/digests referencing the image or any stopped container
// using the image) then it can only be deleted if force is true. If the delete
// succeeds and prune is true, the parent images are also deleted if they do
// not have any soft or hard delete conflicts themselves. Any deleted images
// and untagged references are appended to the given records. If any error or
// conflict is encountered, it will be returned immediately without deleting
// the image. If quiet is true, any encountered conflicts will be ignored and
// the function will return nil immediately without deleting the image.
func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDelete, force, prune, quiet bool) error {
	// First, determine if this image has any conflicts. Ignore soft conflicts
	// if force is true.
	c := conflictHard
	if !force {
		c |= conflictSoft
	}
	if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil {
		if quiet && (!daemon.imageIsDangling(imgID) || conflict.used) {
			// Ignore conflicts UNLESS the image is "dangling" or not being used in
			// which case we want the user to know.
			return nil
		}

		// There was a conflict and it's either a hard conflict OR we are not
		// forcing deletion on soft conflicts.
		return conflict
	}

	parent, err := daemon.imageStore.GetParent(imgID)
	if err != nil {
		// There may be no parent
		parent = ""
	}

	// Delete all repository tag/digest references to this image.
	if err := daemon.removeAllReferencesToImageID(imgID, records); err != nil {
		return err
	}

	removedLayers, err := daemon.imageStore.Delete(imgID)
	if err != nil {
		return err
	}

	daemon.LogImageEvent(imgID.String(), imgID.String(), "delete")
	*records = append(*records, types.ImageDelete{Deleted: imgID.String()})
	for _, removedLayer := range removedLayers {
		*records = append(*records, types.ImageDelete{Deleted: removedLayer.ChainID.String()})
	}

	if !prune || parent == "" {
		return nil
	}

	// We need to prune the parent image. This means delete it if there are
	// no tags/digests referencing it and there are no containers using it (
	// either running or stopped).
	// Do not force prunings, but do so quietly (stopping on any encountered
	// conflicts).
	return daemon.imageDeleteHelper(parent, records, false, true, true)
}

// checkImageDeleteConflict determines whether there are any conflicts
// preventing deletion of the given image from this daemon. A hard conflict is
// any image which has the given image as a parent or any running container
// using the image. A soft conflict is any tags/digest referencing the given
// image or any stopped container using the image. If ignoreSoftConflicts is
// true, this function will not check for soft conflict conditions.
func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict {
	// Check if the image has any descendant images.
	if mask&conflictDependentChild != 0 && len(daemon.imageStore.Children(imgID)) > 0 {
		return &imageDeleteConflict{
			hard:    true,
			imgID:   imgID,
			message: "image has dependent child images",
		}
	}

	if mask&conflictRunningContainer != 0 {
		// Check if any running container is using the image.
		running := func(c *container.Container) bool {
			return c.IsRunning() && c.ImageID == imgID
		}
		if container := daemon.containers.First(running); container != nil {
			return &imageDeleteConflict{
				imgID:   imgID,
				hard:    true,
				used:    true,
				message: fmt.Sprintf("image is being used by running container %s", stringid.TruncateID(container.ID)),
			}
		}
	}

	// Check if any repository tags/digest reference this image.
	if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID)) > 0 {
		return &imageDeleteConflict{
			imgID:   imgID,
			message: "image is referenced in multiple repositories",
		}
	}

	if mask&conflictStoppedContainer != 0 {
		// Check if any stopped containers reference this image.
		stopped := func(c *container.Container) bool {
			return !c.IsRunning() && c.ImageID == imgID
		}
		if container := daemon.containers.First(stopped); container != nil {
			return &imageDeleteConflict{
				imgID:   imgID,
				used:    true,
				message: fmt.Sprintf("image is being used by stopped container %s", stringid.TruncateID(container.ID)),
			}
		}
	}

	return nil
}

// imageIsDangling returns whether the given image is "dangling" which means
// that there are no repository references to the given image and it has no
// child images.
func (daemon *Daemon) imageIsDangling(imgID image.ID) bool {
	return !(len(daemon.referenceStore.References(imgID)) > 0 || len(daemon.imageStore.Children(imgID)) > 0)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hex

import (
	"bytes"
	"testing"
)

type encDecTest struct {
	enc string
	dec []byte
}

var encDecTests = []encDecTest{
	{"", []byte{}},
	{"0001020304050607", []byte{0, 1, 2, 3, 4, 5, 6, 7}},
	{"08090a0b0c0d0e0f", []byte{8, 9, 10, 11, 12, 13, 14, 15}},
	{"f0f1f2f3f4f5f6f7", []byte{0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7}},
	{"f8f9fafbfcfdfeff", []byte{0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff}},
	{"67", []byte{'g'}},
	{"e3a1", []byte{0xe3, 0xa1}},
}

func TestEncode(t *testing.T) {
	for i, test := range encDecTests {
		dst := make([]byte, EncodedLen(len(test.dec)))
		n := Encode(dst, test.dec)
		if n != len(dst) {
			t.Errorf("#%d: bad return value: got: %d want: %d", i, n, len(dst))
		}
		if string(dst) != test.enc {
			t.Errorf("#%d: got: %#v want: %#v", i, dst, test.enc)
		}
	}
}

func TestDecode(t *testing.T) {
	for i, test := range encDecTests {
		dst := make([]byte, DecodedLen(len(test.enc)))
		n, err := Decode(dst, []byte(test.enc))
		if err != nil {
			t.Errorf("#%d: bad return value: got:%d want:%d", i, n, len(dst))
		} else if !bytes.Equal(dst, test.dec) {
			t.Errorf("#%d: got: %#v want: %#v", i, dst, test.dec)
		}
	}
}

func TestEncodeToString(t *testing.T) {
	for i, test := range encDecTests {
		s := EncodeToString(test.dec)
		if s != test.enc {
			t.Errorf("#%d got:%s want:%s", i, s, test.enc)
		}
	}
}

func TestDecodeString(t *testing.T) {
	for i, test := range encDecTests {
		dst, err := DecodeString(test.enc)
		if err != nil {
			t.Errorf("#%d: unexpected err value: %s", i, err)
			continue
		}
		if bytes.Compare(dst, test.dec) != 0 {
			t.Errorf("#%d: got: %#v want: #%v", i, dst, test.dec)
		}
	}
}

type errTest struct {
	in  string
	err string
}

var errTests = []errTest{
	{"0", "encoding/hex: odd length hex string"},
	{"0g", "encoding/hex: invalid byte: U+0067 'g'"},
	{"0\x01", "encoding/hex: invalid byte: U+0001"},
}

func TestInvalidErr(t *testing.T) {
	for i, test := range errTests {
		dst := make([]byte, DecodedLen(len(test.in)))
		_, err := Decode(dst, []byte(test.in))
		if err == nil {
			t.Errorf("#%d: expected error; got none")
		} else if err.Error() != test.err {
			t.Errorf("#%d: got: %v want: %v", i, err, test.err)
		}
	}
}

func TestInvalidStringErr(t *testing.T) {
	for i, test := range errTests {
		_, err := DecodeString(test.in)
		if err == nil {
			t.Errorf("#%d: expected error; got none")
		} else if err.Error() != test.err {
			t.Errorf("#%d: got: %v want: %v", i, err, test.err)
		}
	}
}

func TestDumper(t *testing.T) {
	var in [40]byte
	for i := range in {
		in[i] = byte(i + 30)
	}

	for stride := 1; stride < len(in); stride++ {
		var out bytes.Buffer
		dumper := Dumper(&out)
		done := 0
		for done < len(in) {
			todo := done + stride
			if todo > len(in) {
				todo = len(in)
			}
			dumper.Write(in[done:todo])
			done = todo
		}

		dumper.Close()
		if !bytes.Equal(out.Bytes(), expectedHexDump) {
			t.Errorf("stride: %d failed. got:\n%s\nwant:\n%s", stride, out.Bytes(), expectedHexDump)
		}
	}
}

func TestDump(t *testing.T) {
	var in [40]byte
	for i := range in {
		in[i] = byte(i + 30)
	}

	out := []byte(Dump(in[:]))
	if !bytes.Equal(out, expectedHexDump) {
		t.Errorf("got:\n%s\nwant:\n%s", out, expectedHexDump)
	}
}

var expectedHexDump = []byte(`00000000  1e 1f 20 21 22 23 24 25  26 27 28 29 2a 2b 2c 2d  |.. !"#$%&'()*+,-|
00000010  2e 2f 30 31 32 33 34 35  36 37 38 39 3a 3b 3c 3d  |./0123456789:;<=|
00000020  3e 3f 40 41 42 43 44 45                           |>?@ABCDE|
`)
<commit_msg>encoding/hex: vet the test prints<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hex

import (
	"bytes"
	"testing"
)

type encDecTest struct {
	enc string
	dec []byte
}

var encDecTests = []encDecTest{
	{"", []byte{}},
	{"0001020304050607", []byte{0, 1, 2, 3, 4, 5, 6, 7}},
	{"08090a0b0c0d0e0f", []byte{8, 9, 10, 11, 12, 13, 14, 15}},
	{"f0f1f2f3f4f5f6f7", []byte{0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7}},
	{"f8f9fafbfcfdfeff", []byte{0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff}},
	{"67", []byte{'g'}},
	{"e3a1", []byte{0xe3, 0xa1}},
}

func TestEncode(t *testing.T) {
	for i, test := range encDecTests {
		dst := make([]byte, EncodedLen(len(test.dec)))
		n := Encode(dst, test.dec)
		if n != len(dst) {
			t.Errorf("#%d: bad return value: got: %d want: %d", i, n, len(dst))
		}
		if string(dst) != test.enc {
			t.Errorf("#%d: got: %#v want: %#v", i, dst, test.enc)
		}
	}
}

func TestDecode(t *testing.T) {
	for i, test := range encDecTests {
		dst := make([]byte, DecodedLen(len(test.enc)))
		n, err := Decode(dst, []byte(test.enc))
		if err != nil {
			t.Errorf("#%d: bad return value: got:%d want:%d", i, n, len(dst))
		} else if !bytes.Equal(dst, test.dec) {
			t.Errorf("#%d: got: %#v want: %#v", i, dst, test.dec)
		}
	}
}

func TestEncodeToString(t *testing.T) {
	for i, test := range encDecTests {
		s := EncodeToString(test.dec)
		if s != test.enc {
			t.Errorf("#%d got:%s want:%s", i, s, test.enc)
		}
	}
}

func TestDecodeString(t *testing.T) {
	for i, test := range encDecTests {
		dst, err := DecodeString(test.enc)
		if err != nil {
			t.Errorf("#%d: unexpected err value: %s", i, err)
			continue
		}
		if bytes.Compare(dst, test.dec) != 0 {
			t.Errorf("#%d: got: %#v want: #%v", i, dst, test.dec)
		}
	}
}

type errTest struct {
	in  string
	err string
}

var errTests = []errTest{
	{"0", "encoding/hex: odd length hex string"},
	{"0g", "encoding/hex: invalid byte: U+0067 'g'"},
	{"0\x01", "encoding/hex: invalid byte: U+0001"},
}

func TestInvalidErr(t *testing.T) {
	for i, test := range errTests {
		dst := make([]byte, DecodedLen(len(test.in)))
		_, err := Decode(dst, []byte(test.in))
		if err == nil {
			t.Errorf("#%d: expected error; got none", i)
		} else if err.Error() != test.err {
			t.Errorf("#%d: got: %v want: %v", i, err, test.err)
		}
	}
}

func TestInvalidStringErr(t *testing.T) {
	for i, test := range errTests {
		_, err := DecodeString(test.in)
		if err == nil {
			t.Errorf("#%d: expected error; got none", i)
		} else if err.Error() != test.err {
			t.Errorf("#%d: got: %v want: %v", i, err, test.err)
		}
	}
}

func TestDumper(t *testing.T) {
	var in [40]byte
	for i := range in {
		in[i] = byte(i + 30)
	}

	for stride := 1; stride < len(in); stride++ {
		var out bytes.Buffer
		dumper := Dumper(&out)
		done := 0
		for done < len(in) {
			todo := done + stride
			if todo > len(in) {
				todo = len(in)
			}
			dumper.Write(in[done:todo])
			done = todo
		}

		dumper.Close()
		if !bytes.Equal(out.Bytes(), expectedHexDump) {
			t.Errorf("stride: %d failed. got:\n%s\nwant:\n%s", stride, out.Bytes(), expectedHexDump)
		}
	}
}

func TestDump(t *testing.T) {
	var in [40]byte
	for i := range in {
		in[i] = byte(i + 30)
	}

	out := []byte(Dump(in[:]))
	if !bytes.Equal(out, expectedHexDump) {
		t.Errorf("got:\n%s\nwant:\n%s", out, expectedHexDump)
	}
}

var expectedHexDump = []byte(`00000000  1e 1f 20 21 22 23 24 25  26 27 28 29 2a 2b 2c 2d  |.. !"#$%&'()*+,-|
00000010  2e 2f 30 31 32 33 34 35  36 37 38 39 3a 3b 3c 3d  |./0123456789:;<=|
00000020  3e 3f 40 41 42 43 44 45                           |>?@ABCDE|
`)
<|endoftext|>


<commit_before>package main

import (
	"graph"
	"container/list"
	"fmt"
)

func StepToPolyline (steps []graph.Step,u ,v graph.Node) (Polyline){
	polyline := make([]Point,len(steps)+2)
	first:=make([]float64,2)
	lat1,long1 := u.LatLng()
	first[0]=lat1
	first[1]=long1
	polyline[0]=first
	for i,s:=range steps {
		point:= make([]float64,2)
		point[0]=s.Lat
		point[1]=s.Lng
		polyline[i+1]=point
	}
	last:=make([]float64,2)
	lat2,long2 := v.LatLng()
	last[0]=lat2
	last[1]=long2
	polyline[len(steps)+1]=last
	return polyline
}

func WayToStep(w graph.Way,u,v graph.Node) (Step){
	dist := Distance{"TODO",int(w.Length)}
	dur := Duration{"TODO",42}
	start:=make([]float64,2)
	lat1,long1:=u.LatLng()
	start[0]=lat1
	start[1]=long1
	end :=make([]float64,2)
	lat2,long2:=v.LatLng()
	end[0]=lat2
	end[1]=long2
	poly:=StepToPolyline(w.Steps,u,v)
	instruction:="TODO"
	return Step{dist,dur,start,end,poly,instruction}
}

func EdgeToStep (e graph.Edge,u,v graph.Node) (Step){
	dist := Distance{"TODO",int(e.Length())}
	dur := Duration{"TODO",42}
	start:=make([]float64,2)
	lat1,long1:=u.LatLng()
	start[0]=lat1
	start[1]=long1
	end :=make([]float64,2)
	lat2,long2:=v.LatLng()
	end[0]=lat2
	end[1]=long2
	poly:=StepToPolyline(e.Steps(),u,v)
	instruction:= "TODO"
	return Step{dist,dur,start,end,poly,instruction}
}

func PathToLeg (dist float64, vertex, edge *list.List,startway,endway graph.Way) (Leg) {
	distance := Distance{"TODO",int(dist)}
	dur := Duration{"TODO",42}
	steps:=make([]Step,edge.Len()+2)
	startvertex := vertex.Front().Value.(graph.Node)
	steps[0]=WayToStep(startway,startway.Node,startvertex)
	for v,e,i:=vertex.Front(),edge.Front(),1;e!=edge.Back();v,e,i=v.Next(),e.Next(),i+1 {
		fmt.Printf("v: %v\n", v)
		fmt.Printf("e: %v\n", e)
		fmt.Printf("i: %v\n", i)
		ue:=e.Value.(graph.Edge)
		uv:=v.Value.(graph.Node)
		nuv:=v.Next().Value.(graph.Node)
		steps[i]=EdgeToStep(ue,uv,nuv)
	}
	steps[len(steps)-1]=WayToStep(endway,vertex.Back().Value.(graph.Node),endway.Node)
	return Leg{distance,dur,steps[0].StartLocation,steps[len(steps)-1].EndLocation,steps}
}<commit_msg>not a fix<commit_after>package main

import (
	"graph"
	"container/list"
	"fmt"
)

func StepToPolyline (steps []graph.Step, u, v graph.Node) Polyline {
	polyline := make([]Point,len(steps)+2)
	first:=make([]float64,2)
	lat1,long1 := u.LatLng()
	first[0]=lat1
	first[1]=long1
	polyline[0]=first
	for i,s:=range steps {
		point:= make([]float64,2)
		point[0]=s.Lat
		point[1]=s.Lng
		polyline[i+1]=point
	}
	last:=make([]float64,2)
	lat2,long2 := v.LatLng()
	last[0]=lat2
	last[1]=long2
	polyline[len(steps)+1]=last
	return polyline
}

func WayToStep(w graph.Way,u,v graph.Node) (Step){
	dist := Distance{"TODO",int(w.Length)}
	dur := Duration{"TODO",42}
	start:=make([]float64,2)
	lat1,long1:=u.LatLng()
	start[0]=lat1
	start[1]=long1
	end :=make([]float64,2)
	lat2,long2:=v.LatLng()
	end[0]=lat2
	end[1]=long2
	poly:=StepToPolyline(w.Steps,u,v)
	instruction:="TODO"
	return Step{dist,dur,start,end,poly,instruction}
}

func EdgeToStep (e graph.Edge,u,v graph.Node) (Step){
	dist := Distance{fmt.Sprintf("%.2f m", e.Length()),int(e.Length())}
	dur := Duration{"Time",42}
	start:=make([]float64,2)
	lat1,long1:=u.LatLng()
	start[0]=lat1
	start[1]=long1
	end :=make([]float64,2)
	lat2,long2:=v.LatLng()
	end[0]=lat2
	end[1]=long2
	poly:=StepToPolyline(e.Steps(),u,v)
	instruction:= fmt.Sprintf("(%.4f, %.4f) -> (%.4f, %.4f)", lat1, long1, lat2, long2)
	return Step{dist,dur,start,end,poly,instruction}
}

func PathToLeg (dist float64, vertex, edge *list.List,startway,endway graph.Way) (Leg) {
	distance := Distance{"TODO",int(dist)}
	dur := Duration{"TODO",42}
	steps:=make([]Step,edge.Len()+2)
	startvertex := vertex.Front().Value.(graph.Node)
	steps[0]=WayToStep(startway,startway.Node,startvertex)
	for v,e,i:=vertex.Front().Next(),edge.Front().Next(),1;e!=edge.Back();v,e,i=v.Next(),e.Next(),i+1 {
		fmt.Printf("v: %v\n", v)
		fmt.Printf("e: %v\n", e)
		fmt.Printf("i: %v\n", i)
		ue:=e.Value.(graph.Edge)
		uv:=v.Value.(graph.Node)
		nuv:=v.Next().Value.(graph.Node)
		steps[i]=EdgeToStep(ue,uv,nuv)
	}
	steps[len(steps)-1]=WayToStep(endway,vertex.Back().Value.(graph.Node),endway.Node)
	return Leg{distance,dur,steps[0].StartLocation,steps[len(steps)-1].EndLocation,steps}
}<|endoftext|>