<commit_before>package parser

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/prometheus/client_golang/prometheus"
)

// TODO(JosephMarques) See if there is a better way of determining
// where to send the request (there almost certainly is)
var BaseURL = "https://annotator-dot-" +
	os.Getenv("GCLOUD_PROJECT") +
	".appspot.com/annotate?"

func AddMetaDataPTConnSpec(spec *schema.MLabConnectionSpecification, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if spec.Server_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Server_geolocation, spec.Server_ip, timestamp)
	}
	if spec.Client_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Client_geolocation, spec.Client_ip, timestamp)
	}
}

func AddMetaDataPTHop(hop *schema.ParisTracerouteHop, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT-HOP"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if hop.Src_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Src_geolocation, hop.Src_ip, timestamp)
	}
	if hop.Dest_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Dest_geolocation, hop.Dest_ip, timestamp)
	}
}

func GetAndInsertGeolocationIPStruct(geo *schema.GeolocationIP, ip string, timestamp time.Time) {
	url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
		"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
	annotationData := GetMetaData(url)
	if annotationData != nil && annotationData.Geo != nil {
		*geo = *annotationData.Geo
	}
}

// AddMetaDataNDTConnSpec takes a connection spec and a timestamp and
// annotates the connection spec with metadata associated with each IP
// Address. It will either sucessfully add the metadata or fail
// silently and make no changes.
func AddMetaDataNDTConnSpec(spec schema.Web100ValueMap, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "NDT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)

	GetAndInsertMetaIntoNDTConnSpec("client", spec, timestamp)
	GetAndInsertMetaIntoNDTConnSpec("server", spec, timestamp)
}

// GetAndInsertNDT takes a timestamp, an NDT connection spec, and a
// string indicating whether it should get the metadata for the client
// end or the server end of the connection. It will either insert the
// data into the connection spec or silently fail.
func GetAndInsertMetaIntoNDTConnSpec(side string, spec schema.Web100ValueMap, timestamp time.Time) {
	ip, ok := spec.GetString([]string{side + "_ip"})
	if ok {
		url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
			"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
		annotationData := GetMetaData(url)
		if annotationData != nil && annotationData.Geo != nil {
			CopyStructToMap(annotationData.Geo, spec.Get(side+"_geolocation"))
		} else {
			metrics.AnnotationErrorCount.With(prometheus.
				Labels{"source": "Couldn't get metadata for the " + side + " side."}).Inc()
		}
	}
}

// CopyStructToMap takes a POINTER to an arbitrary struct and copies
// it's fields into a value map. It will also make fields entirely
// lower case, for convienece when working with exported structs. Also,
// NEVER pass in something that is not a pointer to a struct, as this
// will cause a panic.
func CopyStructToMap(sourceStruct interface{}, destinationMap map[string]bigquery.Value) {
	structToCopy := reflect.ValueOf(sourceStruct).Elem()
	typeOfStruct := structToCopy.Type()
	for i := 0; i < typeOfStruct.NumField(); i++ {
		destinationMap[strings.ToLower(typeOfStruct.Field(i).Name)] =
			structToCopy.Field(i).Interface()
	}

}

// GetMetaData combines the functionality of QueryAnnotationService and
// ParseJSONMetaDataResponse to query the annotator service and return
// the corresponding MetaData if it can, or a nil pointer if it
// encounters any error and cannot get the data for any reason
func GetMetaData(url string) *schema.MetaData {
	// Query the service and grab the response safely
	annotatorResponse, err := QueryAnnotationService(url)
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Error querying annotation service"}).Inc()
		log.Println(err)
		return nil
	}

	// Safely parse the JSON response and pass it back to the caller
	metaDataFromResponse, err := ParseJSONMetaDataResponse(annotatorResponse)
	if err != nil {
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Failed to parse JSON"}).Inc()
		log.Println(err)
		return nil
	}
	return metaDataFromResponse
}

// QueryAnnotationService will connect to the annotation service and
// copy the body of a valid response to a byte slice and return it to a
// user, returning an error if any occurs
func QueryAnnotationService(url string) ([]byte, error) {
	// Make the actual request
	resp, err := http.Get(url)

	// Catch http errors
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Request to Annotator failed"}).Inc()
		return nil, err
	}

	// Catch errors reported by the service
	if resp.StatusCode != http.StatusOK {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Webserver gave non-ok response"}).Inc()
		return nil, errors.New("URL:" + url + " gave response code " + resp.Status)
	}
	defer resp.Body.Close()

	// Copy response into a byte slice
	return ioutil.ReadAll(resp.Body)
}

// ParseJSONMetaDataResponse takes a byte slice containing the test of
// the JSON from the annotator service and parses it into a MetaData
// struct, for easy manipulation. It returns a pointer to the struct on
// success and an error if an error occurs.
func ParseJSONMetaDataResponse(jsonBuffer []byte) (*schema.MetaData, error) {
	parsedJSON := &schema.MetaData{}
	err := json.Unmarshal(jsonBuffer, parsedJSON)
	if err != nil {
		return nil, err
	}
	return parsedJSON, nil
}
<commit_msg>Added documentation comments.<commit_after>package parser

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/prometheus/client_golang/prometheus"
)

// TODO(JosephMarques) See if there is a better way of determining
// where to send the request (there almost certainly is)
var BaseURL = "https://annotator-dot-" +
	os.Getenv("GCLOUD_PROJECT") +
	".appspot.com/annotate?"

// AddMetaDataPTConnSpec takes a pointer to a
// MLabConnectionSpecification struct and a timestamp. With these, it
// will fetch the appropriate metadata and add it to the hop struct
// referenced by the pointer.
func AddMetaDataPTConnSpec(spec *schema.MLabConnectionSpecification, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if spec.Server_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Server_geolocation, spec.Server_ip, timestamp)
	}
	if spec.Client_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Client_geolocation, spec.Client_ip, timestamp)
	}
}

// AddMetaDataPTHop takes a pointer to a ParisTracerouteHop and a
// timestamp. With these, it will fetch the appropriate metadata and
// add it to the hop struct referenced by the pointer.
func AddMetaDataPTHop(hop *schema.ParisTracerouteHop, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT-HOP"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if hop.Src_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Src_geolocation, hop.Src_ip, timestamp)
	}
	if hop.Dest_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Dest_geolocation, hop.Dest_ip, timestamp)
	}
}

// GetAndInsertGeolocationIPStruct takes a NON-NIL pointer to a
// pre-allocated schema.GeolocationIP struct, an IP address, and a
// timestamp. It will connect to the annotation service, get the
// metadata, and insert the metadata into the reigion pointed to by
// the schema.GeolocationIP pointer.
func GetAndInsertGeolocationIPStruct(geo *schema.GeolocationIP, ip string, timestamp time.Time) {
	url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
		"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
	annotationData := GetMetaData(url)
	if annotationData != nil && annotationData.Geo != nil {
		*geo = *annotationData.Geo
	}
}

// AddMetaDataNDTConnSpec takes a connection spec and a timestamp and
// annotates the connection spec with metadata associated with each IP
// Address. It will either sucessfully add the metadata or fail
// silently and make no changes.
func AddMetaDataNDTConnSpec(spec schema.Web100ValueMap, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "NDT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)

	GetAndInsertMetaIntoNDTConnSpec("client", spec, timestamp)
	GetAndInsertMetaIntoNDTConnSpec("server", spec, timestamp)
}

// GetAndInsertNDT takes a timestamp, an NDT connection spec, and a
// string indicating whether it should get the metadata for the client
// end or the server end of the connection. It will either insert the
// data into the connection spec or silently fail.
func GetAndInsertMetaIntoNDTConnSpec(side string, spec schema.Web100ValueMap, timestamp time.Time) {
	ip, ok := spec.GetString([]string{side + "_ip"})
	if ok {
		url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
			"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
		annotationData := GetMetaData(url)
		if annotationData != nil && annotationData.Geo != nil {
			CopyStructToMap(annotationData.Geo, spec.Get(side+"_geolocation"))
		} else {
			metrics.AnnotationErrorCount.With(prometheus.
				Labels{"source": "Couldn't get metadata for the " + side + " side."}).Inc()
		}
	}
}

// CopyStructToMap takes a POINTER to an arbitrary struct and copies
// it's fields into a value map. It will also make fields entirely
// lower case, for convienece when working with exported structs. Also,
// NEVER pass in something that is not a pointer to a struct, as this
// will cause a panic.
func CopyStructToMap(sourceStruct interface{}, destinationMap map[string]bigquery.Value) {
	structToCopy := reflect.ValueOf(sourceStruct).Elem()
	typeOfStruct := structToCopy.Type()
	for i := 0; i < typeOfStruct.NumField(); i++ {
		destinationMap[strings.ToLower(typeOfStruct.Field(i).Name)] =
			structToCopy.Field(i).Interface()
	}

}

// GetMetaData combines the functionality of QueryAnnotationService and
// ParseJSONMetaDataResponse to query the annotator service and return
// the corresponding MetaData if it can, or a nil pointer if it
// encounters any error and cannot get the data for any reason
func GetMetaData(url string) *schema.MetaData {
	// Query the service and grab the response safely
	annotatorResponse, err := QueryAnnotationService(url)
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Error querying annotation service"}).Inc()
		log.Println(err)
		return nil
	}

	// Safely parse the JSON response and pass it back to the caller
	metaDataFromResponse, err := ParseJSONMetaDataResponse(annotatorResponse)
	if err != nil {
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Failed to parse JSON"}).Inc()
		log.Println(err)
		return nil
	}
	return metaDataFromResponse
}

// QueryAnnotationService will connect to the annotation service and
// copy the body of a valid response to a byte slice and return it to a
// user, returning an error if any occurs
func QueryAnnotationService(url string) ([]byte, error) {
	// Make the actual request
	resp, err := http.Get(url)

	// Catch http errors
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Request to Annotator failed"}).Inc()
		return nil, err
	}

	// Catch errors reported by the service
	if resp.StatusCode != http.StatusOK {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Webserver gave non-ok response"}).Inc()
		return nil, errors.New("URL:" + url + " gave response code " + resp.Status)
	}
	defer resp.Body.Close()

	// Copy response into a byte slice
	return ioutil.ReadAll(resp.Body)
}

// ParseJSONMetaDataResponse takes a byte slice containing the test of
// the JSON from the annotator service and parses it into a MetaData
// struct, for easy manipulation. It returns a pointer to the struct on
// success and an error if an error occurs.
func ParseJSONMetaDataResponse(jsonBuffer []byte) (*schema.MetaData, error) {
	parsedJSON := &schema.MetaData{}
	err := json.Unmarshal(jsonBuffer, parsedJSON)
	if err != nil {
		return nil, err
	}
	return parsedJSON, nil
}
<|endoftext|>


<commit_before>package bitswap

import (
	"context"
	"sync"
	"time"

	engine "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
	wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"

	metrics "gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface"
	peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer"
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
)

type WantManager struct {
	// sync channels for Run loop
	incoming     chan *wantSet
	connectEvent chan peerStatus     // notification channel for peers connecting/disconnecting
	peerReqs     chan chan []peer.ID // channel to request connected peers on

	// synchronized by Run loop, only touch inside there
	peers map[peer.ID]*msgQueue
	wl    *wantlist.ThreadSafe
	bcwl  *wantlist.ThreadSafe

	network bsnet.BitSwapNetwork
	ctx     context.Context
	cancel  func()

	wantlistGauge metrics.Gauge
	sentHistogram metrics.Histogram
}

type peerStatus struct {
	connect bool
	peer    peer.ID
}

func NewWantManager(ctx context.Context, network bsnet.BitSwapNetwork) *WantManager {
	ctx, cancel := context.WithCancel(ctx)
	wantlistGauge := metrics.NewCtx(ctx, "wantlist_total",
		"Number of items in wantlist.").Gauge()
	sentHistogram := metrics.NewCtx(ctx, "sent_all_blocks_bytes", "Histogram of blocks sent by"+
		" this bitswap").Histogram(metricsBuckets)
	return &WantManager{
		incoming:      make(chan *wantSet, 10),
		connectEvent:  make(chan peerStatus, 10),
		peerReqs:      make(chan chan []peer.ID),
		peers:         make(map[peer.ID]*msgQueue),
		wl:            wantlist.NewThreadSafe(),
		bcwl:          wantlist.NewThreadSafe(),
		network:       network,
		ctx:           ctx,
		cancel:        cancel,
		wantlistGauge: wantlistGauge,
		sentHistogram: sentHistogram,
	}
}

type msgQueue struct {
	p peer.ID

	outlk   sync.Mutex
	out     bsmsg.BitSwapMessage
	network bsnet.BitSwapNetwork
	wl      *wantlist.ThreadSafe

	sender bsnet.MessageSender

	refcnt int

	work chan struct{}
	done chan struct{}
}

// WantBlocks adds the given cids to the wantlist, tracked by the given session
func (pm *WantManager) WantBlocks(ctx context.Context, ks []*cid.Cid, peers []peer.ID, ses uint64) {
	log.Infof("want blocks: %s", ks)
	pm.addEntries(ctx, ks, peers, false, ses)
}

// CancelWants removes the given cids from the wantlist, tracked by the given session
func (pm *WantManager) CancelWants(ctx context.Context, ks []*cid.Cid, peers []peer.ID, ses uint64) {
	pm.addEntries(context.Background(), ks, peers, true, ses)
}

type wantSet struct {
	entries []*bsmsg.Entry
	targets []peer.ID
	from    uint64
}

func (pm *WantManager) addEntries(ctx context.Context, ks []*cid.Cid, targets []peer.ID, cancel bool, ses uint64) {
	entries := make([]*bsmsg.Entry, 0, len(ks))
	for i, k := range ks {
		entries = append(entries, &bsmsg.Entry{
			Cancel: cancel,
			Entry:  wantlist.NewRefEntry(k, kMaxPriority-i),
		})
	}
	select {
	case pm.incoming <- &wantSet{entries: entries, targets: targets, from: ses}:
	case <-pm.ctx.Done():
	case <-ctx.Done():
	}
}

func (pm *WantManager) ConnectedPeers() []peer.ID {
	resp := make(chan []peer.ID)
	pm.peerReqs <- resp
	return <-resp
}

func (pm *WantManager) SendBlock(ctx context.Context, env *engine.Envelope) {
	// Blocks need to be sent synchronously to maintain proper backpressure
	// throughout the network stack
	defer env.Sent()

	pm.sentHistogram.Observe(float64(len(env.Block.RawData())))

	msg := bsmsg.New(false)
	msg.AddBlock(env.Block)
	log.Infof("Sending block %s to %s", env.Block, env.Peer)
	err := pm.network.SendMessage(ctx, env.Peer, msg)
	if err != nil {
		log.Infof("sendblock error: %s", err)
	}
}

func (pm *WantManager) startPeerHandler(p peer.ID) *msgQueue {
	mq, ok := pm.peers[p]
	if ok {
		mq.refcnt++
		return nil
	}

	mq = pm.newMsgQueue(p)

	// new peer, we will want to give them our full wantlist
	fullwantlist := bsmsg.New(true)
	for _, e := range pm.bcwl.Entries() {
		for k := range e.SesTrk {
			mq.wl.AddEntry(e, k)
		}
		fullwantlist.AddEntry(e.Cid, e.Priority)
	}
	mq.out = fullwantlist
	mq.work <- struct{}{}

	pm.peers[p] = mq
	go mq.runQueue(pm.ctx)
	return mq
}

func (pm *WantManager) stopPeerHandler(p peer.ID) {
	pq, ok := pm.peers[p]
	if !ok {
		// TODO: log error?
		return
	}

	pq.refcnt--
	if pq.refcnt > 0 {
		return
	}

	close(pq.done)
	delete(pm.peers, p)
}

func (mq *msgQueue) runQueue(ctx context.Context) {
	for {
		select {
		case <-mq.work: // there is work to be done
			mq.doWork(ctx)
		case <-mq.done:
			if mq.sender != nil {
				mq.sender.Close()
			}
			return
		case <-ctx.Done():
			if mq.sender != nil {
				mq.sender.Reset()
			}
			return
		}
	}
}

func (mq *msgQueue) doWork(ctx context.Context) {
	// grab outgoing message
	mq.outlk.Lock()
	wlm := mq.out
	if wlm == nil || wlm.Empty() {
		mq.outlk.Unlock()
		return
	}
	mq.out = nil
	mq.outlk.Unlock()

	// NB: only open a stream if we actually have data to send
	if mq.sender == nil {
		err := mq.openSender(ctx)
		if err != nil {
			log.Infof("cant open message sender to peer %s: %s", mq.p, err)
			// TODO: cant connect, what now?
			return
		}
	}

	// send wantlist updates
	for { // try to send this message until we fail.
		err := mq.sender.SendMsg(ctx, wlm)
		if err == nil {
			return
		}

		log.Infof("bitswap send error: %s", err)
		mq.sender.Reset()
		mq.sender = nil

		select {
		case <-mq.done:
			return
		case <-ctx.Done():
			return
		case <-time.After(time.Millisecond * 100):
			// wait 100ms in case disconnect notifications are still propogating
			log.Warning("SendMsg errored but neither 'done' nor context.Done() were set")
		}

		err = mq.openSender(ctx)
		if err != nil {
			log.Infof("couldnt open sender again after SendMsg(%s) failed: %s", mq.p, err)
			// TODO(why): what do we do now?
			// I think the *right* answer is to probably put the message we're
			// trying to send back, and then return to waiting for new work or
			// a disconnect.
			return
		}

		// TODO: Is this the same instance for the remote peer?
		// If its not, we should resend our entire wantlist to them
		/*
			if mq.sender.InstanceID() != mq.lastSeenInstanceID {
				wlm = mq.getFullWantlistMessage()
			}
		*/
	}
}

func (mq *msgQueue) openSender(ctx context.Context) error {
	// allow ten minutes for connections this includes looking them up in the
	// dht dialing them, and handshaking
	conctx, cancel := context.WithTimeout(ctx, time.Minute*10)
	defer cancel()

	err := mq.network.ConnectTo(conctx, mq.p)
	if err != nil {
		return err
	}

	nsender, err := mq.network.NewMessageSender(ctx, mq.p)
	if err != nil {
		return err
	}

	mq.sender = nsender
	return nil
}

func (pm *WantManager) Connected(p peer.ID) {
	select {
	case pm.connectEvent <- peerStatus{peer: p, connect: true}:
	case <-pm.ctx.Done():
	}
}

func (pm *WantManager) Disconnected(p peer.ID) {
	select {
	case pm.connectEvent <- peerStatus{peer: p, connect: false}:
	case <-pm.ctx.Done():
	}
}

// TODO: use goprocess here once i trust it
func (pm *WantManager) Run() {
	// NOTE: Do not open any streams or connections from anywhere in this
	// event loop. Really, just don't do anything likely to block.
	for {
		select {
		case ws := <-pm.incoming:

			// is this a broadcast or not?
			brdc := len(ws.targets) == 0

			// add changes to our wantlist
			for _, e := range ws.entries {
				if e.Cancel {
					if brdc {
						pm.bcwl.Remove(e.Cid, ws.from)
					}

					if pm.wl.Remove(e.Cid, ws.from) {
						pm.wantlistGauge.Dec()
					}
				} else {
					if brdc {
						pm.bcwl.AddEntry(e.Entry, ws.from)
					}
					if pm.wl.AddEntry(e.Entry, ws.from) {
						pm.wantlistGauge.Inc()
					}
				}
			}

			// broadcast those wantlist changes
			if len(ws.targets) == 0 {
				for _, p := range pm.peers {
					p.addMessage(ws.entries, ws.from)
				}
			} else {
				for _, t := range ws.targets {
					p, ok := pm.peers[t]
					if !ok {
						log.Warning("tried sending wantlist change to non-partner peer")
						continue
					}
					p.addMessage(ws.entries, ws.from)
				}
			}

		case p := <-pm.connectEvent:
			if p.connect {
				pm.startPeerHandler(p.peer)
			} else {
				pm.stopPeerHandler(p.peer)
			}
		case req := <-pm.peerReqs:
			peers := make([]peer.ID, 0, len(pm.peers))
			for p := range pm.peers {
				peers = append(peers, p)
			}
			req <- peers
		case <-pm.ctx.Done():
			return
		}
	}
}

func (wm *WantManager) newMsgQueue(p peer.ID) *msgQueue {
	return &msgQueue{
		done:    make(chan struct{}),
		work:    make(chan struct{}, 1),
		wl:      wantlist.NewThreadSafe(),
		network: wm.network,
		p:       p,
		refcnt:  1,
	}
}

func (mq *msgQueue) addMessage(entries []*bsmsg.Entry, ses uint64) {
	var work bool
	mq.outlk.Lock()
	defer func() {
		mq.outlk.Unlock()
		if !work {
			return
		}
		select {
		case mq.work <- struct{}{}:
		default:
		}
	}()

	// if we have no message held allocate a new one
	if mq.out == nil {
		mq.out = bsmsg.New(false)
	}

	// TODO: add a msg.Combine(...) method
	// otherwise, combine the one we are holding with the
	// one passed in
	for _, e := range entries {
		if e.Cancel {
			if mq.wl.Remove(e.Cid, ses) {
				work = true
				mq.out.Cancel(e.Cid)
			}
		} else {
			if mq.wl.Add(e.Cid, e.Priority, ses) {
				work = true
				mq.out.AddEntry(e.Cid, e.Priority)
			}
		}
	}
}
<commit_msg>don't warn when trying to send wantlist to disconnected peers<commit_after>package bitswap

import (
	"context"
	"sync"
	"time"

	engine "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
	wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"

	metrics "gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface"
	peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer"
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
)

type WantManager struct {
	// sync channels for Run loop
	incoming     chan *wantSet
	connectEvent chan peerStatus     // notification channel for peers connecting/disconnecting
	peerReqs     chan chan []peer.ID // channel to request connected peers on

	// synchronized by Run loop, only touch inside there
	peers map[peer.ID]*msgQueue
	wl    *wantlist.ThreadSafe
	bcwl  *wantlist.ThreadSafe

	network bsnet.BitSwapNetwork
	ctx     context.Context
	cancel  func()

	wantlistGauge metrics.Gauge
	sentHistogram metrics.Histogram
}

type peerStatus struct {
	connect bool
	peer    peer.ID
}

func NewWantManager(ctx context.Context, network bsnet.BitSwapNetwork) *WantManager {
	ctx, cancel := context.WithCancel(ctx)
	wantlistGauge := metrics.NewCtx(ctx, "wantlist_total",
		"Number of items in wantlist.").Gauge()
	sentHistogram := metrics.NewCtx(ctx, "sent_all_blocks_bytes", "Histogram of blocks sent by"+
		" this bitswap").Histogram(metricsBuckets)
	return &WantManager{
		incoming:      make(chan *wantSet, 10),
		connectEvent:  make(chan peerStatus, 10),
		peerReqs:      make(chan chan []peer.ID),
		peers:         make(map[peer.ID]*msgQueue),
		wl:            wantlist.NewThreadSafe(),
		bcwl:          wantlist.NewThreadSafe(),
		network:       network,
		ctx:           ctx,
		cancel:        cancel,
		wantlistGauge: wantlistGauge,
		sentHistogram: sentHistogram,
	}
}

type msgQueue struct {
	p peer.ID

	outlk   sync.Mutex
	out     bsmsg.BitSwapMessage
	network bsnet.BitSwapNetwork
	wl      *wantlist.ThreadSafe

	sender bsnet.MessageSender

	refcnt int

	work chan struct{}
	done chan struct{}
}

// WantBlocks adds the given cids to the wantlist, tracked by the given session
func (pm *WantManager) WantBlocks(ctx context.Context, ks []*cid.Cid, peers []peer.ID, ses uint64) {
	log.Infof("want blocks: %s", ks)
	pm.addEntries(ctx, ks, peers, false, ses)
}

// CancelWants removes the given cids from the wantlist, tracked by the given session
func (pm *WantManager) CancelWants(ctx context.Context, ks []*cid.Cid, peers []peer.ID, ses uint64) {
	pm.addEntries(context.Background(), ks, peers, true, ses)
}

type wantSet struct {
	entries []*bsmsg.Entry
	targets []peer.ID
	from    uint64
}

func (pm *WantManager) addEntries(ctx context.Context, ks []*cid.Cid, targets []peer.ID, cancel bool, ses uint64) {
	entries := make([]*bsmsg.Entry, 0, len(ks))
	for i, k := range ks {
		entries = append(entries, &bsmsg.Entry{
			Cancel: cancel,
			Entry:  wantlist.NewRefEntry(k, kMaxPriority-i),
		})
	}
	select {
	case pm.incoming <- &wantSet{entries: entries, targets: targets, from: ses}:
	case <-pm.ctx.Done():
	case <-ctx.Done():
	}
}

func (pm *WantManager) ConnectedPeers() []peer.ID {
	resp := make(chan []peer.ID)
	pm.peerReqs <- resp
	return <-resp
}

func (pm *WantManager) SendBlock(ctx context.Context, env *engine.Envelope) {
	// Blocks need to be sent synchronously to maintain proper backpressure
	// throughout the network stack
	defer env.Sent()

	pm.sentHistogram.Observe(float64(len(env.Block.RawData())))

	msg := bsmsg.New(false)
	msg.AddBlock(env.Block)
	log.Infof("Sending block %s to %s", env.Block, env.Peer)
	err := pm.network.SendMessage(ctx, env.Peer, msg)
	if err != nil {
		log.Infof("sendblock error: %s", err)
	}
}

func (pm *WantManager) startPeerHandler(p peer.ID) *msgQueue {
	mq, ok := pm.peers[p]
	if ok {
		mq.refcnt++
		return nil
	}

	mq = pm.newMsgQueue(p)

	// new peer, we will want to give them our full wantlist
	fullwantlist := bsmsg.New(true)
	for _, e := range pm.bcwl.Entries() {
		for k := range e.SesTrk {
			mq.wl.AddEntry(e, k)
		}
		fullwantlist.AddEntry(e.Cid, e.Priority)
	}
	mq.out = fullwantlist
	mq.work <- struct{}{}

	pm.peers[p] = mq
	go mq.runQueue(pm.ctx)
	return mq
}

func (pm *WantManager) stopPeerHandler(p peer.ID) {
	pq, ok := pm.peers[p]
	if !ok {
		// TODO: log error?
		return
	}

	pq.refcnt--
	if pq.refcnt > 0 {
		return
	}

	close(pq.done)
	delete(pm.peers, p)
}

func (mq *msgQueue) runQueue(ctx context.Context) {
	for {
		select {
		case <-mq.work: // there is work to be done
			mq.doWork(ctx)
		case <-mq.done:
			if mq.sender != nil {
				mq.sender.Close()
			}
			return
		case <-ctx.Done():
			if mq.sender != nil {
				mq.sender.Reset()
			}
			return
		}
	}
}

func (mq *msgQueue) doWork(ctx context.Context) {
	// grab outgoing message
	mq.outlk.Lock()
	wlm := mq.out
	if wlm == nil || wlm.Empty() {
		mq.outlk.Unlock()
		return
	}
	mq.out = nil
	mq.outlk.Unlock()

	// NB: only open a stream if we actually have data to send
	if mq.sender == nil {
		err := mq.openSender(ctx)
		if err != nil {
			log.Infof("cant open message sender to peer %s: %s", mq.p, err)
			// TODO: cant connect, what now?
			return
		}
	}

	// send wantlist updates
	for { // try to send this message until we fail.
		err := mq.sender.SendMsg(ctx, wlm)
		if err == nil {
			return
		}

		log.Infof("bitswap send error: %s", err)
		mq.sender.Reset()
		mq.sender = nil

		select {
		case <-mq.done:
			return
		case <-ctx.Done():
			return
		case <-time.After(time.Millisecond * 100):
			// wait 100ms in case disconnect notifications are still propogating
			log.Warning("SendMsg errored but neither 'done' nor context.Done() were set")
		}

		err = mq.openSender(ctx)
		if err != nil {
			log.Infof("couldnt open sender again after SendMsg(%s) failed: %s", mq.p, err)
			// TODO(why): what do we do now?
			// I think the *right* answer is to probably put the message we're
			// trying to send back, and then return to waiting for new work or
			// a disconnect.
			return
		}

		// TODO: Is this the same instance for the remote peer?
		// If its not, we should resend our entire wantlist to them
		/*
			if mq.sender.InstanceID() != mq.lastSeenInstanceID {
				wlm = mq.getFullWantlistMessage()
			}
		*/
	}
}

func (mq *msgQueue) openSender(ctx context.Context) error {
	// allow ten minutes for connections this includes looking them up in the
	// dht dialing them, and handshaking
	conctx, cancel := context.WithTimeout(ctx, time.Minute*10)
	defer cancel()

	err := mq.network.ConnectTo(conctx, mq.p)
	if err != nil {
		return err
	}

	nsender, err := mq.network.NewMessageSender(ctx, mq.p)
	if err != nil {
		return err
	}

	mq.sender = nsender
	return nil
}

func (pm *WantManager) Connected(p peer.ID) {
	select {
	case pm.connectEvent <- peerStatus{peer: p, connect: true}:
	case <-pm.ctx.Done():
	}
}

func (pm *WantManager) Disconnected(p peer.ID) {
	select {
	case pm.connectEvent <- peerStatus{peer: p, connect: false}:
	case <-pm.ctx.Done():
	}
}

// TODO: use goprocess here once i trust it
func (pm *WantManager) Run() {
	// NOTE: Do not open any streams or connections from anywhere in this
	// event loop. Really, just don't do anything likely to block.
	for {
		select {
		case ws := <-pm.incoming:

			// is this a broadcast or not?
			brdc := len(ws.targets) == 0

			// add changes to our wantlist
			for _, e := range ws.entries {
				if e.Cancel {
					if brdc {
						pm.bcwl.Remove(e.Cid, ws.from)
					}

					if pm.wl.Remove(e.Cid, ws.from) {
						pm.wantlistGauge.Dec()
					}
				} else {
					if brdc {
						pm.bcwl.AddEntry(e.Entry, ws.from)
					}
					if pm.wl.AddEntry(e.Entry, ws.from) {
						pm.wantlistGauge.Inc()
					}
				}
			}

			// broadcast those wantlist changes
			if len(ws.targets) == 0 {
				for _, p := range pm.peers {
					p.addMessage(ws.entries, ws.from)
				}
			} else {
				for _, t := range ws.targets {
					p, ok := pm.peers[t]
					if !ok {
						// No longer connected.
						continue
					}
					p.addMessage(ws.entries, ws.from)
				}
			}

		case p := <-pm.connectEvent:
			if p.connect {
				pm.startPeerHandler(p.peer)
			} else {
				pm.stopPeerHandler(p.peer)
			}
		case req := <-pm.peerReqs:
			peers := make([]peer.ID, 0, len(pm.peers))
			for p := range pm.peers {
				peers = append(peers, p)
			}
			req <- peers
		case <-pm.ctx.Done():
			return
		}
	}
}

func (wm *WantManager) newMsgQueue(p peer.ID) *msgQueue {
	return &msgQueue{
		done:    make(chan struct{}),
		work:    make(chan struct{}, 1),
		wl:      wantlist.NewThreadSafe(),
		network: wm.network,
		p:       p,
		refcnt:  1,
	}
}

func (mq *msgQueue) addMessage(entries []*bsmsg.Entry, ses uint64) {
	var work bool
	mq.outlk.Lock()
	defer func() {
		mq.outlk.Unlock()
		if !work {
			return
		}
		select {
		case mq.work <- struct{}{}:
		default:
		}
	}()

	// if we have no message held allocate a new one
	if mq.out == nil {
		mq.out = bsmsg.New(false)
	}

	// TODO: add a msg.Combine(...) method
	// otherwise, combine the one we are holding with the
	// one passed in
	for _, e := range entries {
		if e.Cancel {
			if mq.wl.Remove(e.Cid, ses) {
				work = true
				mq.out.Cancel(e.Cid)
			}
		} else {
			if mq.wl.Add(e.Cid, e.Priority, ses) {
				work = true
				mq.out.AddEntry(e.Cid, e.Priority)
			}
		}
	}
}
<|endoftext|>


<commit_before>package dnsserver

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"

	"github.com/miekg/dns"
)

// ServerTLS represents an instance of a TLS-over-DNS-server.
type ServerTLS struct {
	*Server
	tlsConfig *tls.Config
}

// NewServerTLS returns a new CoreDNS TLS server and compiles all plugin in to it.
func NewServerTLS(addr string, group []*Config) (*ServerTLS, error) {
	s, err := NewServer(addr, group)
	if err != nil {
		return nil, err
	}
	// The *tls* plugin must make sure that multiple conflicting
	// TLS configuration return an error: it can only be specified once.
	var tlsConfig *tls.Config
	for _, conf := range s.zones {
		// Should we error if some configs *don't* have TLS?
		tlsConfig = conf.TLSConfig
	}

	return &ServerTLS{Server: s, tlsConfig: tlsConfig}, nil
}

// Serve implements caddy.TCPServer interface.
func (s *ServerTLS) Serve(l net.Listener) error {
	s.m.Lock()

	if s.tlsConfig != nil {
		l = tls.NewListener(l, s.tlsConfig)
	}

	// Only fill out the TCP server for this one.
	s.server[tcp] = &dns.Server{Listener: l, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
		ctx := context.Background()
		s.ServeDNS(ctx, w, r)
	})}
	s.m.Unlock()

	return s.server[tcp].ActivateAndServe()
}

// ServePacket implements caddy.UDPServer interface.
func (s *ServerTLS) ServePacket(p net.PacketConn) error { return nil }

// Listen implements caddy.TCPServer interface.
func (s *ServerTLS) Listen() (net.Listener, error) {
	l, err := net.Listen("tcp", s.Addr[len(TransportTLS+"://"):])
	if err != nil {
		return nil, err
	}
	return l, nil
}

// ListenPacket implements caddy.UDPServer interface.
func (s *ServerTLS) ListenPacket() (net.PacketConn, error) { return nil, nil }

// OnStartupComplete lists the sites served by this server
// and any relevant information, assuming Quiet is false.
func (s *ServerTLS) OnStartupComplete() {
	if Quiet {
		return
	}

	for zone, config := range s.zones {
		fmt.Println(TransportTLS + "://" + zone + ":" + config.Port)
	}
}
<commit_msg>Uses external context package for server-tls.go (#1471)<commit_after>package dnsserver

import (
	"crypto/tls"
	"fmt"
	"net"

	"github.com/miekg/dns"
	"golang.org/x/net/context"
)

// ServerTLS represents an instance of a TLS-over-DNS-server.
type ServerTLS struct {
	*Server
	tlsConfig *tls.Config
}

// NewServerTLS returns a new CoreDNS TLS server and compiles all plugin in to it.
func NewServerTLS(addr string, group []*Config) (*ServerTLS, error) {
	s, err := NewServer(addr, group)
	if err != nil {
		return nil, err
	}
	// The *tls* plugin must make sure that multiple conflicting
	// TLS configuration return an error: it can only be specified once.
	var tlsConfig *tls.Config
	for _, conf := range s.zones {
		// Should we error if some configs *don't* have TLS?
		tlsConfig = conf.TLSConfig
	}

	return &ServerTLS{Server: s, tlsConfig: tlsConfig}, nil
}

// Serve implements caddy.TCPServer interface.
func (s *ServerTLS) Serve(l net.Listener) error {
	s.m.Lock()

	if s.tlsConfig != nil {
		l = tls.NewListener(l, s.tlsConfig)
	}

	// Only fill out the TCP server for this one.
	s.server[tcp] = &dns.Server{Listener: l, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
		ctx := context.Background()
		s.ServeDNS(ctx, w, r)
	})}
	s.m.Unlock()

	return s.server[tcp].ActivateAndServe()
}

// ServePacket implements caddy.UDPServer interface.
func (s *ServerTLS) ServePacket(p net.PacketConn) error { return nil }

// Listen implements caddy.TCPServer interface.
func (s *ServerTLS) Listen() (net.Listener, error) {
	l, err := net.Listen("tcp", s.Addr[len(TransportTLS+"://"):])
	if err != nil {
		return nil, err
	}
	return l, nil
}

// ListenPacket implements caddy.UDPServer interface.
func (s *ServerTLS) ListenPacket() (net.PacketConn, error) { return nil, nil }

// OnStartupComplete lists the sites served by this server
// and any relevant information, assuming Quiet is false.
func (s *ServerTLS) OnStartupComplete() {
	if Quiet {
		return
	}

	for zone, config := range s.zones {
		fmt.Println(TransportTLS + "://" + zone + ":" + config.Port)
	}
}
<|endoftext|>


<commit_before>package controller

import (
	. "eaciit/wfdemo-git/library/core"
	. "eaciit/wfdemo-git/library/models"
	"strings"
	"time"

	"eaciit/wfdemo-git/web/helper"

	knot "github.com/eaciit/knot/knot.v1"
	tk "github.com/eaciit/toolkit"
)

type MonitoringController struct {
	App
}

func CreateMonitoringController() *MonitoringController {
	var controller = new(MonitoringController)
	return controller
}

func (m *MonitoringController) GetData(k *knot.WebContext) interface{} {
	k.Config.OutputType = knot.OutputJson

	p := tk.M{}
	e := k.GetPayload(&p)

	if e != nil {
		return helper.CreateResult(false, nil, e.Error())
	}

	// get the last data for monitoring

	turbine := p.Get("turbine").([]interface{})
	project := ""
	if p.GetString("project") != "" {
		anProject := strings.Split(p.GetString("project"), "(")
		project = strings.TrimRight(anProject[0], " ")
	}

	match := tk.M{}
	turbines := map[string]tk.M{}
	var projectList []interface{}
	if project != "" {
		match.Set("project", project)
		projectList = append(projectList, project)
	}

	turbines, _, _ = helper.GetProjectTurbineList(projectList)

	if len(turbine) > 0 {
		match.Set("turbine", tk.M{}.Set("$in", turbine))
	}

	group := tk.M{
		"_id": tk.M{
			"project": "$project",
			"turbine": "$turbine",
		},
		"timestamp":     tk.M{"$last": "$timestamp"},
		"windspeed":     tk.M{"$last": "$windspeed"},
		"production":    tk.M{"$last": "$production"},
		"rotorspeedrpm": tk.M{"$last": "$rotorspeedrpm"},
		"status":        tk.M{"$last": "$status"},
		"statuscode":    tk.M{"$last": "$statuscode"},
		"statusdesc":    tk.M{"$last": "$statusdesc"},
	}

	var pipes []tk.M
	pipes = append(pipes, tk.M{}.Set("$match", match))
	pipes = append(pipes, tk.M{}.Set("$group", group))
	pipes = append(pipes, tk.M{}.Set("$sort", tk.M{
		"_id.project": 1,
		"_id.turbine": 1,
	}))

	csr, e := DB().Connection.NewQuery().
		From(new(Monitoring).TableName()).
		Command("pipe", pipes).
		Cursor(nil)

	defer csr.Close()

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	results := make([]tk.M, 0)
	e = csr.Fetch(&results, 0, false)

	projects := tk.M{}

	for _, v := range results {
		ID := v.Get("_id").(tk.M)
		project := ID.GetString("project")
		turbine := ID.GetString("turbine")
		timestamp := v.Get("timestamp").(time.Time).UTC()
		windspeed := checkNAValue(v.GetFloat64("windspeed"))
		production := checkNAValue(v.GetFloat64("production"))
		rotorspeedrpm := checkNAValue(v.GetFloat64("rotorspeedrpm"))
		status := v.GetString("status")
		statuscode := v.GetString("statuscode")
		statusdesc := v.GetString("statusdesc")

		list := []tk.M{}
		newRecord := tk.M{}

		updated := tk.M{}
		capacitymw := 0.0

		if projects.Get(project) != nil {
			if projects.Get(project).(tk.M).Get("turbines") != nil {
				list = projects.Get(project).(tk.M).Get("turbines").([]tk.M)
			}
			updated = projects.Get(project).(tk.M)
		}

		if turbines[project].Get(turbine) != nil {
			capacitymw = turbines[project].Get(turbine).(TurbineMaster).CapacityMW
		}

		newRecord.Set("turbine", turbine)
		newRecord.Set("timestamp", timestamp)
		newRecord.Set("timestampstr", timestamp.Format("02-01-2006 15:04:05"))
		newRecord.Set("windspeed", windspeed)
		newRecord.Set("production", production)
		newRecord.Set("rotorspeedrpm", rotorspeedrpm)
		newRecord.Set("status", status)
		newRecord.Set("statuscode", statuscode)
		newRecord.Set("statusdesc", statusdesc)

		list = append(list, newRecord)
		updated.Set("turbines", list)
		updated.Set("totalturbines", updated.GetInt("totalturbines")+1)
		updated.Set("totalcap", updated.GetFloat64("totalcap")+capacitymw)
		updated.Set("totalprod", updated.GetFloat64("totalprod")+production)

		projects.Set(project, updated)
	}

	// get today total production

	match = tk.M{}
	if project != "" {
		match.Set("project", project)
	}

	if len(turbine) > 0 {
		match.Set("turbine", tk.M{}.Set("$in", turbine))
	}

	dateid, _ := time.Parse("20060102_0504", time.Now().Format("20060102_")+"_0000")
	match.Set("dateinfo.dateid", dateid.UTC())

	group = tk.M{
		"_id":        "$project",
		"production": tk.M{"$sum": "$production"},
	}

	pipes = []tk.M{}
	pipes = append(pipes, tk.M{}.Set("$match", match))
	pipes = append(pipes, tk.M{}.Set("$group", group))

	csr, e = DB().Connection.NewQuery().
		From(new(Monitoring).TableName()).
		Command("pipe", pipes).
		Cursor(nil)

	defer csr.Close()

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	results = make([]tk.M, 0)
	e = csr.Fetch(&results, 0, false)

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	projectProdResult := map[string]float64{}

	for _, v := range results {
		id := v.GetString("_id")
		prod := v.GetFloat64("production")
		projectProdResult[id] = prod
	}

	// combine the data

	res := []tk.M{}

	for proj, v := range projects {
		wsavg := 0.0
		turbineList := v.(tk.M).Get("turbines").([]tk.M)

		for _, i := range turbineList {
			wsavg += i.GetFloat64("windspeed")
		}

		wsavg = tk.Div(wsavg, tk.ToFloat64(len(turbineList), 0, tk.RoundingAuto))
		v.(tk.M).Set("totalwsavg", wsavg)
		// v.(tk.M).Set("totalprod", v.(tk.M).GetFloat64("totalprod")/1000)
		v.(tk.M).Set("totalprod", projectProdResult[proj]/1000)

		v.(tk.M).Set("project", proj)
		res = append(res, v.(tk.M))
	}

	// get latest date update from ScadaDataHFD

	availDate := k.Session("availdate", "")
	date := availDate.(*Availdatedata).ScadaDataHFD[1].UTC()

	finalResult := tk.M{}
	finalResult.Set("data", res)
	finalResult.Set("timestamp", tk.M{"minute": date.Format("15:04"), "date": date.Format("02 Jan 2006")})

	data := struct {
		Data tk.M
	}{
		Data: finalResult,
	}

	return helper.CreateResult(true, data, "success")
}

func (m *MonitoringController) GetEvent(k *knot.WebContext) interface{} {
	k.Config.OutputType = knot.OutputJson

	p := tk.M{}
	e := k.GetPayload(&p)

	if e != nil {
		return helper.CreateResult(false, nil, e.Error())
	}

	turbine := p.Get("turbine").([]interface{})
	project := ""
	if p.GetString("project") != "" {
		anProject := strings.Split(p.GetString("project"), "(")
		project = strings.TrimRight(anProject[0], " ")
	}

	// log.Printf("%#v \n", project)

	match := tk.M{}
	var projectList []interface{}
	if project != "" {
		match.Set("project", project)
		projectList = append(projectList, project)
	}

	if len(turbine) > 0 {
		match.Set("turbine", tk.M{}.Set("$in", turbine))
	}

	var pipes []tk.M
	pipes = append(pipes, tk.M{}.Set("$match", match))
	pipes = append(pipes, tk.M{}.Set("$sort", tk.M{
		"timestamp": 1,
		"turbine":   1,
		"project":   1,
	}))

	csr, e := DB().Connection.NewQuery().
		From(new(MonitoringEvent).TableName()).
		Command("pipe", pipes).
		Cursor(nil)

	defer csr.Close()

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	results := make([]MonitoringEvent, 0)
	e = csr.Fetch(&results, 0, false)

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	data := struct {
		Data []MonitoringEvent
	}{
		Data: results,
	}

	return helper.CreateResult(true, data, "success")
}

func checkNAValue(val float64) (result float64) {
	if val == -9999999.0 {
		result = 0
	} else {
		result = val
	}

	return
}
<commit_msg>change ordering on getevent<commit_after>package controller

import (
	. "eaciit/wfdemo-git/library/core"
	. "eaciit/wfdemo-git/library/models"
	"strings"
	"time"

	"eaciit/wfdemo-git/web/helper"

	knot "github.com/eaciit/knot/knot.v1"
	tk "github.com/eaciit/toolkit"
)

type MonitoringController struct {
	App
}

func CreateMonitoringController() *MonitoringController {
	var controller = new(MonitoringController)
	return controller
}

func (m *MonitoringController) GetData(k *knot.WebContext) interface{} {
	k.Config.OutputType = knot.OutputJson

	p := tk.M{}
	e := k.GetPayload(&p)

	if e != nil {
		return helper.CreateResult(false, nil, e.Error())
	}

	// get the last data for monitoring

	turbine := p.Get("turbine").([]interface{})
	project := ""
	if p.GetString("project") != "" {
		anProject := strings.Split(p.GetString("project"), "(")
		project = strings.TrimRight(anProject[0], " ")
	}

	match := tk.M{}
	turbines := map[string]tk.M{}
	var projectList []interface{}
	if project != "" {
		match.Set("project", project)
		projectList = append(projectList, project)
	}

	turbines, _, _ = helper.GetProjectTurbineList(projectList)

	if len(turbine) > 0 {
		match.Set("turbine", tk.M{}.Set("$in", turbine))
	}

	group := tk.M{
		"_id": tk.M{
			"project": "$project",
			"turbine": "$turbine",
		},
		"timestamp":     tk.M{"$last": "$timestamp"},
		"windspeed":     tk.M{"$last": "$windspeed"},
		"production":    tk.M{"$last": "$production"},
		"rotorspeedrpm": tk.M{"$last": "$rotorspeedrpm"},
		"status":        tk.M{"$last": "$status"},
		"statuscode":    tk.M{"$last": "$statuscode"},
		"statusdesc":    tk.M{"$last": "$statusdesc"},
	}

	var pipes []tk.M
	pipes = append(pipes, tk.M{}.Set("$match", match))
	pipes = append(pipes, tk.M{}.Set("$group", group))
	pipes = append(pipes, tk.M{}.Set("$sort", tk.M{
		"_id.project": 1,
		"_id.turbine": 1,
	}))

	csr, e := DB().Connection.NewQuery().
		From(new(Monitoring).TableName()).
		Command("pipe", pipes).
		Cursor(nil)

	defer csr.Close()

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	results := make([]tk.M, 0)
	e = csr.Fetch(&results, 0, false)

	projects := tk.M{}

	for _, v := range results {
		ID := v.Get("_id").(tk.M)
		project := ID.GetString("project")
		turbine := ID.GetString("turbine")
		timestamp := v.Get("timestamp").(time.Time).UTC()
		windspeed := checkNAValue(v.GetFloat64("windspeed"))
		production := checkNAValue(v.GetFloat64("production"))
		rotorspeedrpm := checkNAValue(v.GetFloat64("rotorspeedrpm"))
		status := v.GetString("status")
		statuscode := v.GetString("statuscode")
		statusdesc := v.GetString("statusdesc")

		list := []tk.M{}
		newRecord := tk.M{}

		updated := tk.M{}
		capacitymw := 0.0

		if projects.Get(project) != nil {
			if projects.Get(project).(tk.M).Get("turbines") != nil {
				list = projects.Get(project).(tk.M).Get("turbines").([]tk.M)
			}
			updated = projects.Get(project).(tk.M)
		}

		if turbines[project].Get(turbine) != nil {
			capacitymw = turbines[project].Get(turbine).(TurbineMaster).CapacityMW
		}

		newRecord.Set("turbine", turbine)
		newRecord.Set("timestamp", timestamp)
		newRecord.Set("timestampstr", timestamp.Format("02-01-2006 15:04:05"))
		newRecord.Set("windspeed", windspeed)
		newRecord.Set("production", production)
		newRecord.Set("rotorspeedrpm", rotorspeedrpm)
		newRecord.Set("status", status)
		newRecord.Set("statuscode", statuscode)
		newRecord.Set("statusdesc", statusdesc)

		list = append(list, newRecord)
		updated.Set("turbines", list)
		updated.Set("totalturbines", updated.GetInt("totalturbines")+1)
		updated.Set("totalcap", updated.GetFloat64("totalcap")+capacitymw)
		updated.Set("totalprod", updated.GetFloat64("totalprod")+production)

		projects.Set(project, updated)
	}

	// get today total production

	match = tk.M{}
	if project != "" {
		match.Set("project", project)
	}

	if len(turbine) > 0 {
		match.Set("turbine", tk.M{}.Set("$in", turbine))
	}

	dateid, _ := time.Parse("20060102_0504", time.Now().Format("20060102_")+"_0000")
	match.Set("dateinfo.dateid", dateid.UTC())

	group = tk.M{
		"_id":        "$project",
		"production": tk.M{"$sum": "$production"},
	}

	pipes = []tk.M{}
	pipes = append(pipes, tk.M{}.Set("$match", match))
	pipes = append(pipes, tk.M{}.Set("$group", group))

	csr, e = DB().Connection.NewQuery().
		From(new(Monitoring).TableName()).
		Command("pipe", pipes).
		Cursor(nil)

	defer csr.Close()

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	results = make([]tk.M, 0)
	e = csr.Fetch(&results, 0, false)

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	projectProdResult := map[string]float64{}

	for _, v := range results {
		id := v.GetString("_id")
		prod := v.GetFloat64("production")
		projectProdResult[id] = prod
	}

	// combine the data

	res := []tk.M{}

	for proj, v := range projects {
		wsavg := 0.0
		turbineList := v.(tk.M).Get("turbines").([]tk.M)

		for _, i := range turbineList {
			wsavg += i.GetFloat64("windspeed")
		}

		wsavg = tk.Div(wsavg, tk.ToFloat64(len(turbineList), 0, tk.RoundingAuto))
		v.(tk.M).Set("totalwsavg", wsavg)
		// v.(tk.M).Set("totalprod", v.(tk.M).GetFloat64("totalprod")/1000)
		v.(tk.M).Set("totalprod", projectProdResult[proj]/1000)

		v.(tk.M).Set("project", proj)
		res = append(res, v.(tk.M))
	}

	// get latest date update from ScadaDataHFD

	availDate := k.Session("availdate", "")
	date := availDate.(*Availdatedata).ScadaDataHFD[1].UTC()

	finalResult := tk.M{}
	finalResult.Set("data", res)
	finalResult.Set("timestamp", tk.M{"minute": date.Format("15:04"), "date": date.Format("02 Jan 2006")})

	data := struct {
		Data tk.M
	}{
		Data: finalResult,
	}

	return helper.CreateResult(true, data, "success")
}

func (m *MonitoringController) GetEvent(k *knot.WebContext) interface{} {
	k.Config.OutputType = knot.OutputJson

	p := tk.M{}
	e := k.GetPayload(&p)

	if e != nil {
		return helper.CreateResult(false, nil, e.Error())
	}

	turbine := p.Get("turbine").([]interface{})
	project := ""
	if p.GetString("project") != "" {
		anProject := strings.Split(p.GetString("project"), "(")
		project = strings.TrimRight(anProject[0], " ")
	}

	// log.Printf("%#v \n", project)

	match := tk.M{}
	var projectList []interface{}
	if project != "" {
		match.Set("project", project)
		projectList = append(projectList, project)
	}

	if len(turbine) > 0 {
		match.Set("turbine", tk.M{}.Set("$in", turbine))
	}

	var pipes []tk.M
	pipes = append(pipes, tk.M{}.Set("$match", match))
	pipes = append(pipes, tk.M{}.Set("$sort", tk.M{
		"timestamp": -1,
		"turbine":   -1,
		"project":   -1,
	}))

	csr, e := DB().Connection.NewQuery().
		From(new(MonitoringEvent).TableName()).
		Command("pipe", pipes).
		Cursor(nil)

	defer csr.Close()

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	results := make([]MonitoringEvent, 0)
	e = csr.Fetch(&results, 0, false)

	if e != nil {
		return helper.CreateResult(false, nil, "Error query : "+e.Error())
	}

	data := struct {
		Data []MonitoringEvent
	}{
		Data: results,
	}

	return helper.CreateResult(true, data, "success")
}

func checkNAValue(val float64) (result float64) {
	if val == -9999999.0 {
		result = 0
	} else {
		result = val
	}

	return
}
<|endoftext|>


<commit_before>package core

import (
	"flag"
	"strings"
)

// ServerConfiguration struct
// Most of this configuration comes from json config
type ServerConfiguration struct {
	SourceDomain          string
	WhitelistedExtensions []string
	MaximumWidth          int
	LocalBasePath         string
	MantaBasePath         string
	DefaultQuality        uint
	GraphiteHost          string
	GraphitePort          int
	Environment           string
	Events                *EventChannels
	Adapters              *Adapters
}

// EventChannels struct
// Available image processing/downloading events
type EventChannels struct {
	ImageProcessed              chan *ImageConfiguration
	ImageProcessedWithErrors    chan *ImageConfiguration
	OriginalDownloaded          chan *ImageConfiguration
	OriginalDownloadUnavailable chan *ImageConfiguration
}

// ServerConfigurationFromFlags initializes a ServerConfiguration from flags
func ServerConfigurationFromFlags() (*ServerConfiguration, error) {
	var (
		whitelistedExtensions = flag.String("extensions", "jpg,gif,webp", "Whitelisted extensions (separated by commas)")
		localBasePath         = flag.String("local_base_path", "public", "Directory where the images will be saved")
		sourceDomain          = flag.String("source_domain", "http://wanelo.s3.amazonaws.com", "Source domain for images")
		mantaBasePath         = flag.String("manta_base_path", "public/images/development", "base path for manta storage")
		graphiteHost          = flag.String("graphite_host", "127.0.0.1", "Graphite Host")
		graphitePort          = flag.Int("graphite_port", 8125, "Graphite port")
		maximumWidth          = flag.Int("maximum_width", 1000, "maximum image width")
		defaultQuality        = flag.Uint("default_quality", 75, "Default image compression quality")
	)
	flag.Parse()

	sc := &ServerConfiguration{
		WhitelistedExtensions: strings.Split(*whitelistedExtensions, ","),
		LocalBasePath:         *localBasePath,
		GraphitePort:          *graphitePort,
		GraphiteHost:          *graphiteHost,
		MaximumWidth:          *maximumWidth,
		MantaBasePath:         *mantaBasePath,
		DefaultQuality:        *defaultQuality,
		SourceDomain:          *sourceDomain,
	}
	return sc, nil
}
<commit_msg>Environment is no longer a server configuration<commit_after>package core

import (
	"flag"
	"strings"
)

// ServerConfiguration struct
// Most of this configuration comes from json config
type ServerConfiguration struct {
	SourceDomain          string
	WhitelistedExtensions []string
	MaximumWidth          int
	LocalBasePath         string
	MantaBasePath         string
	DefaultQuality        uint
	GraphiteHost          string
	GraphitePort          int
	Events                *EventChannels
	Adapters              *Adapters
}

// EventChannels struct
// Available image processing/downloading events
type EventChannels struct {
	ImageProcessed              chan *ImageConfiguration
	ImageProcessedWithErrors    chan *ImageConfiguration
	OriginalDownloaded          chan *ImageConfiguration
	OriginalDownloadUnavailable chan *ImageConfiguration
}

// ServerConfigurationFromFlags initializes a ServerConfiguration from flags
func ServerConfigurationFromFlags() (*ServerConfiguration, error) {
	var (
		whitelistedExtensions = flag.String("extensions", "jpg,gif,webp", "Whitelisted extensions (separated by commas)")
		localBasePath         = flag.String("local_base_path", "public", "Directory where the images will be saved")
		sourceDomain          = flag.String("source_domain", "http://wanelo.s3.amazonaws.com", "Source domain for images")
		mantaBasePath         = flag.String("manta_base_path", "public/images/development", "base path for manta storage")
		graphiteHost          = flag.String("graphite_host", "127.0.0.1", "Graphite Host")
		graphitePort          = flag.Int("graphite_port", 8125, "Graphite port")
		maximumWidth          = flag.Int("maximum_width", 1000, "maximum image width")
		defaultQuality        = flag.Uint("default_quality", 75, "Default image compression quality")
	)
	flag.Parse()

	sc := &ServerConfiguration{
		WhitelistedExtensions: strings.Split(*whitelistedExtensions, ","),
		LocalBasePath:         *localBasePath,
		GraphitePort:          *graphitePort,
		GraphiteHost:          *graphiteHost,
		MaximumWidth:          *maximumWidth,
		MantaBasePath:         *mantaBasePath,
		DefaultQuality:        *defaultQuality,
		SourceDomain:          *sourceDomain,
	}
	return sc, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io"
	"log"
	"sort"
	"time"
)

type page struct {
	n      offset
	body   []byte
	expire time.Time
	cached bool
}

func newPage(n offset, body []byte) *page {
	return &page{
		n:    n,
		body: body,
	}
}

func (p *page) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(p.body)
	return int64(n), err
}

type group string

type entry struct {
	deadline time.Time
	data     []byte
}

func newEntry(data []byte, d time.Duration) *entry {
	return &entry{
		deadline: time.Now().Add(d),
		data:     data,
	}
}

func (ce *entry) invalid(t time.Time) bool {
	return !ce.deadline.After(t)
}

func (ce *entry) asPage(n offset) *page {
	p := newPage(n, ce.data)
	p.expire = ce.deadline
	return p
}

type entries struct {
	ents map[group]map[offset]*entry
}

func newEntries() *entries {
	return &entries{
		ents: make(map[group]map[offset]*entry),
	}
}

func (e *entries) count() int {
	tot := 0
	for k := range e.ents {
		tot += len(e.ents[k])
	}
	return tot
}

func (e *entries) get(cg group, n offset) (*entry, bool) {
	pages, ok := e.ents[cg]
	if !ok {
		return nil, false
	}
	ce, ok := pages[n]
	return ce, ok
}

func (e *entries) put(cg group, n offset, ce *entry) {
	_, ok := e.ents[cg]
	if !ok {
		e.ents[cg] = make(map[offset]*entry)
	}
	e.ents[cg][n] = ce
}

func (e *entries) has(cg group, n offset, t time.Time) bool {
	_, ok := e.ents[cg]
	if !ok {
		return false
	}
	ent, ok := e.ents[cg][n]
	if !ok {
		return false
	}
	return !ent.invalid(t)
}

// remove assumes the entry exists
func (e *entries) remove(cg group, n offset) {
	delete(e.ents[cg], n)
	if len(e.ents[cg]) == 0 {
		delete(e.ents, cg)
	}
}

func (e *entries) purge(cg group, st *stats) {
	st.mem(-e.sizeof(cg))
	delete(e.ents, cg)
}

func (e *entries) sizeof(cg group) int {
	var s int
	for n := range e.ents[cg] {
		s += len(e.ents[cg][n].data)
	}
	return s
}

func (e *entries) oldestDeadline(cg group) time.Time {
	var t time.Time
	for off := range e.ents[cg] {
		entry := e.ents[cg][off]
		if t.IsZero() || t.After(entry.deadline) {
			t = entry.deadline
		}
	}
	return t
}

func (e *entries) gc(t time.Time, st *stats) {
	for cg, ents := range e.ents {
		for n := range ents {
			if ents[n].invalid(t) {
				st.mem(-len(ents[n].data))
				e.remove(cg, n)
			}
		}
	}
}

type cacheFunc func() error

type cache struct {
	entries *entries
	waits   *waiters
	fetcher *fetcher
	config  *config
	stat    *stats
	events  chan cacheFunc
}

func newCache(f *fetcher, cf *config) *cache {
	c := &cache{
		fetcher: f,
		config:  cf,
		events:  make(chan cacheFunc),
		entries: newEntries(),
		waits:   newWaiters(),
		stat:    newStats(),
	}
	go c.gc(cf.gcpause)
	go c.run()
	return c
}

func (c *cache) run() {
	for f := range c.events {
		if err := f(); err != nil {
			log.Print("cache: ", err)
		}
	}
}

func (c *cache) gc(d time.Duration) {
	done := make(chan struct{})
	for {
		time.Sleep(d)
		c.events <- func() error {
			debug("running garbage collector cycle, memory is %d", c.stat.Mem)
			c.entries.gc(time.Now(), c.stat)
			debug("garbage collection done, memory is %d", c.stat.Mem)
			done <- struct{}{}
			return nil
		}
		<-done
	}
}

type timeGroup struct {
	t  time.Time
	cg group
}

type timeGroups struct {
	entries []timeGroup
}

type byTime []timeGroup

func (a byTime) Len() int           { return len(a) }
func (a byTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byTime) Less(i, j int) bool { return a[i].t.After(a[j].t) }

func makeTimeGroups(e *entries) timeGroups {
	var i int
	tg := make([]timeGroup, len(e.ents))
	for cg := range e.ents {
		tg[i] = timeGroup{
			t:  e.oldestDeadline(cg),
			cg: cg,
		}
		i++
	}
	sort.Sort(byTime(tg))
	return timeGroups{tg}
}

func (tg *timeGroups) purgeOldest(c *cache) {
	entry, ts := tg.entries[len(tg.entries)-1], tg.entries[0:len(tg.entries)-1]
	c.entries.purge(entry.cg, c.stat)
	tg.entries = ts
}

func (tg *timeGroups) empty() bool {
	return len(tg.entries) == 0
}

func (c *cache) oom(target int64) {
	debug("OOM called: using %d, limit is %d", c.stat.Mem, target)
	tg := makeTimeGroups(c.entries)
	for {
		tg.purgeOldest(c)
		// Repeat unless we have no more entries or we are using less memory than target
		if tg.empty() || !c.stat.above(target) {
			break
		}
	}
	debug("OOM: mem now %d", c.stat.Mem)
}

// put inserts a page into the cache (after it was fetched).
func (c *cache) put(cg group, p *page, err error) {
	c.events <- func() error {
		ce := newEntry(p.body, c.config.lifetime)
		c.entries.put(cg, p.n, ce)
		debug("added page %s/%d", cg, p.n)
		c.stat.mem(len(p.body))
		if c.stat.above(c.config.maxMemory) {
			go func() {
				c.events <- func() error {
					c.oom(c.config.maxMemory)
					return nil
				}
			}()
		}
		// If there were waiters, signal that the wait is over
		c.waits.done(cg, p.n)
		return err
	}
}

// prefetch requests pages (n-m, n) and (n, n+m) if not already fetched
func (c *cache) prefetch(cg group, n, m int, t time.Time) {
	i := n - m
	if i < 0 {
		i = 0
	}
	for ; i < n+m; i++ {
		if i == n {
			// just fetched
			continue
		}
		off := offset(i * c.config.incr)
		if c.entries.has(cg, off, t) || c.waits.has(cg, off) {
			// already fetched or requested
			continue
		}
		c.waits.wait(cg, off)
		res := newResource(c.config.tmpl, cg, off)
		c.fetcher.request(newJob(res, c))
	}
}

func (c *cache) request(cg group, n int, t time.Time) chan struct{} {
	off := offset(n * c.config.incr)
	wait := c.waits.wait(cg, off)
	res := newResource(c.config.tmpl, cg, off)
	c.fetcher.request(newJob(res, c))
	c.prefetch(cg, n, c.config.npref, t)
	return wait
}

func (c *cache) stats() *stats {
	var st *stats
	wait := make(chan struct{})
	c.events <- func() error {
		c.stat.Entries = c.entries.count()
		c.stat.Waiters = c.waits.count()
		st = c.stat.clone()
		wait <- struct{}{}
		return nil
	}
	<-wait
	return st
}

func (c *cache) get(cg group, n int) *page {
	var (
		page *page
		wait chan struct{}
	)
	cached := true
	requested := make(chan struct{})
	off := offset(n * c.config.incr)
	debug("%s/%d: requesting from cache", cg, off)
	for {
		wait = nil
		c.events <- func() error {
			defer func() { requested <- struct{}{} }()
			var now time.Time
			ce, ok := c.entries.get(cg, off)
			if ok {
				now = time.Now()
			}
			if !ok || ce.invalid(now) {
				debug("%s/%d: not cached, requested", cg, off)
				wait = c.request(cg, n, now)
				return nil
			}
			debug("%s/%d: found", cg, off)
			c.prefetch(cg, n, c.config.npref, now)
			c.stat.hit(cached)
			page = ce.asPage(off)
			return nil
		}
		<-requested
		// content was already in cache, return it
		if wait == nil {
			page.cached = cached
			return page
		}
		// We needed to request the object, it was not cached
		cached = false
		// content is being fetched, wait and try to get again
		<-wait
	}
}
<commit_msg>cache: account for memory used by overwritten entries<commit_after>// Copyright 2017 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io"
	"log"
	"sort"
	"time"
)

type page struct {
	n      offset
	body   []byte
	expire time.Time
	cached bool
}

func newPage(n offset, body []byte) *page {
	return &page{
		n:    n,
		body: body,
	}
}

func (p *page) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(p.body)
	return int64(n), err
}

type group string

type entry struct {
	deadline time.Time
	data     []byte
}

func newEntry(data []byte, d time.Duration) *entry {
	return &entry{
		deadline: time.Now().Add(d),
		data:     data,
	}
}

func (ce *entry) invalid(t time.Time) bool {
	return !ce.deadline.After(t)
}

func (ce *entry) asPage(n offset) *page {
	p := newPage(n, ce.data)
	p.expire = ce.deadline
	return p
}

type entries struct {
	ents map[group]map[offset]*entry
}

func newEntries() *entries {
	return &entries{
		ents: make(map[group]map[offset]*entry),
	}
}

func (e *entries) count() int {
	tot := 0
	for k := range e.ents {
		tot += len(e.ents[k])
	}
	return tot
}

func (e *entries) get(cg group, n offset) (*entry, bool) {
	pages, ok := e.ents[cg]
	if !ok {
		return nil, false
	}
	ce, ok := pages[n]
	return ce, ok
}

func (e *entries) put(cg group, n offset, ce *entry) {
	_, ok := e.ents[cg]
	if !ok {
		e.ents[cg] = make(map[offset]*entry)
	}
	e.ents[cg][n] = ce
}

func (e *entries) has(cg group, n offset, t time.Time) bool {
	_, ok := e.ents[cg]
	if !ok {
		return false
	}
	ent, ok := e.ents[cg][n]
	if !ok {
		return false
	}
	return !ent.invalid(t)
}

// remove assumes the entry exists
func (e *entries) remove(cg group, n offset) {
	delete(e.ents[cg], n)
	if len(e.ents[cg]) == 0 {
		delete(e.ents, cg)
	}
}

func (e *entries) purge(cg group, st *stats) {
	st.mem(-e.sizeof(cg))
	delete(e.ents, cg)
}

func (e *entries) sizeof(cg group) int {
	var s int
	for n := range e.ents[cg] {
		s += len(e.ents[cg][n].data)
	}
	return s
}

func (e *entries) oldestDeadline(cg group) time.Time {
	var t time.Time
	for off := range e.ents[cg] {
		entry := e.ents[cg][off]
		if t.IsZero() || t.After(entry.deadline) {
			t = entry.deadline
		}
	}
	return t
}

func (e *entries) gc(t time.Time, st *stats) {
	for cg, ents := range e.ents {
		for n := range ents {
			if ents[n].invalid(t) {
				st.mem(-len(ents[n].data))
				e.remove(cg, n)
			}
		}
	}
}

type cacheFunc func() error

type cache struct {
	entries *entries
	waits   *waiters
	fetcher *fetcher
	config  *config
	stat    *stats
	events  chan cacheFunc
}

func newCache(f *fetcher, cf *config) *cache {
	c := &cache{
		fetcher: f,
		config:  cf,
		events:  make(chan cacheFunc),
		entries: newEntries(),
		waits:   newWaiters(),
		stat:    newStats(),
	}
	go c.gc(cf.gcpause)
	go c.run()
	return c
}

func (c *cache) run() {
	for f := range c.events {
		if err := f(); err != nil {
			log.Print("cache: ", err)
		}
	}
}

func (c *cache) gc(d time.Duration) {
	done := make(chan struct{})
	for {
		time.Sleep(d)
		c.events <- func() error {
			debug("running garbage collector cycle, memory is %d", c.stat.Mem)
			c.entries.gc(time.Now(), c.stat)
			debug("garbage collection done, memory is %d", c.stat.Mem)
			done <- struct{}{}
			return nil
		}
		<-done
	}
}

type timeGroup struct {
	t  time.Time
	cg group
}

type timeGroups struct {
	entries []timeGroup
}

type byTime []timeGroup

func (a byTime) Len() int           { return len(a) }
func (a byTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byTime) Less(i, j int) bool { return a[i].t.After(a[j].t) }

func makeTimeGroups(e *entries) timeGroups {
	var i int
	tg := make([]timeGroup, len(e.ents))
	for cg := range e.ents {
		tg[i] = timeGroup{
			t:  e.oldestDeadline(cg),
			cg: cg,
		}
		i++
	}
	sort.Sort(byTime(tg))
	return timeGroups{tg}
}

func (tg *timeGroups) purgeOldest(c *cache) {
	entry, ts := tg.entries[len(tg.entries)-1], tg.entries[0:len(tg.entries)-1]
	c.entries.purge(entry.cg, c.stat)
	tg.entries = ts
}

func (tg *timeGroups) empty() bool {
	return len(tg.entries) == 0
}

func (c *cache) oom(target int64) {
	debug("OOM called: using %d, limit is %d", c.stat.Mem, target)
	tg := makeTimeGroups(c.entries)
	for {
		tg.purgeOldest(c)
		// Repeat unless we have no more entries or we are using less memory than target
		if tg.empty() || !c.stat.above(target) {
			break
		}
	}
	debug("OOM: mem now %d", c.stat.Mem)
}

// put inserts a page into the cache (after it was fetched).
func (c *cache) put(cg group, p *page, err error) {
	c.events <- func() error {
		ce := newEntry(p.body, c.config.lifetime)
		// If an entry exists, it will be overwritten
		if ent, ok := c.entries.get(cg, p.n); ok {
			c.stat.mem(-len(ent.data))
		}
		c.entries.put(cg, p.n, ce)
		c.stat.mem(len(p.body))
		debug("added page %s/%d", cg, p.n)
		if c.stat.above(c.config.maxMemory) {
			go func() {
				c.events <- func() error {
					c.oom(c.config.maxMemory)
					return nil
				}
			}()
		}
		// If there were waiters, signal that the wait is over
		c.waits.done(cg, p.n)
		return err
	}
}

// prefetch requests pages (n-m, n) and (n, n+m) if not already fetched
func (c *cache) prefetch(cg group, n, m int, t time.Time) {
	i := n - m
	if i < 0 {
		i = 0
	}
	for ; i < n+m; i++ {
		if i == n {
			// just fetched
			continue
		}
		off := offset(i * c.config.incr)
		if c.entries.has(cg, off, t) || c.waits.has(cg, off) {
			// already fetched or requested
			continue
		}
		c.waits.wait(cg, off)
		res := newResource(c.config.tmpl, cg, off)
		c.fetcher.request(newJob(res, c))
	}
}

func (c *cache) request(cg group, n int, t time.Time) chan struct{} {
	off := offset(n * c.config.incr)
	wait := c.waits.wait(cg, off)
	res := newResource(c.config.tmpl, cg, off)
	c.fetcher.request(newJob(res, c))
	c.prefetch(cg, n, c.config.npref, t)
	return wait
}

func (c *cache) stats() *stats {
	var st *stats
	wait := make(chan struct{})
	c.events <- func() error {
		c.stat.Entries = c.entries.count()
		c.stat.Waiters = c.waits.count()
		st = c.stat.clone()
		wait <- struct{}{}
		return nil
	}
	<-wait
	return st
}

func (c *cache) get(cg group, n int) *page {
	var (
		page *page
		wait chan struct{}
	)
	cached := true
	requested := make(chan struct{})
	off := offset(n * c.config.incr)
	debug("%s/%d: requesting from cache", cg, off)
	for {
		wait = nil
		c.events <- func() error {
			defer func() { requested <- struct{}{} }()
			var now time.Time
			ce, ok := c.entries.get(cg, off)
			if ok {
				now = time.Now()
			}
			if !ok || ce.invalid(now) {
				debug("%s/%d: not cached, requested", cg, off)
				wait = c.request(cg, n, now)
				return nil
			}
			debug("%s/%d: found", cg, off)
			c.prefetch(cg, n, c.config.npref, now)
			c.stat.hit(cached)
			page = ce.asPage(off)
			return nil
		}
		<-requested
		// content was already in cache, return it
		if wait == nil {
			page.cached = cached
			return page
		}
		// We needed to request the object, it was not cached
		cached = false
		// content is being fetched, wait and try to get again
		<-wait
	}
}
<|endoftext|>


<commit_before>package epp

import (
	"encoding/xml"
	"fmt"
	"io"
)

// <epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
//   <command>
//     <check>
//     <domain:check xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
//       <domain:name>dmnr-test-1234.com</domain:name>
//     </domain:check>
//     </check>
//     <clTRID>ABC-12345</clTRID>
//   </command>
// </epp>

// <resData>
//  <domain:chkData xmlns:domain="urn:ietf:params:xml:ns:domain-1.0" xsi:schemaLocation="urn:ietf:params:xml:ns:domain-1.0 domain-1.0.xsd">
//   <domain:cd>
//    <domain:name avail="0">ace.pizza</domain:name>
//    <domain:reason>Premium Domain Name</domain:reason>
//   </domain:cd>
//  </domain:chkData>
// </resData>

// CheckDomain queries the EPP server for the availability status of one or more domains.
func (c *Conn) CheckDomain(domains ...string) (*DomainCheck, error) {
	req := message{
		Command: &command{
			Check: &check{
				DomainCheck: &domainCheck{
					Domains: domains,
				},
			},
		},
	}
	err := c.writeMessage(&req)
	if err != nil {
		return nil, err
	}
	msg := message{}
	err = c.readMessage(&msg)
	if err != nil {
		return nil, err
	}
	res := msg.Response
	if res == nil || res.ResponseData.DomainCheckData == nil {
		return nil, ErrResponseMalformed
	}
	dc := DomainCheck(*res.ResponseData.DomainCheckData)
	return &dc, nil
}

// DomainCheck exists for backwards compatibility.
// FIXME: remove/improve this.
type DomainCheck domainCheckData

func decodeCheckDomainResponse(d *Decoder) (*domainCheckData, error) {
	d.Reset()
	data := &domainCheckData{}
	for {
		t, err := d.Token()
		if err != nil && err != io.EOF {
			return nil, err
		}
		if t == nil {
			break
		}
		switch node := t.(type) {
		case xml.StartElement:
			fmt.Printf("StartElement: %s %s\n", node.Name.Space, node.Name.Local)
		case xml.EndElement:
			fmt.Printf("EndElement: %s %s\n", node.Name.Space, node.Name.Local)
		case xml.CharData:
			fmt.Printf("CharData: %s\n", string(node))
		}
		fmt.Printf("Stack: %+v\n", d.Stack)
	}
	return data, nil
}
<commit_msg>Silence spike logging.<commit_after>package epp

import "io"

// <epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
//   <command>
//     <check>
//     <domain:check xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
//       <domain:name>dmnr-test-1234.com</domain:name>
//     </domain:check>
//     </check>
//     <clTRID>ABC-12345</clTRID>
//   </command>
// </epp>

// <resData>
//  <domain:chkData xmlns:domain="urn:ietf:params:xml:ns:domain-1.0" xsi:schemaLocation="urn:ietf:params:xml:ns:domain-1.0 domain-1.0.xsd">
//   <domain:cd>
//    <domain:name avail="0">ace.pizza</domain:name>
//    <domain:reason>Premium Domain Name</domain:reason>
//   </domain:cd>
//  </domain:chkData>
// </resData>

// CheckDomain queries the EPP server for the availability status of one or more domains.
func (c *Conn) CheckDomain(domains ...string) (*DomainCheck, error) {
	req := message{
		Command: &command{
			Check: &check{
				DomainCheck: &domainCheck{
					Domains: domains,
				},
			},
		},
	}
	err := c.writeMessage(&req)
	if err != nil {
		return nil, err
	}
	msg := message{}
	err = c.readMessage(&msg)
	if err != nil {
		return nil, err
	}
	res := msg.Response
	if res == nil || res.ResponseData.DomainCheckData == nil {
		return nil, ErrResponseMalformed
	}
	dc := DomainCheck(*res.ResponseData.DomainCheckData)
	return &dc, nil
}

// DomainCheck exists for backwards compatibility.
// FIXME: remove/improve this.
type DomainCheck domainCheckData

func decodeCheckDomainResponse(d *Decoder) (*domainCheckData, error) {
	d.Reset()
	data := &domainCheckData{}
	for {
		t, err := d.Token()
		if err != nil && err != io.EOF {
			return nil, err
		}
		if t == nil {
			break
		}
		// switch node := t.(type) {
		// case xml.StartElement:
		// 	fmt.Printf("StartElement: %s %s\n", node.Name.Space, node.Name.Local)
		// case xml.EndElement:
		// 	fmt.Printf("EndElement: %s %s\n", node.Name.Space, node.Name.Local)
		// case xml.CharData:
		// 	fmt.Printf("CharData: %s\n", string(node))
		// }
		// fmt.Printf("Stack: %+v\n", d.Stack)
	}
	return data, nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/importer"
	"go/parser"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

func typesMatch(wanted, got []types.Type) bool {
	if len(wanted) != len(got) {
		return false
	}
	for i, w := range wanted {
		g := got[i]
		if !types.ConvertibleTo(g, w) {
			return false
		}
	}
	return true
}

func usedMatch(t types.Type, usedAs []types.Type) bool {
	if len(usedAs) < 1 {
		return true
	}
	for _, u := range usedAs {
		if !types.ConvertibleTo(t, u) {
			return false
		}
	}
	return true
}

func matchesIface(p *param, iface ifaceSign, canEmpty bool) bool {
	if len(p.calls) > len(iface.funcs) {
		return false
	}
	if !usedMatch(iface.t, p.usedAs) {
		return false
	}
	for name, f := range iface.funcs {
		c, e := p.calls[name]
		if !e {
			return canEmpty
		}
		if !typesMatch(f.params, c.params) {
			return false
		}
		if !typesMatch(f.results, c.results) {
			return false
		}
	}
	for _, to := range p.assigned {
		if to.discard {
			return false
		}
		if !matchesIface(to, iface, true) {
			return false
		}
	}
	return true
}

func isFunc(sign *types.Signature, fsign funcSign) bool {
	params := sign.Params()
	if params.Len() != len(fsign.params) {
		return false
	}
	results := sign.Results()
	if results.Len() != len(fsign.results) {
		return false
	}
	for i, p := range fsign.params {
		ip := params.At(i).Type()
		if p.String() != ip.String() {
			return false
		}
	}
	for i, r := range fsign.results {
		ir := results.At(i).Type()
		if r.String() != ir.String() {
			return false
		}
	}
	return true
}

func implementsIface(sign *types.Signature) bool {
	for _, iface := range ifaces {
		for _, f := range iface.funcs {
			if isFunc(sign, f) {
				return true
			}
		}
	}
	return false
}

func interfaceMatching(p *param) string {
	for name, iface := range ifaces {
		if matchesIface(p, iface, false) {
			return name
		}
	}
	return ""
}

var skipDir = regexp.MustCompile(`^(testdata|vendor|_.*|\.\+)$`)

func getDirs(d string) ([]string, error) {
	var dirs []string
	if d != "." && !strings.HasPrefix(d, "./") {
		return nil, fmt.Errorf("TODO: recursing into non-local import paths")
	}
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if skipDir.MatchString(info.Name()) {
			return filepath.SkipDir
		}
		if info.IsDir() {
			if !strings.HasPrefix(path, "./") {
				path = "./" + path
			}
			dirs = append(dirs, path)
		}
		return nil
	}
	if err := filepath.Walk(d, walkFn); err != nil {
		return nil, err
	}
	return dirs, nil
}

func fileExists(path string) (bool, error) {
	info, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return !info.IsDir(), nil
}

func getPkgs(paths []string) ([]*build.Package, []string, error) {
	ex, err := fileExists(paths[0])
	if err != nil {
		return nil, nil, err
	}
	if ex {
		pkg := &build.Package{
			Name:    ".",
			GoFiles: paths,
		}
		return []*build.Package{pkg}, []string{"."}, nil
	}
	var pkgs []*build.Package
	var basedirs []string
	wd, err := os.Getwd()
	if err != nil {
		return nil, nil, err
	}
	for _, p := range paths {
		var dirs []string
		recursive := filepath.Base(p) == "..."
		if !recursive {
			dirs = []string{p}
		} else {
			d := p[:len(p)-4]
			var err error
			dirs, err = getDirs(d)
			if err != nil {
				return nil, nil, err
			}
		}
		for _, d := range dirs {
			pkg, err := build.Import(d, wd, 0)
			if _, ok := err.(*build.NoGoError); ok {
				continue
			}
			if err != nil {
				return nil, nil, err
			}
			pkgs = append(pkgs, pkg)
			basedirs = append(basedirs, d)
		}
	}
	return pkgs, basedirs, nil
}

func checkPaths(paths []string, w io.Writer) error {
	if paths == nil {
		return fmt.Errorf("No paths given")
	}
	conf := &types.Config{Importer: importer.Default()}
	pkgs, basedirs, err := getPkgs(paths)
	if err != nil {
		return err
	}
	for i, pkg := range pkgs {
		basedir := basedirs[i]
		if err := checkPkg(conf, pkg, basedir, w); err != nil {
			return err
		}
	}
	return nil
}

func checkPkg(conf *types.Config, pkg *build.Package, basedir string, w io.Writer) error {
	if *verbose {
		importPath := pkg.ImportPath
		if importPath == "" {
			importPath = "command-line-arguments"
		}
		fmt.Fprintln(w, importPath)
	}
	gp := &goPkg{
		Package: pkg,
		fset:    token.NewFileSet(),
	}
	for _, p := range pkg.GoFiles {
		fp := filepath.Join(basedir, p)
		if err := gp.parsePath(fp); err != nil {
			return err
		}
	}
	if err := gp.check(conf, w); err != nil {
		return err
	}
	return nil
}

type goPkg struct {
	*build.Package

	fset  *token.FileSet
	files []*ast.File
}

func (gp *goPkg) parsePath(fp string) error {
	f, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer f.Close()
	if err := gp.parseReader(fp, f); err != nil {
		return err
	}
	return nil
}

func (gp *goPkg) parseReader(name string, r io.Reader) error {
	f, err := parser.ParseFile(gp.fset, name, r, 0)
	if err != nil {
		return err
	}
	gp.files = append(gp.files, f)
	return nil
}

func (gp *goPkg) check(conf *types.Config, w io.Writer) error {
	info := &types.Info{
		Types: make(map[ast.Expr]types.TypeAndValue),
		Defs:  make(map[*ast.Ident]types.Object),
		Uses:  make(map[*ast.Ident]types.Object),
	}
	pkg, err := conf.Check(gp.Name, gp.fset, gp.files, info)
	if err != nil {
		return err
	}
	grabFromScope(pkg.Scope(), false)
	for _, ipkg := range pkg.Imports() {
		grabFromScope(ipkg.Scope(), false)
	}
	v := &Visitor{
		Info: info,
		w:    w,
		fset: gp.fset,
	}
	for _, f := range gp.files {
		ast.Walk(v, f)
	}
	return nil
}

type param struct {
	t types.Type

	calls   map[string]funcSign
	usedAs  []types.Type
	discard bool

	assigned []*param
}

type Visitor struct {
	*types.Info

	w     io.Writer
	fset  *token.FileSet
	nodes []ast.Node

	params  map[string]*param
	extras  map[string]*param
	inBlock bool

	skipNext bool
}

func (v *Visitor) top() ast.Node {
	if len(v.nodes) == 0 {
		return nil
	}
	return v.nodes[len(v.nodes)-1]
}

func paramsMap(t *types.Tuple) map[string]*param {
	m := make(map[string]*param, t.Len())
	for i := 0; i < t.Len(); i++ {
		p := t.At(i)
		m[p.Name()] = &param{
			t:     p.Type(),
			calls: make(map[string]funcSign),
		}
	}
	return m
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *Visitor) param(name string) *param {
	if p, e := v.params[name]; e {
		return p
	}
	if p, e := v.extras[name]; e {
		return p
	}
	p := &param{
		calls: make(map[string]funcSign),
	}
	v.extras[name] = p
	return p
}

func (v *Visitor) addUsed(name string, as types.Type) {
	if as == nil {
		return
	}
	p := v.param(name)
	p.usedAs = append(p.usedAs, as)
}

func (v *Visitor) addAssign(to, from string) {
	pto := v.param(to)
	pfrom := v.param(from)
	pfrom.assigned = append(pfrom.assigned, pto)
}

func (v *Visitor) discard(name string) {
	p := v.param(name)
	p.discard = true
}

func (v *Visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	switch x := node.(type) {
	case *ast.FuncDecl:
		sign := v.Defs[x.Name].Type().(*types.Signature)
		if implementsIface(sign) {
			return nil
		}
		v.params = paramsMap(sign.Params())
		v.extras = make(map[string]*param)
	case *ast.BlockStmt:
		if v.params != nil {
			v.inBlock = true
		}
	case *ast.SelectorExpr:
		if !v.inBlock {
			return nil
		}
		v.onSelector(x)
	case *ast.AssignStmt:
		if !v.inBlock {
			return nil
		}
		for i, e := range x.Rhs {
			id, ok := e.(*ast.Ident)
			if !ok {
				continue
			}
			left := x.Lhs[i]
			v.addUsed(id.Name, v.Types[left].Type)
			if lid, ok := left.(*ast.Ident); ok {
				v.addAssign(lid.Name, id.Name)
			}
		}
	case *ast.CallExpr:
		if !v.inBlock {
			return nil
		}
		v.onCall(x)
		switch y := x.Fun.(type) {
		case *ast.Ident:
			v.skipNext = true
		case *ast.SelectorExpr:
			if _, ok := y.X.(*ast.Ident); ok {
				v.skipNext = true
			}
		}
	case nil:
		if fd, ok := v.top().(*ast.FuncDecl); ok {
			v.funcEnded(fd.Pos())
			v.params = nil
			v.extras = nil
			v.inBlock = false
		}
		v.nodes = v.nodes[:len(v.nodes)-1]
	}
	if node != nil {
		v.nodes = append(v.nodes, node)
	}
	return v
}

func funcSignature(t types.Type) *types.Signature {
	switch x := t.(type) {
	case *types.Signature:
		return x
	case *types.Named:
		return funcSignature(x.Underlying())
	default:
		return nil
	}
}

func (v *Visitor) onCall(ce *ast.CallExpr) {
	sign := funcSignature(v.Types[ce.Fun].Type)
	if sign == nil {
		return
	}
	for i, e := range ce.Args {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		v.addUsed(id.Name, paramType(sign, i))
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(left.Name)
	c := funcSign{}
	results := sign.Results()
	for i := 0; i < results.Len(); i++ {
		c.results = append(c.results, results.At(i).Type())
	}
	for _, a := range ce.Args {
		c.params = append(c.params, v.Types[a].Type)
	}
	p.calls[sel.Sel.Name] = c
	return
}

func (v *Visitor) onSelector(sel *ast.SelectorExpr) {
	id, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	v.discard(id.Name)
}

func (v *Visitor) funcEnded(pos token.Pos) {
	for name, p := range v.params {
		if p.discard {
			continue
		}
		iface := interfaceMatching(p)
		if iface == "" {
			continue
		}
		if iface == p.t.String() {
			continue
		}
		pos := v.fset.Position(pos)
		fmt.Fprintf(v.w, "%s:%d: %s can be %s\n",
			pos.Filename, pos.Line, name, iface)
	}
}
<commit_msg>Remove useless length check<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/importer"
	"go/parser"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

func typesMatch(wanted, got []types.Type) bool {
	if len(wanted) != len(got) {
		return false
	}
	for i, w := range wanted {
		g := got[i]
		if !types.ConvertibleTo(g, w) {
			return false
		}
	}
	return true
}

func usedMatch(t types.Type, usedAs []types.Type) bool {
	for _, u := range usedAs {
		if !types.ConvertibleTo(t, u) {
			return false
		}
	}
	return true
}

func matchesIface(p *param, iface ifaceSign, canEmpty bool) bool {
	if len(p.calls) > len(iface.funcs) {
		return false
	}
	if !usedMatch(iface.t, p.usedAs) {
		return false
	}
	for name, f := range iface.funcs {
		c, e := p.calls[name]
		if !e {
			return canEmpty
		}
		if !typesMatch(f.params, c.params) {
			return false
		}
		if !typesMatch(f.results, c.results) {
			return false
		}
	}
	for _, to := range p.assigned {
		if to.discard {
			return false
		}
		if !matchesIface(to, iface, true) {
			return false
		}
	}
	return true
}

func isFunc(sign *types.Signature, fsign funcSign) bool {
	params := sign.Params()
	if params.Len() != len(fsign.params) {
		return false
	}
	results := sign.Results()
	if results.Len() != len(fsign.results) {
		return false
	}
	for i, p := range fsign.params {
		ip := params.At(i).Type()
		if p.String() != ip.String() {
			return false
		}
	}
	for i, r := range fsign.results {
		ir := results.At(i).Type()
		if r.String() != ir.String() {
			return false
		}
	}
	return true
}

func implementsIface(sign *types.Signature) bool {
	for _, iface := range ifaces {
		for _, f := range iface.funcs {
			if isFunc(sign, f) {
				return true
			}
		}
	}
	return false
}

func interfaceMatching(p *param) string {
	for name, iface := range ifaces {
		if matchesIface(p, iface, false) {
			return name
		}
	}
	return ""
}

var skipDir = regexp.MustCompile(`^(testdata|vendor|_.*|\.\+)$`)

func getDirs(d string) ([]string, error) {
	var dirs []string
	if d != "." && !strings.HasPrefix(d, "./") {
		return nil, fmt.Errorf("TODO: recursing into non-local import paths")
	}
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if skipDir.MatchString(info.Name()) {
			return filepath.SkipDir
		}
		if info.IsDir() {
			if !strings.HasPrefix(path, "./") {
				path = "./" + path
			}
			dirs = append(dirs, path)
		}
		return nil
	}
	if err := filepath.Walk(d, walkFn); err != nil {
		return nil, err
	}
	return dirs, nil
}

func fileExists(path string) (bool, error) {
	info, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return !info.IsDir(), nil
}

func getPkgs(paths []string) ([]*build.Package, []string, error) {
	ex, err := fileExists(paths[0])
	if err != nil {
		return nil, nil, err
	}
	if ex {
		pkg := &build.Package{
			Name:    ".",
			GoFiles: paths,
		}
		return []*build.Package{pkg}, []string{"."}, nil
	}
	var pkgs []*build.Package
	var basedirs []string
	wd, err := os.Getwd()
	if err != nil {
		return nil, nil, err
	}
	for _, p := range paths {
		var dirs []string
		recursive := filepath.Base(p) == "..."
		if !recursive {
			dirs = []string{p}
		} else {
			d := p[:len(p)-4]
			var err error
			dirs, err = getDirs(d)
			if err != nil {
				return nil, nil, err
			}
		}
		for _, d := range dirs {
			pkg, err := build.Import(d, wd, 0)
			if _, ok := err.(*build.NoGoError); ok {
				continue
			}
			if err != nil {
				return nil, nil, err
			}
			pkgs = append(pkgs, pkg)
			basedirs = append(basedirs, d)
		}
	}
	return pkgs, basedirs, nil
}

func checkPaths(paths []string, w io.Writer) error {
	if paths == nil {
		return fmt.Errorf("No paths given")
	}
	conf := &types.Config{Importer: importer.Default()}
	pkgs, basedirs, err := getPkgs(paths)
	if err != nil {
		return err
	}
	for i, pkg := range pkgs {
		basedir := basedirs[i]
		if err := checkPkg(conf, pkg, basedir, w); err != nil {
			return err
		}
	}
	return nil
}

func checkPkg(conf *types.Config, pkg *build.Package, basedir string, w io.Writer) error {
	if *verbose {
		importPath := pkg.ImportPath
		if importPath == "" {
			importPath = "command-line-arguments"
		}
		fmt.Fprintln(w, importPath)
	}
	gp := &goPkg{
		Package: pkg,
		fset:    token.NewFileSet(),
	}
	for _, p := range pkg.GoFiles {
		fp := filepath.Join(basedir, p)
		if err := gp.parsePath(fp); err != nil {
			return err
		}
	}
	if err := gp.check(conf, w); err != nil {
		return err
	}
	return nil
}

type goPkg struct {
	*build.Package

	fset  *token.FileSet
	files []*ast.File
}

func (gp *goPkg) parsePath(fp string) error {
	f, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer f.Close()
	if err := gp.parseReader(fp, f); err != nil {
		return err
	}
	return nil
}

func (gp *goPkg) parseReader(name string, r io.Reader) error {
	f, err := parser.ParseFile(gp.fset, name, r, 0)
	if err != nil {
		return err
	}
	gp.files = append(gp.files, f)
	return nil
}

func (gp *goPkg) check(conf *types.Config, w io.Writer) error {
	info := &types.Info{
		Types: make(map[ast.Expr]types.TypeAndValue),
		Defs:  make(map[*ast.Ident]types.Object),
		Uses:  make(map[*ast.Ident]types.Object),
	}
	pkg, err := conf.Check(gp.Name, gp.fset, gp.files, info)
	if err != nil {
		return err
	}
	grabFromScope(pkg.Scope(), false)
	for _, ipkg := range pkg.Imports() {
		grabFromScope(ipkg.Scope(), false)
	}
	v := &Visitor{
		Info: info,
		w:    w,
		fset: gp.fset,
	}
	for _, f := range gp.files {
		ast.Walk(v, f)
	}
	return nil
}

type param struct {
	t types.Type

	calls   map[string]funcSign
	usedAs  []types.Type
	discard bool

	assigned []*param
}

type Visitor struct {
	*types.Info

	w     io.Writer
	fset  *token.FileSet
	nodes []ast.Node

	params  map[string]*param
	extras  map[string]*param
	inBlock bool

	skipNext bool
}

func (v *Visitor) top() ast.Node {
	if len(v.nodes) == 0 {
		return nil
	}
	return v.nodes[len(v.nodes)-1]
}

func paramsMap(t *types.Tuple) map[string]*param {
	m := make(map[string]*param, t.Len())
	for i := 0; i < t.Len(); i++ {
		p := t.At(i)
		m[p.Name()] = &param{
			t:     p.Type(),
			calls: make(map[string]funcSign),
		}
	}
	return m
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *Visitor) param(name string) *param {
	if p, e := v.params[name]; e {
		return p
	}
	if p, e := v.extras[name]; e {
		return p
	}
	p := &param{
		calls: make(map[string]funcSign),
	}
	v.extras[name] = p
	return p
}

func (v *Visitor) addUsed(name string, as types.Type) {
	if as == nil {
		return
	}
	p := v.param(name)
	p.usedAs = append(p.usedAs, as)
}

func (v *Visitor) addAssign(to, from string) {
	pto := v.param(to)
	pfrom := v.param(from)
	pfrom.assigned = append(pfrom.assigned, pto)
}

func (v *Visitor) discard(name string) {
	p := v.param(name)
	p.discard = true
}

func (v *Visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	switch x := node.(type) {
	case *ast.FuncDecl:
		sign := v.Defs[x.Name].Type().(*types.Signature)
		if implementsIface(sign) {
			return nil
		}
		v.params = paramsMap(sign.Params())
		v.extras = make(map[string]*param)
	case *ast.BlockStmt:
		if v.params != nil {
			v.inBlock = true
		}
	case *ast.SelectorExpr:
		if !v.inBlock {
			return nil
		}
		v.onSelector(x)
	case *ast.AssignStmt:
		if !v.inBlock {
			return nil
		}
		for i, e := range x.Rhs {
			id, ok := e.(*ast.Ident)
			if !ok {
				continue
			}
			left := x.Lhs[i]
			v.addUsed(id.Name, v.Types[left].Type)
			if lid, ok := left.(*ast.Ident); ok {
				v.addAssign(lid.Name, id.Name)
			}
		}
	case *ast.CallExpr:
		if !v.inBlock {
			return nil
		}
		v.onCall(x)
		switch y := x.Fun.(type) {
		case *ast.Ident:
			v.skipNext = true
		case *ast.SelectorExpr:
			if _, ok := y.X.(*ast.Ident); ok {
				v.skipNext = true
			}
		}
	case nil:
		if fd, ok := v.top().(*ast.FuncDecl); ok {
			v.funcEnded(fd.Pos())
			v.params = nil
			v.extras = nil
			v.inBlock = false
		}
		v.nodes = v.nodes[:len(v.nodes)-1]
	}
	if node != nil {
		v.nodes = append(v.nodes, node)
	}
	return v
}

func funcSignature(t types.Type) *types.Signature {
	switch x := t.(type) {
	case *types.Signature:
		return x
	case *types.Named:
		return funcSignature(x.Underlying())
	default:
		return nil
	}
}

func (v *Visitor) onCall(ce *ast.CallExpr) {
	sign := funcSignature(v.Types[ce.Fun].Type)
	if sign == nil {
		return
	}
	for i, e := range ce.Args {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		v.addUsed(id.Name, paramType(sign, i))
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(left.Name)
	c := funcSign{}
	results := sign.Results()
	for i := 0; i < results.Len(); i++ {
		c.results = append(c.results, results.At(i).Type())
	}
	for _, a := range ce.Args {
		c.params = append(c.params, v.Types[a].Type)
	}
	p.calls[sel.Sel.Name] = c
	return
}

func (v *Visitor) onSelector(sel *ast.SelectorExpr) {
	id, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	v.discard(id.Name)
}

func (v *Visitor) funcEnded(pos token.Pos) {
	for name, p := range v.params {
		if p.discard {
			continue
		}
		iface := interfaceMatching(p)
		if iface == "" {
			continue
		}
		if iface == p.t.String() {
			continue
		}
		pos := v.fset.Position(pos)
		fmt.Fprintf(v.w, "%s:%d: %s can be %s\n",
			pos.Filename, pos.Line, name, iface)
	}
}
<|endoftext|>