<commit_before>// Copyright (c) 2021 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package limits

import (
	"fmt"
	"sync"
	"time"

	"github.com/uber-go/tally"
	"go.uber.org/atomic"
	"go.uber.org/zap"

	xerrors "github.com/m3db/m3/src/x/errors"
	"github.com/m3db/m3/src/x/instrument"
)

const (
	disabledLimitValue = 0
	defaultLookback    = time.Second * 15
)

type queryLimits struct {
	docsLimit           *lookbackLimit
	bytesReadLimit      *lookbackLimit
	aggregatedDocsLimit *lookbackLimit
}

type lookbackLimit struct {
	name      string
	started   bool
	options   LookbackLimitOptions
	metrics   lookbackLimitMetrics
	logger    *zap.Logger
	recent    *atomic.Int64
	stopCh    chan struct{}
	stoppedCh chan struct{}
	lock      sync.RWMutex
	iOpts     instrument.Options
}

type lookbackLimitMetrics struct {
	optionsLimit    tally.Gauge
	optionsLookback tally.Gauge
	recentCount     tally.Gauge
	recentMax       tally.Gauge
	total           tally.Counter
	exceeded        tally.Counter

	sourceLogger SourceLogger
}

var (
	_ QueryLimits   = (*queryLimits)(nil)
	_ LookbackLimit = (*lookbackLimit)(nil)
)

// DefaultLookbackLimitOptions returns a new query limits manager.
func DefaultLookbackLimitOptions() LookbackLimitOptions {
	return LookbackLimitOptions{
		// Default to no limit.
		Limit:    disabledLimitValue,
		Lookback: defaultLookback,
	}
}

// NewQueryLimits returns a new query limits manager.
func NewQueryLimits(options Options) (QueryLimits, error) {
	if err := options.Validate(); err != nil {
		return nil, err
	}

	var (
		iOpts               = options.InstrumentOptions()
		docsLimitOpts       = options.DocsLimitOpts()
		bytesReadLimitOpts  = options.BytesReadLimitOpts()
		aggDocsLimitOpts    = options.AggregateDocsLimitOpts()
		sourceLoggerBuilder = options.SourceLoggerBuilder()

		docsMatched      = "docs-matched"
		bytesRead        = "disk-bytes-read"
		aggregateMatched = "aggregate-matched"
		docsLimit        = newLookbackLimit(
			iOpts, docsLimitOpts, docsMatched, docsMatched,
			sourceLoggerBuilder, map[string]string{"type": "fetch"})
		bytesReadLimit = newLookbackLimit(
			iOpts, bytesReadLimitOpts, bytesRead, bytesRead,
			sourceLoggerBuilder, nil)

		aggregatedDocsLimit = newLookbackLimit(
			iOpts, aggDocsLimitOpts, docsMatched, aggregateMatched,
			sourceLoggerBuilder, map[string]string{"type": "aggregate"})
	)

	return &queryLimits{
		docsLimit:           docsLimit,
		bytesReadLimit:      bytesReadLimit,
		aggregatedDocsLimit: aggregatedDocsLimit,
	}, nil
}

// NewLookbackLimit returns a new lookback limit.
func NewLookbackLimit(
	instrumentOpts instrument.Options,
	opts LookbackLimitOptions,
	name string,
	sourceLoggerBuilder SourceLoggerBuilder,
	tags map[string]string,
) LookbackLimit {
	return newLookbackLimit(instrumentOpts, opts, name, name, sourceLoggerBuilder, tags)
}

func newLookbackLimit(
	instrumentOpts instrument.Options,
	opts LookbackLimitOptions,
	metricName string,
	name string,
	sourceLoggerBuilder SourceLoggerBuilder,
	tags map[string]string,
) *lookbackLimit {
	metrics := newLookbackLimitMetrics(
		instrumentOpts,
		metricName,
		sourceLoggerBuilder,
		tags,
	)

	return &lookbackLimit{
		name:      name,
		options:   opts,
		metrics:   metrics,
		logger:    instrumentOpts.Logger(),
		recent:    atomic.NewInt64(0),
		stopCh:    make(chan struct{}),
		stoppedCh: make(chan struct{}),
		iOpts:     instrumentOpts,
	}
}

func newLookbackLimitMetrics(
	instrumentOpts instrument.Options,
	name string,
	sourceLoggerBuilder SourceLoggerBuilder,
	tags map[string]string,
) lookbackLimitMetrics {
	scope := instrumentOpts.
		MetricsScope().
		SubScope("query-limit")

	if tags != nil {
		scope = scope.Tagged(tags)
	}

	return lookbackLimitMetrics{
		optionsLimit:    scope.Gauge(fmt.Sprintf("current-limit%s", name)),
		optionsLookback: scope.Gauge(fmt.Sprintf("current-lookback-%s", name)),
		recentCount:     scope.Gauge(fmt.Sprintf("recent-count-%s", name)),
		recentMax:       scope.Gauge(fmt.Sprintf("recent-max-%s", name)),
		total:           scope.Counter(fmt.Sprintf("total-%s", name)),
		exceeded:        scope.Tagged(map[string]string{"limit": name}).Counter("exceeded"),

		// nb: no need to provide query-limits subscope to source logger,
		// as it's not directly related to limits.
		sourceLogger: sourceLoggerBuilder.NewSourceLogger(name, instrumentOpts),
	}
}

func (q *queryLimits) FetchDocsLimit() LookbackLimit {
	return q.docsLimit
}

func (q *queryLimits) BytesReadLimit() LookbackLimit {
	return q.bytesReadLimit
}

func (q *queryLimits) AggregateDocsLimit() LookbackLimit {
	return q.aggregatedDocsLimit
}

func (q *queryLimits) Start() {
	q.docsLimit.Start()
	q.bytesReadLimit.Start()
	q.aggregatedDocsLimit.Start()
}

func (q *queryLimits) Stop() {
	q.docsLimit.Stop()
	q.bytesReadLimit.Stop()
	q.aggregatedDocsLimit.Stop()
}

func (q *queryLimits) AnyFetchExceeded() error {
	if err := q.docsLimit.exceeded(); err != nil {
		return err
	}

	return q.bytesReadLimit.exceeded()
}

func (q *lookbackLimit) Options() LookbackLimitOptions {
	q.lock.RLock()
	o := q.options
	q.lock.RUnlock()
	return o
}

// Update updates the limit.
func (q *lookbackLimit) Update(opts LookbackLimitOptions) error {
	if err := opts.validate(); err != nil {
		return err
	}

	q.lock.Lock()
	defer q.lock.Unlock()

	old := q.options
	q.options = opts

	// If the lookback changed, replace the background goroutine that manages the periodic resetting.
	if q.options.Lookback != old.Lookback {
		q.stop()
		q.start()
	}

	q.logger.Info("query limit options updated",
		zap.String("name", q.name),
		zap.Any("new", opts),
		zap.Any("old", old))

	return nil
}

// Inc increments the current value and returns an error if above the limit.
func (q *lookbackLimit) Inc(val int, source []byte) error {
	if val < 0 {
		return fmt.Errorf("invalid negative query limit inc %d", val)
	}
	if val == 0 {
		return q.exceeded()
	}

	// Add the new stats to the global state.
	valI64 := int64(val)
	recent := q.recent.Add(valI64)

	// Update metrics.
	q.metrics.recentCount.Update(float64(recent))
	q.metrics.total.Inc(valI64)
	q.metrics.sourceLogger.LogSourceValue(valI64, source)

	// Enforce limit (if specified).
	return q.checkLimit(recent)
}

func (q *lookbackLimit) exceeded() error {
	return q.checkLimit(q.recent.Load())
}

func (q *lookbackLimit) checkLimit(recent int64) error {
	q.lock.RLock()
	currentOpts := q.options
	q.lock.RUnlock()

	if currentOpts.ForceExceeded {
		q.metrics.exceeded.Inc(1)

		return xerrors.NewInvalidParamsError(NewQueryLimitExceededError(fmt.Sprintf(
			"query aborted due to forced limit: name=%s", q.name)))
	}

	if currentOpts.Limit == disabledLimitValue {
		return nil
	}

	if recent >= currentOpts.Limit {
		q.metrics.exceeded.Inc(1)

		return xerrors.NewInvalidParamsError(NewQueryLimitExceededError(fmt.Sprintf(
			"query aborted due to limit: name=%s, limit=%d, current=%d, within=%s",
			q.name, q.options.Limit, recent, q.options.Lookback)))
	}

	return nil
}

func (q *lookbackLimit) Start() {
	// Lock on explicit start to avoid any collision with asynchronous updating
	// which will call stop/start if the lookback has changed.
	q.lock.Lock()
	defer q.lock.Unlock()
	q.start()
}

func (q *lookbackLimit) Stop() {
	// Lock on explicit stop to avoid any collision with asynchronous updating
	// which will call stop/start if the lookback has changed.
	q.lock.Lock()
	defer q.lock.Unlock()
	q.stop()
}

func (q *lookbackLimit) start() {
	q.started = true
	ticker := time.NewTicker(q.options.Lookback)
	go func() {
		q.logger.Info("query limit interval started", zap.String("name", q.name))
		for {
			select {
			case <-ticker.C:
				q.reset()
			case <-q.stopCh:
				ticker.Stop()
				q.stoppedCh <- struct{}{}
				return
			}
		}
	}()

	q.metrics.optionsLimit.Update(float64(q.options.Limit))
	q.metrics.optionsLookback.Update(q.options.Lookback.Seconds())
}

func (q *lookbackLimit) stop() {
	if !q.started {
		// NB: this lookback limit has not yet been started.
		instrument.EmitAndLogInvariantViolation(q.iOpts, func(l *zap.Logger) {
			l.With(
				zap.Any("limit_name", q.name),
			).Error("cannot stop non-started lookback limit")
		})
		return
	}

	close(q.stopCh)
	<-q.stoppedCh
	q.stopCh = make(chan struct{})
	q.stoppedCh = make(chan struct{})

	q.logger.Info("query limit interval stopped", zap.String("name", q.name))
}

func (q *lookbackLimit) current() int64 {
	return q.recent.Load()
}

func (q *lookbackLimit) reset() {
	// Update peak gauge only on resets so it only tracks
	// the peak values for each lookback period.
	recent := q.recent.Load()

	q.metrics.recentMax.Update(float64(recent))
	// Update the standard recent gauge to reflect drop back to zero.
	q.metrics.recentCount.Update(0)
	q.recent.Store(0)
}

// Equals returns true if the other options match the current.
func (opts LookbackLimitOptions) Equals(other LookbackLimitOptions) bool {
	return opts.Limit == other.Limit &&
		opts.Lookback == other.Lookback &&
		opts.ForceExceeded == other.ForceExceeded
}

func (opts LookbackLimitOptions) validate() error {
	if opts.Limit < 0 {
		return fmt.Errorf("query limit requires limit >= 0 (%d)", opts.Limit)
	}
	if opts.Lookback <= 0 {
		return fmt.Errorf("query limit requires lookback > 0 (%d)", opts.Lookback)
	}
	return nil
}
<commit_msg>[instrumentation] Clean up some metric names (#3241)<commit_after>// Copyright (c) 2021 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package limits

import (
	"fmt"
	"sync"
	"time"

	"github.com/uber-go/tally"
	"go.uber.org/atomic"
	"go.uber.org/zap"

	xerrors "github.com/m3db/m3/src/x/errors"
	"github.com/m3db/m3/src/x/instrument"
)

const (
	disabledLimitValue = 0
	defaultLookback    = time.Second * 15
)

type queryLimits struct {
	docsLimit           *lookbackLimit
	bytesReadLimit      *lookbackLimit
	aggregatedDocsLimit *lookbackLimit
}

type lookbackLimit struct {
	name      string
	started   bool
	options   LookbackLimitOptions
	metrics   lookbackLimitMetrics
	logger    *zap.Logger
	recent    *atomic.Int64
	stopCh    chan struct{}
	stoppedCh chan struct{}
	lock      sync.RWMutex
	iOpts     instrument.Options
}

type lookbackLimitMetrics struct {
	optionsLimit    tally.Gauge
	optionsLookback tally.Gauge
	recentCount     tally.Gauge
	recentMax       tally.Gauge
	total           tally.Counter
	exceeded        tally.Counter

	sourceLogger SourceLogger
}

var (
	_ QueryLimits   = (*queryLimits)(nil)
	_ LookbackLimit = (*lookbackLimit)(nil)
)

// DefaultLookbackLimitOptions returns a new query limits manager.
func DefaultLookbackLimitOptions() LookbackLimitOptions {
	return LookbackLimitOptions{
		// Default to no limit.
		Limit:    disabledLimitValue,
		Lookback: defaultLookback,
	}
}

// NewQueryLimits returns a new query limits manager.
func NewQueryLimits(options Options) (QueryLimits, error) {
	if err := options.Validate(); err != nil {
		return nil, err
	}

	var (
		iOpts               = options.InstrumentOptions()
		docsLimitOpts       = options.DocsLimitOpts()
		bytesReadLimitOpts  = options.BytesReadLimitOpts()
		aggDocsLimitOpts    = options.AggregateDocsLimitOpts()
		sourceLoggerBuilder = options.SourceLoggerBuilder()

		docsMatched      = "docs-matched"
		bytesRead        = "disk-bytes-read"
		aggregateMatched = "aggregate-matched"
		docsLimit        = newLookbackLimit(
			iOpts, docsLimitOpts, docsMatched, docsMatched,
			sourceLoggerBuilder, map[string]string{"type": "fetch"})
		bytesReadLimit = newLookbackLimit(
			iOpts, bytesReadLimitOpts, bytesRead, bytesRead,
			sourceLoggerBuilder, nil)

		aggregatedDocsLimit = newLookbackLimit(
			iOpts, aggDocsLimitOpts, docsMatched, aggregateMatched,
			sourceLoggerBuilder, map[string]string{"type": "aggregate"})
	)

	return &queryLimits{
		docsLimit:           docsLimit,
		bytesReadLimit:      bytesReadLimit,
		aggregatedDocsLimit: aggregatedDocsLimit,
	}, nil
}

// NewLookbackLimit returns a new lookback limit.
func NewLookbackLimit(
	instrumentOpts instrument.Options,
	opts LookbackLimitOptions,
	name string,
	sourceLoggerBuilder SourceLoggerBuilder,
	tags map[string]string,
) LookbackLimit {
	return newLookbackLimit(instrumentOpts, opts, name, name, sourceLoggerBuilder, tags)
}

func newLookbackLimit(
	instrumentOpts instrument.Options,
	opts LookbackLimitOptions,
	metricName string,
	name string,
	sourceLoggerBuilder SourceLoggerBuilder,
	tags map[string]string,
) *lookbackLimit {
	metrics := newLookbackLimitMetrics(
		instrumentOpts,
		metricName,
		sourceLoggerBuilder,
		tags,
	)

	return &lookbackLimit{
		name:      name,
		options:   opts,
		metrics:   metrics,
		logger:    instrumentOpts.Logger(),
		recent:    atomic.NewInt64(0),
		stopCh:    make(chan struct{}),
		stoppedCh: make(chan struct{}),
		iOpts:     instrumentOpts,
	}
}

func newLookbackLimitMetrics(
	instrumentOpts instrument.Options,
	name string,
	sourceLoggerBuilder SourceLoggerBuilder,
	tags map[string]string,
) lookbackLimitMetrics {
	loggerScope := instrumentOpts.MetricsScope()
	if tags != nil {
		loggerScope = loggerScope.Tagged(tags)
	}

	var (
		loggerOpts  = instrumentOpts.SetMetricsScope(loggerScope)
		metricScope = loggerScope.SubScope("query-limit")
	)

	return lookbackLimitMetrics{
		optionsLimit:    metricScope.Gauge(fmt.Sprintf("current-limit-%s", name)),
		optionsLookback: metricScope.Gauge(fmt.Sprintf("current-lookback-%s", name)),
		recentCount:     metricScope.Gauge(fmt.Sprintf("recent-count-%s", name)),
		recentMax:       metricScope.Gauge(fmt.Sprintf("recent-max-%s", name)),
		total:           metricScope.Counter(fmt.Sprintf("total-%s", name)),
		exceeded:        metricScope.Tagged(map[string]string{"limit": name}).Counter("exceeded"),

		sourceLogger: sourceLoggerBuilder.NewSourceLogger(name, loggerOpts),
	}
}

func (q *queryLimits) FetchDocsLimit() LookbackLimit {
	return q.docsLimit
}

func (q *queryLimits) BytesReadLimit() LookbackLimit {
	return q.bytesReadLimit
}

func (q *queryLimits) AggregateDocsLimit() LookbackLimit {
	return q.aggregatedDocsLimit
}

func (q *queryLimits) Start() {
	q.docsLimit.Start()
	q.bytesReadLimit.Start()
	q.aggregatedDocsLimit.Start()
}

func (q *queryLimits) Stop() {
	q.docsLimit.Stop()
	q.bytesReadLimit.Stop()
	q.aggregatedDocsLimit.Stop()
}

func (q *queryLimits) AnyFetchExceeded() error {
	if err := q.docsLimit.exceeded(); err != nil {
		return err
	}

	return q.bytesReadLimit.exceeded()
}

func (q *lookbackLimit) Options() LookbackLimitOptions {
	q.lock.RLock()
	o := q.options
	q.lock.RUnlock()
	return o
}

// Update updates the limit.
func (q *lookbackLimit) Update(opts LookbackLimitOptions) error {
	if err := opts.validate(); err != nil {
		return err
	}

	q.lock.Lock()
	defer q.lock.Unlock()

	old := q.options
	q.options = opts

	// If the lookback changed, replace the background goroutine that manages the periodic resetting.
	if q.options.Lookback != old.Lookback {
		q.stop()
		q.start()
	}

	q.logger.Info("query limit options updated",
		zap.String("name", q.name),
		zap.Any("new", opts),
		zap.Any("old", old))

	return nil
}

// Inc increments the current value and returns an error if above the limit.
func (q *lookbackLimit) Inc(val int, source []byte) error {
	if val < 0 {
		return fmt.Errorf("invalid negative query limit inc %d", val)
	}
	if val == 0 {
		return q.exceeded()
	}

	// Add the new stats to the global state.
	valI64 := int64(val)
	recent := q.recent.Add(valI64)

	// Update metrics.
	q.metrics.recentCount.Update(float64(recent))
	q.metrics.total.Inc(valI64)
	q.metrics.sourceLogger.LogSourceValue(valI64, source)

	// Enforce limit (if specified).
	return q.checkLimit(recent)
}

func (q *lookbackLimit) exceeded() error {
	return q.checkLimit(q.recent.Load())
}

func (q *lookbackLimit) checkLimit(recent int64) error {
	q.lock.RLock()
	currentOpts := q.options
	q.lock.RUnlock()

	if currentOpts.ForceExceeded {
		q.metrics.exceeded.Inc(1)

		return xerrors.NewInvalidParamsError(NewQueryLimitExceededError(fmt.Sprintf(
			"query aborted due to forced limit: name=%s", q.name)))
	}

	if currentOpts.Limit == disabledLimitValue {
		return nil
	}

	if recent >= currentOpts.Limit {
		q.metrics.exceeded.Inc(1)

		return xerrors.NewInvalidParamsError(NewQueryLimitExceededError(fmt.Sprintf(
			"query aborted due to limit: name=%s, limit=%d, current=%d, within=%s",
			q.name, q.options.Limit, recent, q.options.Lookback)))
	}

	return nil
}

func (q *lookbackLimit) Start() {
	// Lock on explicit start to avoid any collision with asynchronous updating
	// which will call stop/start if the lookback has changed.
	q.lock.Lock()
	defer q.lock.Unlock()
	q.start()
}

func (q *lookbackLimit) Stop() {
	// Lock on explicit stop to avoid any collision with asynchronous updating
	// which will call stop/start if the lookback has changed.
	q.lock.Lock()
	defer q.lock.Unlock()
	q.stop()
}

func (q *lookbackLimit) start() {
	q.started = true
	ticker := time.NewTicker(q.options.Lookback)
	go func() {
		q.logger.Info("query limit interval started", zap.String("name", q.name))
		for {
			select {
			case <-ticker.C:
				q.reset()
			case <-q.stopCh:
				ticker.Stop()
				q.stoppedCh <- struct{}{}
				return
			}
		}
	}()

	q.metrics.optionsLimit.Update(float64(q.options.Limit))
	q.metrics.optionsLookback.Update(q.options.Lookback.Seconds())
}

func (q *lookbackLimit) stop() {
	if !q.started {
		// NB: this lookback limit has not yet been started.
		instrument.EmitAndLogInvariantViolation(q.iOpts, func(l *zap.Logger) {
			l.With(
				zap.Any("limit_name", q.name),
			).Error("cannot stop non-started lookback limit")
		})
		return
	}

	close(q.stopCh)
	<-q.stoppedCh
	q.stopCh = make(chan struct{})
	q.stoppedCh = make(chan struct{})

	q.logger.Info("query limit interval stopped", zap.String("name", q.name))
}

func (q *lookbackLimit) current() int64 {
	return q.recent.Load()
}

func (q *lookbackLimit) reset() {
	// Update peak gauge only on resets so it only tracks
	// the peak values for each lookback period.
	recent := q.recent.Load()

	q.metrics.recentMax.Update(float64(recent))
	// Update the standard recent gauge to reflect drop back to zero.
	q.metrics.recentCount.Update(0)
	q.recent.Store(0)
}

// Equals returns true if the other options match the current.
func (opts LookbackLimitOptions) Equals(other LookbackLimitOptions) bool {
	return opts.Limit == other.Limit &&
		opts.Lookback == other.Lookback &&
		opts.ForceExceeded == other.ForceExceeded
}

func (opts LookbackLimitOptions) validate() error {
	if opts.Limit < 0 {
		return fmt.Errorf("query limit requires limit >= 0 (%d)", opts.Limit)
	}
	if opts.Lookback <= 0 {
		return fmt.Errorf("query limit requires lookback > 0 (%d)", opts.Lookback)
	}
	return nil
}
<|endoftext|>


<commit_before>package df2014

type WorldDat struct {
	Header

	Unused000 int16 `df2014_assert_equals:"0" df2014_version_min:"1205"`
	Unused001 int32 `df2014_assert_equals:"0" df2014_version_max:"1169"`

	// SaveSlot is the original region number of this world. A value of 0 means
	// the save will be titled region1. 1 means region2, and so on.
	SaveSlot int32 `df2014_assert_gte:"0" df2014_version_max:"1169"`

	NextIDs WorldNextID

	Name *Name

	// all I know about the next five variables is that they add up to 15 bytes.
	Unk100 int8  `df2014_assert_equals:"1"`
	Unk101 int16 `df2014_assert_equals:"0"`
	Unk102 int32 `df2014_assert_equals:"1"`
	Unk103 int32 `df2014_assert_equals:"0"`
	Unk104 int32 `df2014_assert_equals:"0"`

	Title string `df2014_version_min:"1110"`

	GeneratedRaws WorldGeneratedRaws `df2014_version_min:"1287"`
	StringTables  WorldStringTables

	STOP struct{} `df2014_assert_equals:"STOP" df2014_version_min:"1205"`

	ItemIDs     []int32 `df2014_assert_next_id:"Item"`
	BuildingIDs []int32 `df2014_assert_next_id:"Building"`
	EntityIDs   []int32 `df2014_assert_next_id:"Entity"`
	NemesisIDs  []int32 `df2014_assert_next_id:"Nemesis"`
	ArtifactIDs []int32 `df2014_assert_next_id:"Artifact"`
	CoinBatches int32   `df2014_assert_gte:"0"`
	TaskTypes   []int16

	Items    []Item   `df2014_get_length_from:"ItemIDs" df2014_assert_id_set:"ItemIDs"`
	Entities []Entity `df2014_get_length_from:"EntityIDs" df2014_assert_id_set:"EntityIDs"`
}

type WorldNextID struct {
	Unit          int32 `df2014_assert_gte:"-1"`
	Item          int32 `df2014_assert_gte:"-1"`
	Entity        int32 `df2014_assert_gte:"-1"`
	Nemesis       int32 `df2014_assert_gte:"-1"`
	Artifact      int32 `df2014_assert_gte:"-1"`
	Building      int32 `df2014_assert_gte:"-1"`
	HistFigure    int32 `df2014_assert_gte:"-1"`
	HistEvent     int32 `df2014_assert_gte:"-1"`
	UnitChunk     int32 `df2014_assert_gte:"-1"`
	ArtImageChunk int32 `df2014_assert_gte:"-1"`
	Task          int32 `df2014_assert_gte:"-1"`
	Unk011        int32 `df2014_assert_gte:"-1" df2014_version_min:"1205"`
	Unk012        int32 `df2014_assert_gte:"-1" df2014_version_min:"1254"`
	Unk013        int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	Unk014        int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	Unk015        int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	Unk016        int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	Unk017        int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Unk018        int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Unk019        int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Unk020        int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Unk021        int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Unk022        int32 `df2014_assert_gte:"-1" df2014_version_min:"1400"`
	Unk023        int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk024        int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk025        int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk026        int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk027        int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
}

type WorldGeneratedRaws struct {
	Inorganic   [][]string `df2014_version_min:"1372"`
	Unk000      [][]string `df2014_version_min:"1400"`
	Item        [][]string `df2014_version_min:"1441"`
	Creature    [][]string
	Entity      [][]string `df2014_version_min:"1441"`
	Interaction [][]string `df2014_version_min:"1372"`
	Language    [][]string `df2014_version_min:"1441"`
}

type WorldStringTables struct {
	Tree              []string `df2014_version_max:"1268"`
	Inorganic         []string
	Gem               []string `df2014_version_max:"1169"`
	Metal             []string `df2014_version_min:"1205" df2014_version_max:"1268"`
	Plant             []string
	Body              []string
	BodyGloss         []string
	Creature          []string
	Item              []string
	Building          []string `df2014_version_min:"1287"`
	Entity            []string
	Word              []string
	Symbol            []string
	Translation       []string
	Color             []string `df2014_version_min:"1139"`
	Shape             []string `df2014_version_min:"1139"`
	Pattern           []string `df2014_version_min:"1205"`
	Reaction          []string `df2014_version_min:"1287"`
	MaterialTemplate  []string `df2014_version_min:"1287"`
	TissueTemplate    []string `df2014_version_min:"1287"`
	BodyDetailPlan    []string `df2014_version_min:"1287"`
	CreatureVariation []string `df2014_version_min:"1287"`
	Interaction       []string `df2014_version_min:"1287"`
}
<commit_msg>identify most of the "next id" fields<commit_after>package df2014

type WorldDat struct {
	Header

	Unused000 int16 `df2014_assert_equals:"0" df2014_version_min:"1205"`
	Unused001 int32 `df2014_assert_equals:"0" df2014_version_max:"1169"`

	// SaveSlot is the original region number of this world. A value of 0 means
	// the save will be titled region1. 1 means region2, and so on.
	SaveSlot int32 `df2014_assert_gte:"0" df2014_version_max:"1169"`

	NextIDs WorldNextID

	Name *Name

	// all I know about the next five variables is that they add up to 15 bytes.
	Unk100 int8  `df2014_assert_equals:"1"`
	Unk101 int16 `df2014_assert_equals:"0"`
	Unk102 int32 `df2014_assert_equals:"1"`
	Unk103 int32 `df2014_assert_equals:"0"`
	Unk104 int32 `df2014_assert_equals:"0"`

	Title string `df2014_version_min:"1110"`

	GeneratedRaws WorldGeneratedRaws `df2014_version_min:"1287"`
	StringTables  WorldStringTables

	STOP struct{} `df2014_assert_equals:"STOP" df2014_version_min:"1205"`

	ItemIDs     []int32 `df2014_assert_next_id:"Item"`
	BuildingIDs []int32 `df2014_assert_next_id:"Building"`
	EntityIDs   []int32 `df2014_assert_next_id:"Entity"`
	NemesisIDs  []int32 `df2014_assert_next_id:"Nemesis"`
	ArtifactIDs []int32 `df2014_assert_next_id:"Artifact"`
	CoinBatches int32   `df2014_assert_gte:"0"`
	TaskTypes   []int16

	Items    []Item   `df2014_get_length_from:"ItemIDs" df2014_assert_id_set:"ItemIDs"`
	Entities []Entity `df2014_get_length_from:"EntityIDs" df2014_assert_id_set:"EntityIDs"`
}

type WorldNextID struct {
	Unit                int32 `df2014_assert_gte:"-1"`
	Unit2               int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	Item                int32 `df2014_assert_gte:"-1"`
	Entity              int32 `df2014_assert_gte:"-1"`
	Nemesis             int32 `df2014_assert_gte:"-1"`
	Artifact            int32 `df2014_assert_gte:"-1"`
	Building            int32 `df2014_assert_gte:"-1"`
	Unk007              int32 `df2014_assert_gte:"-1" df2014_version_min:"1205"`
	HistFigure          int32 `df2014_assert_gte:"-1"`
	HistEvent           int32 `df2014_assert_gte:"-1"`
	HistEventCollection int32 `df2014_assert_gte:"-1" df2014_version_min:"1254"`
	UnitChunk           int32 `df2014_assert_gte:"-1"`
	ArtImageChunk       int32 `df2014_assert_gte:"-1"`
	Task                int32 `df2014_assert_gte:"-1"`
	Squad               int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	Schedule            int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	Activity            int32 `df2014_assert_gte:"-1" df2014_version_min:"1287"`
	InteractionInstance int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	WrittenContent      int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Identity            int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Incident            int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Crime               int32 `df2014_assert_gte:"-1" df2014_version_min:"1372"`
	Vehicle             int32 `df2014_assert_gte:"-1" df2014_version_min:"1400"`
	Unk023              int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk024              int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk025              int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk026              int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
	Unk027              int32 `df2014_assert_gte:"-1" df2014_version_min:"1441"`
}

type WorldGeneratedRaws struct {
	Inorganic   [][]string `df2014_version_min:"1372"`
	Unk000      [][]string `df2014_version_min:"1400"`
	Item        [][]string `df2014_version_min:"1441"`
	Creature    [][]string
	Entity      [][]string `df2014_version_min:"1441"`
	Interaction [][]string `df2014_version_min:"1372"`
	Language    [][]string `df2014_version_min:"1441"`
}

type WorldStringTables struct {
	Tree              []string `df2014_version_max:"1268"`
	Inorganic         []string
	Gem               []string `df2014_version_max:"1169"`
	Metal             []string `df2014_version_min:"1205" df2014_version_max:"1268"`
	Plant             []string
	Body              []string
	BodyGloss         []string
	Creature          []string
	Item              []string
	Building          []string `df2014_version_min:"1287"`
	Entity            []string
	Word              []string
	Symbol            []string
	Translation       []string
	Color             []string `df2014_version_min:"1139"`
	Shape             []string `df2014_version_min:"1139"`
	Pattern           []string `df2014_version_min:"1205"`
	Reaction          []string `df2014_version_min:"1287"`
	MaterialTemplate  []string `df2014_version_min:"1287"`
	TissueTemplate    []string `df2014_version_min:"1287"`
	BodyDetailPlan    []string `df2014_version_min:"1287"`
	CreatureVariation []string `df2014_version_min:"1287"`
	Interaction       []string `df2014_version_min:"1287"`
}
<|endoftext|>


<commit_before>package termui

type WrapPanel struct {
	BaseElement

	width, height int
	orientation   Orientation
	children      []Element
	childpos      map[Element]rect
}

type rect struct {
	x, y, width, height int
}

var _ Element = new(WrapPanel)

func NewWrapPanel(o Orientation) *WrapPanel {
	return &WrapPanel{
		orientation: o,
		childpos:    make(map[Element]rect),
	}
}

// Name returns the constant name of the WrapPanel for css styling.
func (v *WrapPanel) Name() string {
	return "wrappanel"
}

// Children returns all nested elements.
func (v *WrapPanel) Children() []Element {
	return v.children
}

// AddChild adds the given children to the WrapPanel
func (v *WrapPanel) AddChild(e ...Element) {
	v.children = append(v.children, e...)
	for _, c := range e {
		c.SetParent(v)
	}
}

// Removes all children.
func (v *WrapPanel) Clear() {
	for _, c := range v.children {
		c.SetParent(nil)
	}
	v.childpos = make(map[Element]rect)
	v.children = nil
}

func (v *WrapPanel) measureVertical(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colHeight+ch > availableHeight && availableHeight != 0 {
				if colHeight > height {
					height = colHeight
				}
				width += colWidth
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					x:      width,
					width:  cw,
					height: ch,
				}
			}

			if cw > colWidth {
				colWidth = cw
			}
			colHeight += ch
			cEnd = cIdx
			cIdx++
		}

		if arrange {
			y := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.y = y
				y += rect.height
				v.childpos[child] = rect
			}
		}
	}
	return
}

func (v *WrapPanel) measureHorizontal(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colWidth+cw > availableWidth && availableWidth != 0 {
				if colWidth > width {
					width = colWidth
				}
				height += colHeight
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					y:      height,
					width:  cw,
					height: ch,
				}
			}

			if ch > colHeight {
				colHeight = ch
			}
			colWidth += cw
			cEnd = cIdx
			cIdx++
		}

		if arrange {
			x := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.x = x
				x += rect.width
				v.childpos[child] = rect
			}
		}
	}
	return
}

// Measure gets the "wanted" size of the element based on the available size
func (v *WrapPanel) Measure(availableWidth, availableHeight int) (width int, height int) {
	if v.orientation == Vertical {
		return v.measureVertical(availableWidth, availableHeight, false)
	} else {
		return v.measureHorizontal(availableWidth, availableHeight, false)
	}
}

// Arrange sets the final size for the Element end tells it to Arrange itself
func (v *WrapPanel) Arrange(finalWidth, finalHeight int) {
	v.width, v.height = finalWidth, finalHeight
	if v.orientation == Vertical {
		v.measureVertical(finalWidth, finalHeight, true)
	} else {
		v.measureHorizontal(finalWidth, finalHeight, true)
	}
}

// Render renders the element on the given Renderer
func (v *WrapPanel) Render(rn Renderer) {
	for el, pos := range v.childpos {
		rn.RenderChild(el, pos.width, pos.height, pos.x, pos.y)
	}
}

// Width returns the width of the WrapPanel
func (v *WrapPanel) Width() int {
	return v.width
}

// Height returns the height of the WrapPanel.
func (v *WrapPanel) Height() int {
	return v.height
}
<commit_msg>fixed WrapPanel layouting<commit_after>package termui

type WrapPanel struct {
	BaseElement

	width, height int
	orientation   Orientation
	children      []Element
	childpos      map[Element]rect
}

type rect struct {
	x, y, width, height int
}

var _ Element = new(WrapPanel)

func NewWrapPanel(o Orientation) *WrapPanel {
	return &WrapPanel{
		orientation: o,
		childpos:    make(map[Element]rect),
	}
}

// Name returns the constant name of the WrapPanel for css styling.
func (v *WrapPanel) Name() string {
	return "wrappanel"
}

// Children returns all nested elements.
func (v *WrapPanel) Children() []Element {
	return v.children
}

// AddChild adds the given children to the WrapPanel
func (v *WrapPanel) AddChild(e ...Element) {
	v.children = append(v.children, e...)
	for _, c := range e {
		c.SetParent(v)
	}
}

// Removes all children.
func (v *WrapPanel) Clear() {
	for _, c := range v.children {
		c.SetParent(nil)
	}
	v.childpos = make(map[Element]rect)
	v.children = nil
}

func (v *WrapPanel) measureVertical(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colHeight+ch > availableHeight && availableHeight != 0 {
				if colHeight > height {
					height = colHeight
				}
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					x:      width,
					width:  cw,
					height: ch,
				}
			}

			if cw > colWidth {
				colWidth = cw
			}
			colHeight += ch
			cEnd = cIdx
			cIdx++
		}
		width += colWidth

		if arrange {
			y := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.y = y
				y += rect.height
				v.childpos[child] = rect
			}
		}
	}
	return
}

func (v *WrapPanel) measureHorizontal(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colWidth+cw > availableWidth && availableWidth != 0 {
				if colWidth > width {
					width = colWidth
				}
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					y:      height,
					width:  cw,
					height: ch,
				}
			}

			if ch > colHeight {
				colHeight = ch
			}
			colWidth += cw
			cEnd = cIdx
			cIdx++
		}
		height += colHeight

		if arrange {
			x := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.x = x
				x += rect.width
				v.childpos[child] = rect
			}
		}
	}
	return
}

// Measure gets the "wanted" size of the element based on the available size
func (v *WrapPanel) Measure(availableWidth, availableHeight int) (width int, height int) {
	if v.orientation == Vertical {
		return v.measureVertical(availableWidth, availableHeight, false)
	} else {
		return v.measureHorizontal(availableWidth, availableHeight, false)
	}
}

// Arrange sets the final size for the Element end tells it to Arrange itself
func (v *WrapPanel) Arrange(finalWidth, finalHeight int) {
	v.width, v.height = finalWidth, finalHeight
	if v.orientation == Vertical {
		v.measureVertical(finalWidth, finalHeight, true)
	} else {
		v.measureHorizontal(finalWidth, finalHeight, true)
	}
}

// Render renders the element on the given Renderer
func (v *WrapPanel) Render(rn Renderer) {
	for el, pos := range v.childpos {
		rn.RenderChild(el, pos.width, pos.height, pos.x, pos.y)
	}
}

// Width returns the width of the WrapPanel
func (v *WrapPanel) Width() int {
	return v.width
}

// Height returns the height of the WrapPanel.
func (v *WrapPanel) Height() int {
	return v.height
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func dataSourceAwsEip() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEipRead,

		Schema: map[string]*schema.Schema{
			"association_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"filter": ec2CustomFiltersSchema(),
			"id": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"instance_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"network_interface_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"network_interface_owner_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"private_ip": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"private_dns": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"public_ip": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"public_dns": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"public_ipv4_pool": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"customer_owned_ipv4_pool": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"customer_owned_ip": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"tags": tagsSchemaComputed(),
		},
	}
}

func dataSourceAwsEipRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	req := &ec2.DescribeAddressesInput{}

	if v, ok := d.GetOk("id"); ok {
		req.AllocationIds = []*string{aws.String(v.(string))}
	}

	if v, ok := d.GetOk("public_ip"); ok {
		req.PublicIps = []*string{aws.String(v.(string))}
	}

	req.Filters = []*ec2.Filter{}

	req.Filters = append(req.Filters, buildEC2CustomFilterList(
		d.Get("filter").(*schema.Set),
	)...)

	if tags, tagsOk := d.GetOk("tags"); tagsOk {
		req.Filters = append(req.Filters, buildEC2TagFilterList(
			keyvaluetags.New(tags.(map[string]interface{})).Ec2Tags(),
		)...)
	}

	if len(req.Filters) == 0 {
		// Don't send an empty filters list; the EC2 API won't accept it.
		req.Filters = nil
	}

	resp, err := conn.DescribeAddresses(req)
	if err != nil {
		return fmt.Errorf("error describing EC2 Address: %s", err)
	}
	if resp == nil || len(resp.Addresses) == 0 {
		return fmt.Errorf("no matching Elastic IP found")
	}
	if len(resp.Addresses) > 1 {
		return fmt.Errorf("multiple Elastic IPs matched; use additional constraints to reduce matches to a single Elastic IP")
	}

	eip := resp.Addresses[0]

	if aws.StringValue(eip.Domain) == ec2.DomainTypeVpc {
		d.SetId(aws.StringValue(eip.AllocationId))
	} else {
		log.Printf("[DEBUG] Reading EIP, has no AllocationId, this means we have a Classic EIP, the id will also be the public ip : %s", req)
		d.SetId(aws.StringValue(eip.PublicIp))
	}

	d.Set("association_id", eip.AssociationId)
	d.Set("domain", eip.Domain)
	d.Set("instance_id", eip.InstanceId)
	d.Set("network_interface_id", eip.NetworkInterfaceId)
	d.Set("network_interface_owner_id", eip.NetworkInterfaceOwnerId)

	region := *conn.Config.Region

	d.Set("private_ip", eip.PrivateIpAddress)
	if eip.PrivateIpAddress != nil {
		dashIP := strings.Replace(*eip.PrivateIpAddress, ".", "-", -1)

		if region == "us-east-1" {
			d.Set("private_dns", fmt.Sprintf("ip-%s.ec2.internal", dashIP))
		} else {
			d.Set("private_dns", fmt.Sprintf("ip-%s.%s.compute.internal", dashIP, region))
		}
	}

	d.Set("public_ip", eip.PublicIp)
	if eip.PublicIp != nil {
		dashIP := strings.Replace(*eip.PublicIp, ".", "-", -1)

		if region == "us-east-1" {
			d.Set("public_dns", meta.(*AWSClient).PartitionHostname(fmt.Sprintf("ec2-%s.compute-1", dashIP)))
		} else {
			d.Set("public_dns", meta.(*AWSClient).PartitionHostname(fmt.Sprintf("ec2-%s.%s.compute", dashIP, region)))
		}
	}

	d.Set("customer_owned_ipv4_pool", eip.CustomerOwnedIpv4Pool)
	d.Set("customer_owned_ip", eip.CustomerOwnedIp)

	if err := d.Set("tags", keyvaluetags.Ec2KeyValueTags(eip.Tags).IgnoreAws().Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}
<commit_msg>Delete mistake<commit_after>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func dataSourceAwsEip() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEipRead,

		Schema: map[string]*schema.Schema{
			"association_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"filter": ec2CustomFiltersSchema(),
			"id": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"instance_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"network_interface_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"network_interface_owner_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"private_ip": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"private_dns": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"public_ip": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"public_dns": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"public_ipv4_pool": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"customer_owned_ipv4_pool": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"customer_owned_ip": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"tags": tagsSchemaComputed(),
		},
	}
}

func dataSourceAwsEipRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	req := &ec2.DescribeAddressesInput{}

	if v, ok := d.GetOk("id"); ok {
		req.AllocationIds = []*string{aws.String(v.(string))}
	}

	if v, ok := d.GetOk("public_ip"); ok {
		req.PublicIps = []*string{aws.String(v.(string))}
	}

	req.Filters = []*ec2.Filter{}

	req.Filters = append(req.Filters, buildEC2CustomFilterList(
		d.Get("filter").(*schema.Set),
	)...)

	if tags, tagsOk := d.GetOk("tags"); tagsOk {
		req.Filters = append(req.Filters, buildEC2TagFilterList(
			keyvaluetags.New(tags.(map[string]interface{})).Ec2Tags(),
		)...)
	}

	if len(req.Filters) == 0 {
		// Don't send an empty filters list; the EC2 API won't accept it.
		req.Filters = nil
	}

	resp, err := conn.DescribeAddresses(req)
	if err != nil {
		return fmt.Errorf("error describing EC2 Address: %s", err)
	}
	if resp == nil || len(resp.Addresses) == 0 {
		return fmt.Errorf("no matching Elastic IP found")
	}
	if len(resp.Addresses) > 1 {
		return fmt.Errorf("multiple Elastic IPs matched; use additional constraints to reduce matches to a single Elastic IP")
	}

	eip := resp.Addresses[0]

	if aws.StringValue(eip.Domain) == ec2.DomainTypeVpc {
		d.SetId(aws.StringValue(eip.AllocationId))
	} else {
		log.Printf("[DEBUG] Reading EIP, has no AllocationId, this means we have a Classic EIP, the id will also be the public ip : %s", req)
		d.SetId(aws.StringValue(eip.PublicIp))
	}

	d.Set("association_id", eip.AssociationId)
	d.Set("domain", eip.Domain)
	d.Set("instance_id", eip.InstanceId)
	d.Set("network_interface_id", eip.NetworkInterfaceId)
	d.Set("network_interface_owner_id", eip.NetworkInterfaceOwnerId)

	region := *conn.Config.Region

	d.Set("private_ip", eip.PrivateIpAddress)
	if eip.PrivateIpAddress != nil {
		dashIP := strings.Replace(*eip.PrivateIpAddress, ".", "-", -1)

		if region == "us-east-1" {
			d.Set("private_dns", fmt.Sprintf("ip-%s.ec2.internal", dashIP))
		} else {
			d.Set("private_dns", fmt.Sprintf("ip-%s.%s.compute.internal", dashIP, region))
		}
	}

	d.Set("public_ip", eip.PublicIp)
	if eip.PublicIp != nil {
		dashIP := strings.Replace(*eip.PublicIp, ".", "-", -1)

		if region == "us-east-1" {
			d.Set("public_dns", meta.(*AWSClient).PartitionHostname(fmt.Sprintf("ec2-%s.compute-1", dashIP)))
		} else {
			d.Set("public_dns", meta.(*AWSClient).PartitionHostname(fmt.Sprintf("ec2-%s.%s.compute", dashIP, region)))
		}
	}
	d.Set("public_ipv4_pool", eip.PublicIpv4Pool)
	d.Set("customer_owned_ipv4_pool", eip.CustomerOwnedIpv4Pool)
	d.Set("customer_owned_ip", eip.CustomerOwnedIp)

	if err := d.Set("tags", keyvaluetags.Ec2KeyValueTags(eip.Tags).IgnoreAws().Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/elb"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceAwsElb() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsElbRead,
		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},

			"access_logs": {
				Type:     schema.TypeList,
				Computed: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"interval": {
							Type:     schema.TypeInt,
							Computed: true,
							Default:  60,
						},
						"bucket": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"bucket_prefix": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"enabled": {
							Type:     schema.TypeBool,
							Computed: true,
							Default:  true,
						},
					},
				},
			},

			"availability_zones": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"connection_draining": {
				Type:     schema.TypeBool,
				Computed: true,
			},

			"connection_draining_timeout": {
				Type:     schema.TypeInt,
				Computed: true,
			},

			"cross_zone_load_balancing": {
				Type:     schema.TypeBool,
				Computed: true,
			},

			"dns_name": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"health_check": {
				Type:     schema.TypeList,
				Computed: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"healthy_threshold": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"unhealthy_threshold": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"target": {
							Type:     schema.TypeString,
							Computed: true,
						},

						"interval": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"timeout": {
							Type:     schema.TypeInt,
							Computed: true,
						},
					},
				},
			},

			"idle_timeout": {
				Type:     schema.TypeInt,
				Computed: true,
			},

			"instances": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"internal": {
				Type:     schema.TypeBool,
				Computed: true,
			},

			"listener": {
				Type:     schema.TypeSet,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"instance_port": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"instance_protocol": {
							Type:     schema.TypeString,
							Computed: true,
						},

						"lb_port": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"lb_protocol": {
							Type:     schema.TypeString,
							Computed: true,
						},

						"ssl_certificate_id": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
				Set: resourceAwsElbListenerHash,
			},

			"security_groups": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"source_security_group": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"source_security_group_id": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"subnets": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"tags": tagsSchemaComputed(),

			"zone_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func dataSourceAwsElbRead(d *schema.ResourceData, meta interface{}) error {
	elbconn := meta.(*AWSClient).elbconn
	lbName := d.Get("name").(string)

	input := &elb.DescribeLoadBalancersInput{
		LoadBalancerNames: []*string{aws.String(lbName)},
	}

	log.Printf("[DEBUG] Reading ELBs: %#v", input)
	resp, err := elbconn.DescribeLoadBalancers(input)
	if err != nil {
		return errwrap.Wrapf("Error retrieving LB: {{err}}", err)
	}
	if len(resp.LoadBalancerDescriptions) != 1 {
		return fmt.Errorf("Search returned %d results, please revise so only one is returned", len(resp.LoadBalancerDescriptions))
	}
	d.SetId(*resp.LoadBalancerDescriptions[0].LoadBalancerName)

	return flattenAwsELbResource(d, meta, resp.LoadBalancerDescriptions[0])
}
<commit_msg>Remove dataSourceAwsElb schema defaults for nested computed fields too<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/elb"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceAwsElb() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsElbRead,
		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},

			"access_logs": {
				Type:     schema.TypeList,
				Computed: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"interval": {
							Type:     schema.TypeInt,
							Computed: true,
						},
						"bucket": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"bucket_prefix": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"enabled": {
							Type:     schema.TypeBool,
							Computed: true,
						},
					},
				},
			},

			"availability_zones": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"connection_draining": {
				Type:     schema.TypeBool,
				Computed: true,
			},

			"connection_draining_timeout": {
				Type:     schema.TypeInt,
				Computed: true,
			},

			"cross_zone_load_balancing": {
				Type:     schema.TypeBool,
				Computed: true,
			},

			"dns_name": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"health_check": {
				Type:     schema.TypeList,
				Computed: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"healthy_threshold": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"unhealthy_threshold": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"target": {
							Type:     schema.TypeString,
							Computed: true,
						},

						"interval": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"timeout": {
							Type:     schema.TypeInt,
							Computed: true,
						},
					},
				},
			},

			"idle_timeout": {
				Type:     schema.TypeInt,
				Computed: true,
			},

			"instances": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"internal": {
				Type:     schema.TypeBool,
				Computed: true,
			},

			"listener": {
				Type:     schema.TypeSet,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"instance_port": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"instance_protocol": {
							Type:     schema.TypeString,
							Computed: true,
						},

						"lb_port": {
							Type:     schema.TypeInt,
							Computed: true,
						},

						"lb_protocol": {
							Type:     schema.TypeString,
							Computed: true,
						},

						"ssl_certificate_id": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
				Set: resourceAwsElbListenerHash,
			},

			"security_groups": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"source_security_group": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"source_security_group_id": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"subnets": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"tags": tagsSchemaComputed(),

			"zone_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func dataSourceAwsElbRead(d *schema.ResourceData, meta interface{}) error {
	elbconn := meta.(*AWSClient).elbconn
	lbName := d.Get("name").(string)

	input := &elb.DescribeLoadBalancersInput{
		LoadBalancerNames: []*string{aws.String(lbName)},
	}

	log.Printf("[DEBUG] Reading ELBs: %#v", input)
	resp, err := elbconn.DescribeLoadBalancers(input)
	if err != nil {
		return errwrap.Wrapf("Error retrieving LB: {{err}}", err)
	}
	if len(resp.LoadBalancerDescriptions) != 1 {
		return fmt.Errorf("Search returned %d results, please revise so only one is returned", len(resp.LoadBalancerDescriptions))
	}
	d.SetId(*resp.LoadBalancerDescriptions[0].LoadBalancerName)

	return flattenAwsELbResource(d, meta, resp.LoadBalancerDescriptions[0])
}
<|endoftext|>


<commit_before>package awsworkflow

import "github.com/aws/aws-sdk-go-v2/aws"

type AwsService struct {
	Id               string       `yaml:"id"`
	Name             string       `yaml:"name"`
	ShortName        string       `yaml:"short_name"`
	Description      string       `yaml:"description"`
	Url              string       `yaml:"url"`
	HomeID           string       `yaml:"home_id"`
	ExtraSearchTerms []string     `yaml:"extra_search_terms"`
	SubServices      []AwsService `yaml:"sub_services"`
	HasGlobalRegion  bool         `yaml:"has_global_region"`
}

func (this AwsService) GetName() string {
	if this.ShortName != "" {
		return this.ShortName + " – " + this.Name
	}
	return this.Name
}

func (this AwsService) GetRegion(cfg aws.Config) string {
	region := cfg.Region
	if this.HasGlobalRegion {
		region = ""
	}
	return region
}

func (this AwsService) HasSubServices() bool {
	return this.SubServices != nil && len(this.SubServices) > 0
}
<commit_msg>optimized<commit_after>package awsworkflow

import "github.com/aws/aws-sdk-go-v2/aws"

type AwsService struct {
	Id               string       `yaml:"id"`
	Name             string       `yaml:"name"`
	ShortName        string       `yaml:"short_name"`
	Description      string       `yaml:"description"`
	Url              string       `yaml:"url"`
	HomeID           string       `yaml:"home_id"`
	ExtraSearchTerms []string     `yaml:"extra_search_terms"`
	SubServices      []AwsService `yaml:"sub_services"`
	HasGlobalRegion  bool         `yaml:"has_global_region"`
}

func (s *AwsService) GetName() string {
	if s.ShortName != "" {
		return s.ShortName + " – " + s.Name
	}
	return s.Name
}

func (s *AwsService) GetRegion(cfg aws.Config) string {
	if s.HasGlobalRegion {
		return ""
	}
	return cfg.Region
}

func (s *AwsService) HasSubServices() bool {
	return s.SubServices != nil && len(s.SubServices) > 0
}
<|endoftext|>


<commit_before>package detour

import (
	"bytes"
	"net"
	"syscall"
)

// Detector is just a set of rules to check if a site is potentially blocked or not
type Detector struct {
	DNSPoisoned        func(net.Conn) bool
	TamperingSuspected func(error) bool
	FakeResponse       func([]byte) bool
}

var detectors = make(map[string]*Detector)

var iranRedirectAddr = "10.10.34.34:80"

func init() {
	http403 := []byte("HTTP/1.1 403 Forbidden")
	iranIFrame := []byte(`<iframe src="http://10.10.34.34`)
	// see tests and https://github.com/getlantern/lantern/issues/2099#issuecomment-78015418
	// for the facts behind detection rules for Iran
	detectors["IR"] = &Detector{
		DNSPoisoned: func(c net.Conn) bool {
			if ra := c.RemoteAddr(); ra != nil {
				return ra.String() == iranRedirectAddr
			}
			return false
		},
		FakeResponse: func(b []byte) bool {
			return bytes.HasPrefix(b, http403) && bytes.Contains(b, iranIFrame)
		},
		TamperingSuspected: func(err error) bool {
			return false
		},
	}
}

var defaultDetector = Detector{
	DNSPoisoned: func(net.Conn) bool { return false },
	TamperingSuspected: func(err error) bool {
		if ne, ok := err.(net.Error); ok && ne.Timeout() {
			return true
		}
		if oe, ok := err.(*net.OpError); ok && (oe.Err == syscall.EPIPE || oe.Err == syscall.ECONNRESET) {
			return true
		}
		return false
	},
	FakeResponse: func([]byte) bool { return false },
}

func detectorByCountry(country string) *Detector {
	d := detectors[country]
	if d == nil {
		return &defaultDetector
	}
	return &Detector{d.DNSPoisoned,
		func(err error) bool {
			return defaultDetector.TamperingSuspected(err) || d.TamperingSuspected(err)
		},
		d.FakeResponse,
	}
}
<commit_msg>detour check ECONNREFUSED on Android<commit_after>package detour

import (
	"bytes"
	"net"
	"runtime"
	"syscall"
)

// Detector is just a set of rules to check if a site is potentially blocked or not
type Detector struct {
	DNSPoisoned        func(net.Conn) bool
	TamperingSuspected func(error) bool
	FakeResponse       func([]byte) bool
}

var detectors = make(map[string]*Detector)

var iranRedirectAddr = "10.10.34.34:80"

func init() {
	http403 := []byte("HTTP/1.1 403 Forbidden")
	iranIFrame := []byte(`<iframe src="http://10.10.34.34`)
	// see tests and https://github.com/getlantern/lantern/issues/2099#issuecomment-78015418
	// for the facts behind detection rules for Iran
	detectors["IR"] = &Detector{
		DNSPoisoned: func(c net.Conn) bool {
			if ra := c.RemoteAddr(); ra != nil {
				return ra.String() == iranRedirectAddr
			}
			return false
		},
		FakeResponse: func(b []byte) bool {
			return bytes.HasPrefix(b, http403) && bytes.Contains(b, iranIFrame)
		},
		TamperingSuspected: func(err error) bool {
			return false
		},
	}
}

var defaultDetector = Detector{
	DNSPoisoned: func(net.Conn) bool { return false },
	TamperingSuspected: func(err error) bool {
		if ne, ok := err.(net.Error); ok && ne.Timeout() {
			return true
		}
		if oe, ok := err.(*net.OpError); ok {
			if oe.Err == syscall.EPIPE || oe.Err == syscall.ECONNRESET {
				return true
			}
			// TCP RST triggers ECONNREFUSED instead of ECONNRESET on Android
			// https://github.com/getlantern/lantern/issues/2375
			if runtime.GOOS == "android" && oe.Err == syscall.ECONNREFUSED {
				return true
			}
		}
		return false
	},
	FakeResponse: func([]byte) bool { return false },
}

func detectorByCountry(country string) *Detector {
	d := detectors[country]
	if d == nil {
		return &defaultDetector
	}
	return &Detector{d.DNSPoisoned,
		func(err error) bool {
			return defaultDetector.TamperingSuspected(err) || d.TamperingSuspected(err)
		},
		d.FakeResponse,
	}
}
<|endoftext|>


<commit_before>package enproxy

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

const (
	BAD_GATEWAY = 502

	DEFAULT_BYTES_BEFORE_FLUSH = 1024768
	DEFAULT_READ_BUFFER_SIZE   = 65536
)

// Proxy is the server side to an enproxy.Client.  Proxy implements the
// http.Handler interface for plugging into an HTTP server, and it also
// provides a convenience ListenAndServe() function for quickly starting up
// a dedicated HTTP server using this Proxy as its handler.
type Proxy struct {
	// Dial: function used to dial the destination server.  If nil, a default
	// TCP dialer is used.
	Dial dialFunc

	// Host: FQDN of this particular proxy.  Either this or HostFn is required
	// if this server was originally reached by DNS round robin.
	Host string

	// HostFn: given a http.Request, return the FQDN of this particular proxy,
	// hopefully through the same front.  This is used to support multiple
	// domain fronts.  Either this or Host is required if this server was
	// originally reached by DNS round robin.
	HostFn func(*http.Request) string

	// FlushTimeout: how long to let reads idle before writing out a
	// response to the client.  Defaults to 35 milliseconds.
	FlushTimeout time.Duration

	// BytesBeforeFlush: how many bytes to read before flushing response to
	// client.  Periodically flushing the response keeps the response buffer
	// from getting too big when processing big downloads.
	BytesBeforeFlush int

	// IdleTimeout: how long to wait before closing an idle connection, defaults
	// to 70 seconds
	IdleTimeout time.Duration

	// ReadBufferSize: size of read buffer in bytes
	ReadBufferSize int

	// OnBytesReceived is an optional callback for learning about bytes received
	// from a client
	OnBytesReceived statCallback

	// OnBytesSent is an optional callback for learning about bytes sent to a
	// client
	OnBytesSent statCallback

	// connMap: map of outbound connections by their id
	connMap map[string]*lazyConn

	// connMapMutex: synchronizes access to connMap
	connMapMutex sync.Mutex
}

// statCallback is a function for receiving stat information.
//
// clientIp: ip address of client
// destAddr: the destination address to which we're proxying
// req: the http.Request that's being served
// countryCode: the country-code of the client (only available when using CloudFlare)
// bytes: the number of bytes sent/received
type statCallback func(
	clientIp string,
	destAddr string,
	req *http.Request,
	bytes int64)

// Start() starts this proxy
func (p *Proxy) Start() {
	if p.Dial == nil {
		p.Dial = func(addr string) (net.Conn, error) {
			return net.Dial("tcp", addr)
		}
	}
	if p.FlushTimeout == 0 {
		p.FlushTimeout = defaultReadFlushTimeout
	}
	if p.IdleTimeout == 0 {
		p.IdleTimeout = defaultIdleTimeoutServer
	}
	if p.ReadBufferSize == 0 {
		p.ReadBufferSize = DEFAULT_READ_BUFFER_SIZE
	}
	if p.BytesBeforeFlush == 0 {
		p.BytesBeforeFlush = DEFAULT_BYTES_BEFORE_FLUSH
	}
	p.connMap = make(map[string]*lazyConn)
}

// ListenAndServe: convenience function for quickly starting up a dedicated HTTP
// server using this Proxy as its handler
func (p *Proxy) ListenAndServe(addr string) error {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return fmt.Errorf("Unable to listen at %v: %v", addr, err)
	}
	return p.Serve(l)
}

// Serve: convenience function for quickly starting up a dedicated HTTP server
// using this Proxy as its handler
func (p *Proxy) Serve(l net.Listener) error {
	p.Start()
	httpServer := &http.Server{
		Handler:      p,
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,
	}
	return httpServer.Serve(l)
}

// ServeHTTP: implements the http.Handler interface
func (p *Proxy) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	if req.Method == "HEAD" {
		// Just respond OK to HEAD requests (used for health checks)
		resp.WriteHeader(200)
		return
	}

	id := req.Header.Get(X_ENPROXY_ID)
	if id == "" {
		badGateway(resp, fmt.Sprintf("No id found in header %s", X_ENPROXY_ID))
		return
	}

	addr := req.Header.Get(X_ENPROXY_DEST_ADDR)
	if addr == "" {
		badGateway(resp, fmt.Sprintf("No address found in header %s", X_ENPROXY_DEST_ADDR))
		return
	}

	lc, isNew := p.getLazyConn(id, addr)
	connOut, err := lc.get()
	if err != nil {
		badGateway(resp, fmt.Sprintf("Unable to get connOut: %s", err))
		return
	}

	op := req.Header.Get(X_ENPROXY_OP)
	if op == OP_WRITE {
		p.handleWrite(resp, req, lc, connOut, isNew)
	} else if op == OP_READ {
		p.handleRead(resp, req, lc, connOut, true)
	} else {
		badGateway(resp, fmt.Sprintf("Op %s not supported", op))
	}
}

// handleWrite forwards the data from a POST to the outbound connection
func (p *Proxy) handleWrite(resp http.ResponseWriter, req *http.Request, lc *lazyConn, connOut net.Conn, first bool) {
	// Pipe request
	n, err := io.Copy(connOut, req.Body)
	if p.OnBytesReceived != nil && n > 0 {
		clientIp := clientIpFor(req)
		if clientIp != "" {
			p.OnBytesReceived(clientIp, lc.addr, req, n)
		}
	}
	if err != nil && err != io.EOF {
		badGateway(resp, fmt.Sprintf("Unable to write to connOut: %s", err))
		return
	}
	host := ""
	if p.HostFn != nil {
		host = p.HostFn(req)
	}
	if host == "" {
		host = p.Host
	}
	if host != "" {
		resp.Header().Set(X_ENPROXY_PROXY_HOST, host)
	}
	if first {
		// On first write, immediately do some reading
		p.handleRead(resp, req, lc, connOut, false)
	} else {
		resp.WriteHeader(200)
	}
}

// handleRead streams the data from the outbound connection to the client as
// a response body.  If no data is read for more than FlushTimeout, then the
// response is finished and client needs to make a new GET request.
func (p *Proxy) handleRead(resp http.ResponseWriter, req *http.Request, lc *lazyConn, connOut net.Conn, waitForData bool) {
	if lc.hitEOF {
		// We hit EOF on the server while processing a previous request,
		// immediately return EOF to the client
		resp.Header().Set(X_ENPROXY_EOF, "true")
		// Echo back connection id (for debugging purposes)
		resp.Header().Set(X_ENPROXY_ID, lc.id)
		resp.WriteHeader(200)
		return
	}

	// Get clientIp for reporting stats
	clientIp := clientIpFor(req)

	b := make([]byte, p.ReadBufferSize)
	first := true
	haveRead := false
	bytesInBatch := 0
	lastReadTime := time.Now()
	for {
		readDeadline := time.Now().Add(p.FlushTimeout)
		connOut.SetReadDeadline(readDeadline)

		// Read
		n, readErr := connOut.Read(b)
		if first {
			if readErr == io.EOF {
				// Reached EOF, tell client using a special header
				resp.Header().Set(X_ENPROXY_EOF, "true")
			}
			// Echo back connection id (for debugging purposes)
			resp.Header().Set(X_ENPROXY_ID, lc.id)
			// Always respond 200 OK
			resp.WriteHeader(200)
			first = false
		}

		// Write if necessary
		if n > 0 {
			if clientIp != "" && p.OnBytesSent != nil && n > 0 {
				p.OnBytesSent(clientIp, lc.addr, req, int64(n))
			}

			haveRead = true
			lastReadTime = time.Now()
			bytesInBatch = bytesInBatch + n
			_, writeErr := resp.Write(b[:n])
			if writeErr != nil {
				log.Errorf("Error writing to response: %s", writeErr)
				connOut.Close()
				return
			}
		}

		// Inspect readErr to decide whether or not to continue reading
		if readErr != nil {
			switch e := readErr.(type) {
			case net.Error:
				if e.Timeout() {
					if n == 0 {
						// We didn't read anything, might be time to return to
						// client
						if !waitForData {
							// We're not supposed to wait for data, so just
							// return right away
							return
						}
						if haveRead {
							// We've read some data, so return right away so
							// that client doesn't have to wait
							return
						}
					}
				} else {
					return
				}
			default:
				if readErr == io.EOF {
					lc.hitEOF = true
				} else {
					log.Errorf("Unexpected error reading from upstream: %s", readErr)
					// TODO: probably want to close connOut right away
				}
				return
			}
		}

		if time.Now().Sub(lastReadTime) > 10*time.Second {
			// We've spent more than 10 seconds without reading, return so that
			// CloudFlare doesn't time us out
			// TODO: Fastly has much more configurable timeouts, might be able to bump this up
			return
		}

		if bytesInBatch > p.BytesBeforeFlush {
			// We've read a good chunk, flush the response to keep its buffer
			// from getting too big.
			resp.(http.Flusher).Flush()
			bytesInBatch = 0
		}
	}
}

// getLazyConn gets the lazyConn corresponding to the given id and addr, or
// creates a new one and saves it to connMap.
func (p *Proxy) getLazyConn(id string, addr string) (l *lazyConn, isNew bool) {
	p.connMapMutex.Lock()
	defer p.connMapMutex.Unlock()
	l = p.connMap[id]
	if l == nil {
		l = p.newLazyConn(id, addr)
		p.connMap[id] = l
		isNew = true
	}
	return
}

func clientIpFor(req *http.Request) string {
	clientIp := req.Header.Get("X-Forwarded-For")
	if clientIp == "" {
		clientIp, _, err := net.SplitHostPort(req.RemoteAddr)
		if err != nil {
			log.Debugf("Unable to split RemoteAddr %v: %v", err)
			return ""
		}
		return clientIp
	}
	// clientIp may contain multiple ips, use the first
	ips := strings.Split(clientIp, ",")
	return strings.TrimSpace(ips[0])
}

func badGateway(resp http.ResponseWriter, msg string) {
	log.Errorf("Responding Bad Gateway: %s", msg)
	resp.WriteHeader(BAD_GATEWAY)
}
<commit_msg>comments on HostFn and deprecated Host, where they're used<commit_after>package enproxy

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

const (
	BAD_GATEWAY = 502

	DEFAULT_BYTES_BEFORE_FLUSH = 1024768
	DEFAULT_READ_BUFFER_SIZE   = 65536
)

// Proxy is the server side to an enproxy.Client.  Proxy implements the
// http.Handler interface for plugging into an HTTP server, and it also
// provides a convenience ListenAndServe() function for quickly starting up
// a dedicated HTTP server using this Proxy as its handler.
type Proxy struct {
	// Dial: function used to dial the destination server.  If nil, a default
	// TCP dialer is used.
	Dial dialFunc

	// Host: (Deprecated; use HostFn instead) FQDN of this particular proxy.
	// Either this or HostFn is required if this server was originally reached
	// by DNS round robin.
	Host string

	// HostFn: given a http.Request, return the FQDN of this particular proxy,
	// hopefully through the same front.  This is used to support multiple
	// domain fronts.  Either this or Host is required if this server was
	// originally reached by DNS round robin.
	HostFn func(*http.Request) string

	// FlushTimeout: how long to let reads idle before writing out a
	// response to the client.  Defaults to 35 milliseconds.
	FlushTimeout time.Duration

	// BytesBeforeFlush: how many bytes to read before flushing response to
	// client.  Periodically flushing the response keeps the response buffer
	// from getting too big when processing big downloads.
	BytesBeforeFlush int

	// IdleTimeout: how long to wait before closing an idle connection, defaults
	// to 70 seconds
	IdleTimeout time.Duration

	// ReadBufferSize: size of read buffer in bytes
	ReadBufferSize int

	// OnBytesReceived is an optional callback for learning about bytes received
	// from a client
	OnBytesReceived statCallback

	// OnBytesSent is an optional callback for learning about bytes sent to a
	// client
	OnBytesSent statCallback

	// connMap: map of outbound connections by their id
	connMap map[string]*lazyConn

	// connMapMutex: synchronizes access to connMap
	connMapMutex sync.Mutex
}

// statCallback is a function for receiving stat information.
//
// clientIp: ip address of client
// destAddr: the destination address to which we're proxying
// req: the http.Request that's being served
// countryCode: the country-code of the client (only available when using CloudFlare)
// bytes: the number of bytes sent/received
type statCallback func(
	clientIp string,
	destAddr string,
	req *http.Request,
	bytes int64)

// Start() starts this proxy
func (p *Proxy) Start() {
	if p.Dial == nil {
		p.Dial = func(addr string) (net.Conn, error) {
			return net.Dial("tcp", addr)
		}
	}
	if p.FlushTimeout == 0 {
		p.FlushTimeout = defaultReadFlushTimeout
	}
	if p.IdleTimeout == 0 {
		p.IdleTimeout = defaultIdleTimeoutServer
	}
	if p.ReadBufferSize == 0 {
		p.ReadBufferSize = DEFAULT_READ_BUFFER_SIZE
	}
	if p.BytesBeforeFlush == 0 {
		p.BytesBeforeFlush = DEFAULT_BYTES_BEFORE_FLUSH
	}
	p.connMap = make(map[string]*lazyConn)
}

// ListenAndServe: convenience function for quickly starting up a dedicated HTTP
// server using this Proxy as its handler
func (p *Proxy) ListenAndServe(addr string) error {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return fmt.Errorf("Unable to listen at %v: %v", addr, err)
	}
	return p.Serve(l)
}

// Serve: convenience function for quickly starting up a dedicated HTTP server
// using this Proxy as its handler
func (p *Proxy) Serve(l net.Listener) error {
	p.Start()
	httpServer := &http.Server{
		Handler:      p,
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,
	}
	return httpServer.Serve(l)
}

// ServeHTTP: implements the http.Handler interface
func (p *Proxy) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	if req.Method == "HEAD" {
		// Just respond OK to HEAD requests (used for health checks)
		resp.WriteHeader(200)
		return
	}

	id := req.Header.Get(X_ENPROXY_ID)
	if id == "" {
		badGateway(resp, fmt.Sprintf("No id found in header %s", X_ENPROXY_ID))
		return
	}

	addr := req.Header.Get(X_ENPROXY_DEST_ADDR)
	if addr == "" {
		badGateway(resp, fmt.Sprintf("No address found in header %s", X_ENPROXY_DEST_ADDR))
		return
	}

	lc, isNew := p.getLazyConn(id, addr)
	connOut, err := lc.get()
	if err != nil {
		badGateway(resp, fmt.Sprintf("Unable to get connOut: %s", err))
		return
	}

	op := req.Header.Get(X_ENPROXY_OP)
	if op == OP_WRITE {
		p.handleWrite(resp, req, lc, connOut, isNew)
	} else if op == OP_READ {
		p.handleRead(resp, req, lc, connOut, true)
	} else {
		badGateway(resp, fmt.Sprintf("Op %s not supported", op))
	}
}

// handleWrite forwards the data from a POST to the outbound connection
func (p *Proxy) handleWrite(resp http.ResponseWriter, req *http.Request, lc *lazyConn, connOut net.Conn, first bool) {
	// Pipe request
	n, err := io.Copy(connOut, req.Body)
	if p.OnBytesReceived != nil && n > 0 {
		clientIp := clientIpFor(req)
		if clientIp != "" {
			p.OnBytesReceived(clientIp, lc.addr, req, n)
		}
	}
	if err != nil && err != io.EOF {
		badGateway(resp, fmt.Sprintf("Unable to write to connOut: %s", err))
		return
	}
	host := ""
	if p.HostFn != nil {
		host = p.HostFn(req)
	}
	// Falling back on deprecated mechanism for backwards compatibility
	if host == "" {
		host = p.Host
	}
	if host != "" {
		// Enable sticky routing (see the comment on HostFn above).
		resp.Header().Set(X_ENPROXY_PROXY_HOST, host)
	}
	if first {
		// On first write, immediately do some reading
		p.handleRead(resp, req, lc, connOut, false)
	} else {
		resp.WriteHeader(200)
	}
}

// handleRead streams the data from the outbound connection to the client as
// a response body.  If no data is read for more than FlushTimeout, then the
// response is finished and client needs to make a new GET request.
func (p *Proxy) handleRead(resp http.ResponseWriter, req *http.Request, lc *lazyConn, connOut net.Conn, waitForData bool) {
	if lc.hitEOF {
		// We hit EOF on the server while processing a previous request,
		// immediately return EOF to the client
		resp.Header().Set(X_ENPROXY_EOF, "true")
		// Echo back connection id (for debugging purposes)
		resp.Header().Set(X_ENPROXY_ID, lc.id)
		resp.WriteHeader(200)
		return
	}

	// Get clientIp for reporting stats
	clientIp := clientIpFor(req)

	b := make([]byte, p.ReadBufferSize)
	first := true
	haveRead := false
	bytesInBatch := 0
	lastReadTime := time.Now()
	for {
		readDeadline := time.Now().Add(p.FlushTimeout)
		connOut.SetReadDeadline(readDeadline)

		// Read
		n, readErr := connOut.Read(b)
		if first {
			if readErr == io.EOF {
				// Reached EOF, tell client using a special header
				resp.Header().Set(X_ENPROXY_EOF, "true")
			}
			// Echo back connection id (for debugging purposes)
			resp.Header().Set(X_ENPROXY_ID, lc.id)
			// Always respond 200 OK
			resp.WriteHeader(200)
			first = false
		}

		// Write if necessary
		if n > 0 {
			if clientIp != "" && p.OnBytesSent != nil && n > 0 {
				p.OnBytesSent(clientIp, lc.addr, req, int64(n))
			}

			haveRead = true
			lastReadTime = time.Now()
			bytesInBatch = bytesInBatch + n
			_, writeErr := resp.Write(b[:n])
			if writeErr != nil {
				log.Errorf("Error writing to response: %s", writeErr)
				connOut.Close()
				return
			}
		}

		// Inspect readErr to decide whether or not to continue reading
		if readErr != nil {
			switch e := readErr.(type) {
			case net.Error:
				if e.Timeout() {
					if n == 0 {
						// We didn't read anything, might be time to return to
						// client
						if !waitForData {
							// We're not supposed to wait for data, so just
							// return right away
							return
						}
						if haveRead {
							// We've read some data, so return right away so
							// that client doesn't have to wait
							return
						}
					}
				} else {
					return
				}
			default:
				if readErr == io.EOF {
					lc.hitEOF = true
				} else {
					log.Errorf("Unexpected error reading from upstream: %s", readErr)
					// TODO: probably want to close connOut right away
				}
				return
			}
		}

		if time.Now().Sub(lastReadTime) > 10*time.Second {
			// We've spent more than 10 seconds without reading, return so that
			// CloudFlare doesn't time us out
			// TODO: Fastly has much more configurable timeouts, might be able to bump this up
			return
		}

		if bytesInBatch > p.BytesBeforeFlush {
			// We've read a good chunk, flush the response to keep its buffer
			// from getting too big.
			resp.(http.Flusher).Flush()
			bytesInBatch = 0
		}
	}
}

// getLazyConn gets the lazyConn corresponding to the given id and addr, or
// creates a new one and saves it to connMap.
func (p *Proxy) getLazyConn(id string, addr string) (l *lazyConn, isNew bool) {
	p.connMapMutex.Lock()
	defer p.connMapMutex.Unlock()
	l = p.connMap[id]
	if l == nil {
		l = p.newLazyConn(id, addr)
		p.connMap[id] = l
		isNew = true
	}
	return
}

func clientIpFor(req *http.Request) string {
	clientIp := req.Header.Get("X-Forwarded-For")
	if clientIp == "" {
		clientIp, _, err := net.SplitHostPort(req.RemoteAddr)
		if err != nil {
			log.Debugf("Unable to split RemoteAddr %v: %v", err)
			return ""
		}
		return clientIp
	}
	// clientIp may contain multiple ips, use the first
	ips := strings.Split(clientIp, ",")
	return strings.TrimSpace(ips[0])
}

func badGateway(resp http.ResponseWriter, msg string) {
	log.Errorf("Responding Bad Gateway: %s", msg)
	resp.WriteHeader(BAD_GATEWAY)
}
<|endoftext|>


<commit_before>/*
    User : https://api.github.com/users/Omie
        returns a dict
        has repos_url : https://api.github.com/users/Omie/repos
    Repos : https://api.github.com/users/Omie/repos
        returns a list of dict
        has contributors_url : https://api.github.com/repos/Omie/configfiles/contributors
    Contributors : https://api.github.com/repos/Omie/configfiles/contributors
        returns a list of dict
        has repos_url for each user
*/

package main

import (
        "os"
        "fmt"
        "encoding/json"
        "io/ioutil"
        "net/http"
        "log"
        "errors"
        "github.com/omie/ghlib"
)

type node struct {
    Name string `json:"name"`
    Group int `json:"group"`
    Image string `json:"image"`
}

type connection struct {
    Source int `json:"source"`
    Target int `json:"target"`
    Value int `json:"value"`
}

type graphdata struct {
    Nodes []node `json:"nodes"`
    Connections []connection `json:"links"`
}

var visited = make(map[string]string)
var knownUsers = make(map[string]int)

var requestsLeft int = 60

var username, password string

var maxDepth int = 0

var nodes []node
var connections []connection

//because Math.Min is for float64
func min(a, b int) int {
    if a <= b {
        return a
    }
    return b
}

func getData(url string) ([]byte, error) {
        log.Println("--- reached getData for ", url)

        requestsLeft--
        if requestsLeft < 0 {
            log.Println("--- LIMIT REACHED ")
            return nil, errors.New("limit reached")
        }

        client := &http.Client{}

        /* Authenticate */
        req, err := http.NewRequest("GET", url, nil)
        req.SetBasicAuth(username, password)
        resp, err := client.Do(req)
        if err != nil {
            return nil, err
        }
        defer resp.Body.Close()

        body, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            return nil, err
        }

        return body, nil
}

func getApiLimit() (int, error) {
    jsonData, err := getData("https://api.github.com/rate_limit")
    if err != nil {
        return 0, err
    }

    var limitData ghlib.GhLimit
    if err := json.Unmarshal(jsonData, &limitData); err != nil {
        return 0, err
    }
    return limitData.Rate.Remaining, nil
}

func getReposURL(username string) (string, error) {
        log.Println("--- reached getReposURL for ", username)

        userJsonData, err := getData("https://api.github.com/users/" + username)
        if err != nil {
            return "", err
        }

        var user ghlib.GhUser
        if err := json.Unmarshal(userJsonData, &user); err != nil {
            return "", err
        }
        return user.ReposUrl, nil
}

func processContributors(contribURL string, currentDepth int, parent int) {
        log.Println("--- reached processContributors for ", contribURL)
        if _, exists := visited[contribURL]; exists {
            log.Println("--- skipped ", contribURL)
            return
        }
        visited[contribURL] = contribURL

        jsonData, err := getData(contribURL)
        if err != nil {
            return
        }

        var contributors []*ghlib.GhUser
        err = json.Unmarshal(jsonData, &contributors)
        if err != nil {
            log.Println("Error while parsing contributors: ", err)
            return
        }
        //for each contributor
        for _, contributor := range contributors {
            //handle user if not previously listed
            tempUser := contributor.Login
            if nodeIdx, exists := knownUsers[tempUser]; exists {
                connections = append(connections, connection{parent, nodeIdx, 1})
                continue
            }
            //We found new user in network
            for t:=0; t<=currentDepth; t++ {
                fmt.Print("\t|")
            }
            fmt.Print(tempUser, "\n")
            nodes = append(nodes, node{tempUser, 1, contributor.AvatarUrl})
            nodeIdx := len(nodes)-1
            connections = append(connections, connection{parent, nodeIdx, 1})

            knownUsers[tempUser] = nodeIdx
            tempRepoURL := contributor.ReposUrl

            //make a call to processRepo(tempRepoURL)
            processRepos(tempRepoURL, currentDepth+1, nodeIdx)
        } //end for
}

func processRepos(repoURL string, currentDepth int, parent int) {
        log.Println("--- reached processRepos for ", repoURL)
        if currentDepth > maxDepth {
            log.Println("maxDepth reached")
            return
        }

        if _, exists := visited[repoURL]; exists {
            log.Println("--- skipped ", repoURL)
            return
        }
        visited[repoURL] = repoURL

        repoData, err := getData(repoURL) //get a list of repositories
        if err != nil {
            log.Println("err while getting data", err)
            return
        }

        var repoList []*ghlib.GhRepository
        err = json.Unmarshal(repoData, &repoList)
        if err != nil {
            log.Println("Error while parsing repo list: ", err)
            return
        }

        //m := min(len(repoList), 2)
        //repoList = repoList[:m] //limit to only 2 entries for time being

        for _, repo := range repoList {
            contribURL := repo.ContributorsUrl
            log.Println(contribURL)
            processContributors(contribURL, currentDepth, parent)
        }

} //end processRepos

func main() {
    f, err := os.OpenFile("/tmp/linkedhub.log", os.O_WRONLY | os.O_CREATE | os.O_APPEND, 0666)
    if err != nil {
        fmt.Println("Could not open file for logging")
        return
    }
    defer f.Close()

    //log.SetOutput(f)
    log.SetOutput(ioutil.Discard)

    fmt.Println("Enter github credentials")
    fmt.Print("username: ")
    fmt.Scanln(&username)
    fmt.Print("password: ")
    fmt.Scanln(&password)
    fmt.Print("Max depth: ")
    fmt.Scanln(&maxDepth)

    //find out current API limit
    limit, err := getApiLimit()
    if err != nil {
        fmt.Println("error while getting limit: ", err)
        return
    }
    if limit <= 10 {
        fmt.Println("Too few of API calls left. Not worth it.")
        return
    }
    requestsLeft = limit
    fmt.Println("requestsLeft: ", requestsLeft)

    //get username from command line
    var u string
    fmt.Println("Enter github username: ")
    fmt.Scanln(&u)

    repoURL, err := getReposURL(u)
    if err != nil {
        log.Println("error while getting repo url for: ", u)
        return
    }

    processRepos(repoURL, 0, 0)

    fw, err := os.OpenFile("graph.json", os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0666)
    if err != nil {
        fmt.Println("Could not open file for writing json")
        return
    }
    defer fw.Close()

    var gdata = &graphdata {
        Nodes: nodes,
        Connections: connections,
    }
    var toWrite []byte
    toWrite, err = json.Marshal(gdata)
    if err != nil {
        fmt.Println("Error marshalling data: ", err)
    }
    fw.Write(toWrite)

}

<commit_msg>code cleanup. removed dead code, unnecessary logging. Made changes suggested by golint. Added comments<commit_after>/*
    User : https://api.github.com/users/Omie
        returns a dict
        has repos_url : https://api.github.com/users/Omie/repos
    Repos : https://api.github.com/users/Omie/repos
        returns a list of dict
        has contributors_url : https://api.github.com/repos/Omie/configfiles/contributors
    Contributors : https://api.github.com/repos/Omie/configfiles/contributors
        returns a list of dict
        has repos_url for each user
*/

package main

import (
        "os"
        "fmt"
        "encoding/json"
        "io/ioutil"
        "net/http"
        "log"
        "errors"
        "github.com/omie/ghlib"
)

/*  types used for marhsalling data to json */
type node struct {
    Name string `json:"name"`
    Group int `json:"group"`
    Image string `json:"image"`
}

type connection struct {
    Source int `json:"source"`
    Target int `json:"target"`
    Value int `json:"value"`
}

type graphdata struct {
    Nodes []node `json:"nodes"`
    Connections []connection `json:"links"`
}

// keep track of visited URLs and previously known users
var visited = make(map[string]string)
var knownUsers = make(map[string]int)

// gets set to current limit in runtime
var requestsLeft = 60

// github a/c credentials
var username, password string

// determines how deep to crawl
var maxDepth int

// holds list of users found as nodes
// node-to-node connections using 0 based index
// as required for d3
var nodes []node
var connections []connection

// get data from remote url and return unparsed output
func getData(url string) ([]byte, error) {
        log.Println("--- reached getData for ", url)

        requestsLeft--
        if requestsLeft < 0 {
            log.Println("--- LIMIT REACHED ")
            return nil, errors.New("limit reached")
        }

        client := &http.Client{}

        /* Authenticate */
        req, err := http.NewRequest("GET", url, nil)
        req.SetBasicAuth(username, password)
        resp, err := client.Do(req)
        if err != nil {
            log.Println("error in http request: ", err)
            return nil, err
        }
        defer resp.Body.Close()

        body, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            log.Println("error reading request body: ", err)
            return nil, err
        }

        return body, nil
}

// determine current API limit
func getAPILimit() (int, error) {
    jsonData, err := getData("https://api.github.com/rate_limit")
    if err != nil {
        return 0, err
    }

    var limitData ghlib.GhLimit
    if err := json.Unmarshal(jsonData, &limitData); err != nil {
        return 0, err
    }

    limit := limitData.Rate.Remaining
    if limit <= 10 {
        return 0, errors.New("Too few of API calls left. Not worth it.")
    }

    return limit, nil
}

// get User details from API, retrieve repos_url for the user and
// return the same
func getReposURL(username string) (string, error) {
        log.Println("--- reached getReposURL for ", username)

        userJSONData, err := getData("https://api.github.com/users/" + username)
        if err != nil {
            return "", err
        }

        var user ghlib.GhUser
        if err := json.Unmarshal(userJSONData, &user); err != nil {
            return "", err
        }
        return user.ReposUrl, nil
}

// repo contributors is a list of maps [ {}, {}]
// retrieve the list, parse json, for each user:
//      process user's repositories to move to further depth
func processContributors(contribURL string, currentDepth int, parent int) {
        log.Println("--- reached processContributors for ", contribURL)
        if _, exists := visited[contribURL]; exists {
            log.Println("--- skipped ", contribURL)
            return
        }
        visited[contribURL] = contribURL

        jsonData, err := getData(contribURL)
        if err != nil {
            log.Println("error while getting contributors data: ", err)
            return
        }

        var contributors []*ghlib.GhUser
        err = json.Unmarshal(jsonData, &contributors)
        if err != nil {
            log.Println("error while unmarshalling contributors: ", err)
            return
        }
        //for each contributor
        for _, contributor := range contributors {
            //if user is already visited, just mark a connection and move to next
            tempUser := contributor.Login
            if nodeIdx, exists := knownUsers[tempUser]; exists {
                connections = append(connections, connection{parent, nodeIdx, 1})
                continue
            }
            //We found new user in network
            /* this might slow down
            for t:=0; t<=currentDepth; t++ {
                fmt.Print("\t|")
            }
            */
            fmt.Println(tempUser)

            //push to nodes list and connection list
            nodes = append(nodes, node{tempUser, 1, contributor.AvatarUrl})
            nodeIdx := len(nodes)-1
            connections = append(connections, connection{parent, nodeIdx, 1})

            knownUsers[tempUser] = nodeIdx
            tempRepoURL := contributor.ReposUrl

            //get repositories of this new user
            processRepos(tempRepoURL, currentDepth+1, nodeIdx)

        } //end for
}

// process a list of repositories
// for each repository, find and process collaborators
func processRepos(repoURL string, currentDepth int, parent int) {
        log.Println("--- reached processRepos for ", repoURL)
        if currentDepth > maxDepth {
            log.Println("maxDepth reached")
            return
        }

        if _, exists := visited[repoURL]; exists {
            return
        }
        visited[repoURL] = repoURL

        repoData, err := getData(repoURL) //get a list of repositories
        if err != nil {
            log.Println("error while getting repo list data: ", err)
            return
        }

        var repoList []*ghlib.GhRepository
        err = json.Unmarshal(repoData, &repoList)
        if err != nil {
            log.Println("error while unmarshalling repo list: ", err)
            return
        }

        for _, repo := range repoList {
            contribURL := repo.ContributorsUrl
            processContributors(contribURL, currentDepth, parent)
        }

} //end processRepos

func handleUserInput() {
    fmt.Println("Enter Github credentials -")
    fmt.Print("username(Email address): ")
    fmt.Scanln(&username)

    fmt.Print("password: ")
    fmt.Scanln(&password)

    fmt.Print("Max depth: ")
    fmt.Scanln(&maxDepth)
}

func dumpD3Json() {
    fw, err := os.OpenFile("graph.json", os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0666)
    if err != nil {
        fmt.Println("Could not open file for writing json")
        return
    }
    defer fw.Close()

    var gdata = &graphdata {
        Nodes: nodes,
        Connections: connections,
    }
    var toWrite []byte
    toWrite, err = json.Marshal(gdata)
    if err != nil {
        fmt.Println("Error marshalling data: ", err)
    }
    fw.Write(toWrite)
}

func main() {
    f, err := os.OpenFile("/tmp/linkedhub.log", os.O_WRONLY | os.O_CREATE | os.O_APPEND, 0666)
    if err != nil {
        fmt.Println("Could not open file for logging")
        return
    }
    defer f.Close()

    //log.SetOutput(f)
    log.SetOutput(ioutil.Discard)

    handleUserInput()

    //find out current API limit
    requestsLeft, err := getAPILimit()
    if err != nil {
        fmt.Println("error while getting api limit: ", err)
        return
    }
    fmt.Println("requests left for this hour: ", requestsLeft)

    //get username from command line
    var u string
    fmt.Println("Enter github username to start with: ")
    fmt.Scanln(&u)

    repoURL, err := getReposURL(u)
    if err != nil {
        log.Println("error while getting repo url for: ", u)
        return
    }

    processRepos(repoURL, 0, 0)
    dumpD3Json()
}

<|endoftext|>