<commit_before>// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

const ASSEMBLE_INTERFACE_GL_ES = `/*
 * Copyright 2019 Google LLC
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "gl/GrGLAssembleInterface.h"
#include "gl/GrGLAssembleHelpers.h"
#include "gl/GrGLUtil.h"

#define GET_PROC(F) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F)
#define GET_PROC_SUFFIX(F, S) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F #S)
#define GET_PROC_LOCAL(F) GrGL##F##Fn* F = (GrGL##F##Fn*)get(ctx, "gl" #F)

#define GET_EGL_PROC_SUFFIX(F, S) functions->fEGL##F = (GrEGL##F##Fn*)get(ctx, "egl" #F #S)

#if SK_DISABLE_GL_ES_INTERFACE
sk_sp<const GrGLInterface> GrGLMakeAssembledGLESInterface(void *ctx, GrGLGetProc get) {
    return nullptr;
}
#else
sk_sp<const GrGLInterface> GrGLMakeAssembledGLESInterface(void *ctx, GrGLGetProc get) {
    GET_PROC_LOCAL(GetString);
    if (nullptr == GetString) {
        return nullptr;
    }

    const char* verStr = reinterpret_cast<const char*>(GetString(GR_GL_VERSION));
    GrGLVersion glVer = GrGLGetVersionFromString(verStr);

    if (glVer < GR_GL_VER(2,0)) {
        return nullptr;
    }

    GET_PROC_LOCAL(GetIntegerv);
    GET_PROC_LOCAL(GetStringi);
    GrEGLQueryStringFn* queryString;
    GrEGLDisplay display;
    GrGetEGLQueryAndDisplay(&queryString, &display, ctx, get);
    GrGLExtensions extensions;
    if (!extensions.init(kGLES_GrGLStandard, GetString, GetStringi, GetIntegerv, queryString,
                         display)) {
        return nullptr;
    }

    sk_sp<GrGLInterface> interface(new GrGLInterface);
    GrGLInterface::Functions* functions = &interface->fFunctions;

    // Autogenerated content follows
[[content]]
    // End autogenerated content
    // TODO(kjlubick): Do we want a feature that removes the extension if it doesn't have
    // the function? This is common on some low-end GPUs.

    if (extensions.has("GL_KHR_debug")) {
        // In general we have a policy against removing extension strings when the driver does
        // not provide function pointers for an advertised extension. However, because there is a
        // known device that advertises GL_KHR_debug but fails to provide the functions and this is
        // a debugging- only extension we've made an exception. This also can happen when using
        // APITRACE.
        if (!interface->fFunctions.fDebugMessageControl) {
            extensions.remove("GL_KHR_debug");
        }
    }
    interface->fStandard = kGLES_GrGLStandard;
    interface->fExtensions.swap(&extensions);

    return std::move(interface);
}
#endif
`

const ASSEMBLE_INTERFACE_GL = `/*
 * Copyright 2019 Google LLC
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "gl/GrGLAssembleInterface.h"
#include "gl/GrGLAssembleHelpers.h"
#include "gl/GrGLUtil.h"

#define GET_PROC(F) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F)
#define GET_PROC_SUFFIX(F, S) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F #S)
#define GET_PROC_LOCAL(F) GrGL##F##Fn* F = (GrGL##F##Fn*)get(ctx, "gl" #F)

#define GET_EGL_PROC_SUFFIX(F, S) functions->fEGL##F = (GrEGL##F##Fn*)get(ctx, "egl" #F #S)

#if SK_DISABLE_GL_INTERFACE
sk_sp<const GrGLInterface> GrGLMakeAssembledGLInterface(void *ctx, GrGLGetProc get) {
    return nullptr;
}
#else
sk_sp<const GrGLInterface> GrGLMakeAssembledGLInterface(void *ctx, GrGLGetProc get) {
    GET_PROC_LOCAL(GetString);
    GET_PROC_LOCAL(GetStringi);
    GET_PROC_LOCAL(GetIntegerv);

    // GetStringi may be nullptr depending on the GL version.
    if (nullptr == GetString || nullptr == GetIntegerv) {
        return nullptr;
    }

    const char* versionString = (const char*) GetString(GR_GL_VERSION);
    GrGLVersion glVer = GrGLGetVersionFromString(versionString);

    if (glVer < GR_GL_VER(2,0) || GR_GL_INVALID_VER == glVer) {
        // This is our minimum for non-ES GL.
        return nullptr;
    }

    GrEGLQueryStringFn* queryString;
    GrEGLDisplay display;
    GrGetEGLQueryAndDisplay(&queryString, &display, ctx, get);
    GrGLExtensions extensions;
    if (!extensions.init(kGL_GrGLStandard, GetString, GetStringi, GetIntegerv, queryString,
                         display)) {
        return nullptr;
    }

    sk_sp<GrGLInterface> interface(new GrGLInterface());
    GrGLInterface::Functions* functions = &interface->fFunctions;

    // Autogenerated content follows
[[content]]
    // End autogenerated content
    interface->fStandard = kGL_GrGLStandard;
    interface->fExtensions.swap(&extensions);

    return std::move(interface);
}
#endif
`

const ASSEMBLE_INTERFACE_WEBGL = `/*
 * Copyright 2019 Google LLC
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "gl/GrGLAssembleInterface.h"
#include "gl/GrGLAssembleHelpers.h"
#include "gl/GrGLUtil.h"

#define GET_PROC(F) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F)
#define GET_PROC_SUFFIX(F, S) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F #S)
#define GET_PROC_LOCAL(F) GrGL##F##Fn* F = (GrGL##F##Fn*)get(ctx, "gl" #F)

#define GET_EGL_PROC_SUFFIX(F, S) functions->fEGL##F = (GrEGL##F##Fn*)get(ctx, "egl" #F #S)

#if SK_DISABLE_WEBGL_INTERFACE
sk_sp<const GrGLInterface> GrGLMakeAssembledWebGLInterface(void *ctx, GrGLGetProc get) {
    return nullptr;
}
#else
sk_sp<const GrGLInterface> GrGLMakeAssembledWebGLInterface(void *ctx, GrGLGetProc get) {
    GET_PROC_LOCAL(GetString);
    if (nullptr == GetString) {
        return nullptr;
    }

    const char* verStr = reinterpret_cast<const char*>(GetString(GR_GL_VERSION));
    GrGLVersion glVer = GrGLGetVersionFromString(verStr);

    if (glVer < GR_GL_VER(1,0)) {
        return nullptr;
    }

    GET_PROC_LOCAL(GetIntegerv);
    GET_PROC_LOCAL(GetStringi);
    GrEGLQueryStringFn* queryString;
    GrEGLDisplay display;
    GrGetEGLQueryAndDisplay(&queryString, &display, ctx, get);
    GrGLExtensions extensions;
    if (!extensions.init(kWebGL_GrGLStandard, GetString, GetStringi, GetIntegerv, queryString,
                         display)) {
        return nullptr;
    }

    sk_sp<GrGLInterface> interface(new GrGLInterface);
    GrGLInterface::Functions* functions = &interface->fFunctions;

    // Autogenerated content follows
[[content]]
    // End autogenerated content

    interface->fStandard = kWebGL_GrGLStandard;
    interface->fExtensions.swap(&extensions);

    return std::move(interface);
}
#endif
`

const VALIDATE_INTERFACE = `/*
 * Copyright 2011 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "gl/GrGLInterface.h"
#include "gl/GrGLExtensions.h"
#include "gl/GrGLUtil.h"

#include <stdio.h>

GrGLInterface::GrGLInterface() {
    fStandard = kNone_GrGLStandard;
}

#define RETURN_FALSE_INTERFACE                                                 \
    SkDEBUGF("%s:%d GrGLInterface::validate() failed.\n", __FILE__, __LINE__); \
    return false

bool GrGLInterface::validate() const {

    if (kNone_GrGLStandard == fStandard) {
        RETURN_FALSE_INTERFACE;
    }

    if (!fExtensions.isInitialized()) {
        RETURN_FALSE_INTERFACE;
    }

    GrGLVersion glVer = GrGLGetVersion(this);
    if (GR_GL_INVALID_VER == glVer) {
        RETURN_FALSE_INTERFACE;
    }
    // Autogenerated content follows
[[content]]
    // End autogenerated content
    return true;
}

#if GR_TEST_UTILS

void GrGLInterface::abandon() const {
    const_cast<GrGLInterface*>(this)->fFunctions = GrGLInterface::Functions();
}

#endif // GR_TEST_UTILS
`
<commit_msg>update GL interface codegen<commit_after>// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

const ASSEMBLE_INTERFACE_GL_ES = `/*
 * Copyright 2019 Google LLC
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "include/gpu/gl/GrGLAssembleHelpers.h"
#include "include/gpu/gl/GrGLAssembleInterface.h"
#include "src/gpu/gl/GrGLUtil.h"

#define GET_PROC(F) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F)
#define GET_PROC_SUFFIX(F, S) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F #S)
#define GET_PROC_LOCAL(F) GrGL##F##Fn* F = (GrGL##F##Fn*)get(ctx, "gl" #F)

#define GET_EGL_PROC_SUFFIX(F, S) functions->fEGL##F = (GrEGL##F##Fn*)get(ctx, "egl" #F #S)

#if SK_DISABLE_GL_ES_INTERFACE
sk_sp<const GrGLInterface> GrGLMakeAssembledGLESInterface(void *ctx, GrGLGetProc get) {
    return nullptr;
}
#else
sk_sp<const GrGLInterface> GrGLMakeAssembledGLESInterface(void *ctx, GrGLGetProc get) {
    GET_PROC_LOCAL(GetString);
    if (nullptr == GetString) {
        return nullptr;
    }

    const char* verStr = reinterpret_cast<const char*>(GetString(GR_GL_VERSION));
    GrGLVersion glVer = GrGLGetVersionFromString(verStr);

    if (glVer < GR_GL_VER(2,0)) {
        return nullptr;
    }

    GET_PROC_LOCAL(GetIntegerv);
    GET_PROC_LOCAL(GetStringi);
    GrEGLQueryStringFn* queryString;
    GrEGLDisplay display;
    GrGetEGLQueryAndDisplay(&queryString, &display, ctx, get);
    GrGLExtensions extensions;
    if (!extensions.init(kGLES_GrGLStandard, GetString, GetStringi, GetIntegerv, queryString,
                         display)) {
        return nullptr;
    }

    sk_sp<GrGLInterface> interface(new GrGLInterface);
    GrGLInterface::Functions* functions = &interface->fFunctions;

    // Autogenerated content follows
[[content]]
    // End autogenerated content
    // TODO(kjlubick): Do we want a feature that removes the extension if it doesn't have
    // the function? This is common on some low-end GPUs.

    if (extensions.has("GL_KHR_debug")) {
        // In general we have a policy against removing extension strings when the driver does
        // not provide function pointers for an advertised extension. However, because there is a
        // known device that advertises GL_KHR_debug but fails to provide the functions and this is
        // a debugging- only extension we've made an exception. This also can happen when using
        // APITRACE.
        if (!interface->fFunctions.fDebugMessageControl) {
            extensions.remove("GL_KHR_debug");
        }
    }
    interface->fStandard = kGLES_GrGLStandard;
    interface->fExtensions.swap(&extensions);

    return std::move(interface);
}
#endif
`

const ASSEMBLE_INTERFACE_GL = `/*
 * Copyright 2019 Google LLC
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "include/gpu/gl/GrGLAssembleHelpers.h"
#include "include/gpu/gl/GrGLAssembleInterface.h"
#include "src/gpu/gl/GrGLUtil.h"

#define GET_PROC(F) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F)
#define GET_PROC_SUFFIX(F, S) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F #S)
#define GET_PROC_LOCAL(F) GrGL##F##Fn* F = (GrGL##F##Fn*)get(ctx, "gl" #F)

#define GET_EGL_PROC_SUFFIX(F, S) functions->fEGL##F = (GrEGL##F##Fn*)get(ctx, "egl" #F #S)

#if SK_DISABLE_GL_INTERFACE
sk_sp<const GrGLInterface> GrGLMakeAssembledGLInterface(void *ctx, GrGLGetProc get) {
    return nullptr;
}
#else
sk_sp<const GrGLInterface> GrGLMakeAssembledGLInterface(void *ctx, GrGLGetProc get) {
    GET_PROC_LOCAL(GetString);
    GET_PROC_LOCAL(GetStringi);
    GET_PROC_LOCAL(GetIntegerv);

    // GetStringi may be nullptr depending on the GL version.
    if (nullptr == GetString || nullptr == GetIntegerv) {
        return nullptr;
    }

    const char* versionString = (const char*) GetString(GR_GL_VERSION);
    GrGLVersion glVer = GrGLGetVersionFromString(versionString);

    if (glVer < GR_GL_VER(2,0) || GR_GL_INVALID_VER == glVer) {
        // This is our minimum for non-ES GL.
        return nullptr;
    }

    GrEGLQueryStringFn* queryString;
    GrEGLDisplay display;
    GrGetEGLQueryAndDisplay(&queryString, &display, ctx, get);
    GrGLExtensions extensions;
    if (!extensions.init(kGL_GrGLStandard, GetString, GetStringi, GetIntegerv, queryString,
                         display)) {
        return nullptr;
    }

    sk_sp<GrGLInterface> interface(new GrGLInterface());
    GrGLInterface::Functions* functions = &interface->fFunctions;

    // Autogenerated content follows
[[content]]
    // End autogenerated content
    interface->fStandard = kGL_GrGLStandard;
    interface->fExtensions.swap(&extensions);

    return std::move(interface);
}
#endif
`

const ASSEMBLE_INTERFACE_WEBGL = `/*
 * Copyright 2019 Google LLC
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "include/gpu/gl/GrGLAssembleHelpers.h"
#include "include/gpu/gl/GrGLAssembleInterface.h"
#include "src/gpu/gl/GrGLUtil.h"

#define GET_PROC(F) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F)
#define GET_PROC_SUFFIX(F, S) functions->f##F = (GrGL##F##Fn*)get(ctx, "gl" #F #S)
#define GET_PROC_LOCAL(F) GrGL##F##Fn* F = (GrGL##F##Fn*)get(ctx, "gl" #F)

#define GET_EGL_PROC_SUFFIX(F, S) functions->fEGL##F = (GrEGL##F##Fn*)get(ctx, "egl" #F #S)

#if SK_DISABLE_WEBGL_INTERFACE
sk_sp<const GrGLInterface> GrGLMakeAssembledWebGLInterface(void *ctx, GrGLGetProc get) {
    return nullptr;
}
#else
sk_sp<const GrGLInterface> GrGLMakeAssembledWebGLInterface(void *ctx, GrGLGetProc get) {
    GET_PROC_LOCAL(GetString);
    if (nullptr == GetString) {
        return nullptr;
    }

    const char* verStr = reinterpret_cast<const char*>(GetString(GR_GL_VERSION));
    GrGLVersion glVer = GrGLGetVersionFromString(verStr);

    if (glVer < GR_GL_VER(1,0)) {
        return nullptr;
    }

    GET_PROC_LOCAL(GetIntegerv);
    GET_PROC_LOCAL(GetStringi);
    GrEGLQueryStringFn* queryString;
    GrEGLDisplay display;
    GrGetEGLQueryAndDisplay(&queryString, &display, ctx, get);
    GrGLExtensions extensions;
    if (!extensions.init(kWebGL_GrGLStandard, GetString, GetStringi, GetIntegerv, queryString,
                         display)) {
        return nullptr;
    }

    sk_sp<GrGLInterface> interface(new GrGLInterface);
    GrGLInterface::Functions* functions = &interface->fFunctions;

    // Autogenerated content follows
[[content]]
    // End autogenerated content

    interface->fStandard = kWebGL_GrGLStandard;
    interface->fExtensions.swap(&extensions);

    return std::move(interface);
}
#endif
`

const VALIDATE_INTERFACE = `/*
 * Copyright 2011 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * THIS FILE IS AUTOGENERATED
 * Make edits to tools/gpu/gl/interface/templates.go or they will
 * be overwritten.
 */

#include "include/gpu/gl/GrGLExtensions.h"
#include "include/gpu/gl/GrGLInterface.h"
#include "src/gpu/gl/GrGLUtil.h"

#include <stdio.h>

GrGLInterface::GrGLInterface() {
    fStandard = kNone_GrGLStandard;
}

#define RETURN_FALSE_INTERFACE                                                 \
    SkDEBUGF("%s:%d GrGLInterface::validate() failed.\n", __FILE__, __LINE__); \
    return false

bool GrGLInterface::validate() const {

    if (kNone_GrGLStandard == fStandard) {
        RETURN_FALSE_INTERFACE;
    }

    if (!fExtensions.isInitialized()) {
        RETURN_FALSE_INTERFACE;
    }

    GrGLVersion glVer = GrGLGetVersion(this);
    if (GR_GL_INVALID_VER == glVer) {
        RETURN_FALSE_INTERFACE;
    }
    // Autogenerated content follows
[[content]]
    // End autogenerated content
    return true;
}

#if GR_TEST_UTILS

void GrGLInterface::abandon() const {
    const_cast<GrGLInterface*>(this)->fFunctions = GrGLInterface::Functions();
}

#endif // GR_TEST_UTILS
`
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/elasticache/elasticacheiface"
	"github.com/cloudfoundry-community/go-cfclient"
	"net/url"
	"regexp"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	awsec "github.com/aws/aws-sdk-go/service/elasticache"

	"github.com/alphagov/paas-cf/tools/metrics/pkg/elasticache"

	m "github.com/alphagov/paas-cf/tools/metrics/pkg/metrics"
)

const NonClusteredIdPattern = "[a-z0-9]+-\\d+"
const ClusteredIdPattern = "[a-z0-9]+-\\d+-\\d+"

type RedisServiceDetails struct {
	ServiceInstance cfclient.ServiceInstance
	Space           cfclient.Space
	Org             cfclient.Org
}

func ElasticCacheInstancesGauge(
	logger lager.Logger,
	ecs *elasticache.ElasticacheService,
	cfAPI cfclient.CloudFoundryClient,
	clusterIdHashingFunction elasticache.ElasticacheClusterIdHashingFunction,
	interval time.Duration,
) m.MetricReadCloser {
	return m.NewMetricPoller(interval, func(w m.MetricWriter) error {
		lsess := logger.Session("elasticache-gauges")

		metrics, err := cacheParameterMetrics(ecs)
		if err != nil {
			return err
		}

		clusterMetrics, err := cacheClusterMetrics(lsess, ecs, cfAPI, clusterIdHashingFunction)
		if err != nil {
			return err
		}
		metrics = append(metrics, clusterMetrics...)

		return w.WriteMetrics(metrics)
	})
}

func cacheParameterMetrics(ecs *elasticache.ElasticacheService) ([]m.Metric, error) {
	cacheParameterGroupCount := 0
	err := iterateCacheParameterGroups(
		ecs.Client,
		func(cacheParameterGroup *awsec.CacheParameterGroup) error {
			if !strings.HasPrefix(*cacheParameterGroup.CacheParameterGroupName, "default.") {
				cacheParameterGroupCount++
			}
			return nil
		},
	)
	if err != nil {
		return nil, err
	}

	return []m.Metric{{
		Kind:  m.Gauge,
		Time:  time.Now(),
		Name:  "aws.elasticache.cache_parameter_group.count",
		Value: float64(cacheParameterGroupCount),
		Unit:  "count",
	}}, nil
}

func cacheClusterMetrics(
	logger lager.Logger,
	ecs *elasticache.ElasticacheService,
	cfAPI cfclient.CloudFoundryClient,
	clusterIdHashingFunction elasticache.ElasticacheClusterIdHashingFunction,
) ([]m.Metric, error) {
	redisServiceDetails, err := fetchRedisServiceInstances(cfAPI)
	if err != nil {
		return nil, err
	}

	userSuppliedClusterIdToDetail := map[string]RedisServiceDetails{}
	for _, details := range redisServiceDetails {
		userSuppliedClusterId := clusterIdHashingFunction(details.ServiceInstance.Guid)
		userSuppliedClusterIdToDetail[userSuppliedClusterId] = details
	}

	metrics := []m.Metric{}
	nodeCount := int64(0)
	err = iterateCacheClusterPages(
		ecs.Client,
		func(cacheCluster *awsec.CacheCluster) error {
			nodeCount = nodeCount + *cacheCluster.NumCacheNodes
			elasticacheClusterId := aws.StringValue(cacheCluster.CacheClusterId)
			userSuppliedClusterId := cleanClusterId(cacheCluster)

			details, ok := userSuppliedClusterIdToDetail[userSuppliedClusterId]
			if !ok {
				e := fmt.Errorf("unable to find details of cluster with user supplied cluster id of %s", userSuppliedClusterId)
				logger.Error("details-not-found", e, lager.Data{
					"cluster_id":               elasticacheClusterId,
					"user_supplied_cluster_id": userSuppliedClusterId,
				})
				return e
			}

			metrics = append(metrics, m.Metric{
				Kind:  m.Gauge,
				Time:  time.Now(),
				Name:  "aws.elasticache.cluster.nodes.count",
				Value: float64(*cacheCluster.NumCacheNodes),
				Unit:  "count",
				Tags: m.MetricTags{
					{Label: "cluster_id", Value: userSuppliedClusterId},
					{Label: "elasticache_cluster_id", Value: elasticacheClusterId},
					{Label: "service_instance_guid", Value: details.ServiceInstance.Guid},
					{Label: "space_name", Value: details.Space.Name},
					{Label: "space_guid", Value: details.Space.Guid},
					{Label: "org_name", Value: details.Org.Name},
					{Label: "org_guid", Value: details.Org.Guid},
				},
			})
			return nil
		},
	)
	if err != nil {
		return nil, err
	}

	metrics = append(metrics, m.Metric{
		Kind:  m.Gauge,
		Time:  time.Now(),
		Name:  "aws.elasticache.node.count",
		Value: float64(nodeCount),
		Unit:  "count",
	})
	return metrics, nil
}

func fetchRedisServiceInstances(cfAPI cfclient.CloudFoundryClient) ([]RedisServiceDetails, error) {
	spacesCache := map[string]cfclient.Space{}
	orgsCache := map[string]cfclient.Org{}

	servicesWithRedisLabel, err := cfAPI.ListServicesByQuery(url.Values{
		"q": []string{"label:redis"},
	})

	if err != nil {
		return []RedisServiceDetails{}, err
	}

	if len(servicesWithRedisLabel) == 0 {
		return nil, fmt.Errorf("could not find service with label=redis")
	}

	redisServiceGuid := servicesWithRedisLabel[0].Guid
	redisServicePlans, err := cfAPI.ListServicePlansByQuery(url.Values{
		"q": []string{"service_guid:" + redisServiceGuid},
	})

	if err != nil {
		return []RedisServiceDetails{}, err
	}

	var serviceDetails []RedisServiceDetails
	for _, plan := range redisServicePlans {
		planInstances, err := cfAPI.ListServiceInstancesByQuery(url.Values{
			"q": []string{"service_plan_guid:" + plan.Guid},
		})

		if err != nil {
			return []RedisServiceDetails{}, err
		}

		for _, instance := range planInstances {
			var space cfclient.Space
			if s, ok := spacesCache[instance.SpaceGuid]; ok {
				space = s
			} else {
				space, err = cfAPI.GetSpaceByGuid(instance.SpaceGuid)
				if err != nil {
					return []RedisServiceDetails{}, err
				}

				spacesCache[space.Guid] = space
			}

			var org cfclient.Org
			if o, ok := orgsCache[space.OrganizationGuid]; ok {
				org = o
			} else {
				org, err = cfAPI.GetOrgByGuid(space.OrganizationGuid)
				if err != nil {
					return []RedisServiceDetails{}, err
				}

				orgsCache[org.Guid] = org
			}

			serviceDetails = append(serviceDetails, RedisServiceDetails{
				ServiceInstance: instance,
				Space:           space,
				Org:             org,
			})
		}
	}

	return serviceDetails, nil
}

// The AWS API documentation says that
// CacheClusterId is the user-supplied name
// of the cache cluster. However, in reality,
// it returns that value, plus some extra
// information.
//
// It is in the format {user-supplied}-{n}[-{m}].
// Typically, our cache cluster names have the
// format "cf-{FNV hash of guid}", but that's
// not guaranteed, so this method only strips
// the last one/two parts.
func cleanClusterId(cluster *awsec.CacheCluster) string {
	clusteredRegex := regexp.MustCompile(ClusteredIdPattern)
	unclusteredRegex := regexp.MustCompile(NonClusteredIdPattern)

	strBytes := []byte(aws.StringValue(cluster.CacheClusterId))

	parts := strings.Split(
		aws.StringValue(cluster.CacheClusterId),
		"-",
	)

	var topIndex int
	if clusteredRegex.Match(strBytes) {
		topIndex = len(parts) - 2 // Minus two because we want to stop before the second-to-last part
		return strings.Join(parts[:topIndex], "-")

	} else if unclusteredRegex.Match(strBytes) {
		topIndex = len(parts) - 1 // Minus one because we want to stop before the last part
		return strings.Join(parts[:topIndex], "-")

	} else {
		// Return the original value if it doesn't match either pattern
		return aws.StringValue(cluster.CacheClusterId)
	}
}

func iterateCacheParameterGroups(client elasticacheiface.ElastiCacheAPI, fn func(*awsec.CacheParameterGroup) error) error {
	errs := []error{}
	err := client.DescribeCacheParameterGroupsPages(
		&awsec.DescribeCacheParameterGroupsInput{},
		func(page *awsec.DescribeCacheParameterGroupsOutput, lastPage bool) bool {
			for _, cacheParameterGroup := range page.CacheParameterGroups {
				err := fn(cacheParameterGroup)
				if err != nil {
					errs = append(errs, err)
				}
			}
			return true
		},
	)
	if err != nil {
		return err
	}
	if len(errs) > 0 {
		return errs[0]
	}
	return nil
}

func iterateCacheClusterPages(client elasticacheiface.ElastiCacheAPI, fn func(cluster *awsec.CacheCluster) error) error {
	errs := []error{}
	err := client.DescribeCacheClustersPages(
		&awsec.DescribeCacheClustersInput{},
		func(page *awsec.DescribeCacheClustersOutput, lastPage bool) bool {
			for _, cacheCluster := range page.CacheClusters {
				err := fn(cacheCluster)
				if err != nil {
					errs = append(errs, err)
				}
			}
			return true
		},
	)
	if err != nil {
		return err
	}
	if len(errs) > 0 {
		return errs[0]
	}
	return nil
}
<commit_msg>Rename method working with elasticache cluster IDs<commit_after>package main

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/elasticache/elasticacheiface"
	"github.com/cloudfoundry-community/go-cfclient"
	"net/url"
	"regexp"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	awsec "github.com/aws/aws-sdk-go/service/elasticache"

	"github.com/alphagov/paas-cf/tools/metrics/pkg/elasticache"

	m "github.com/alphagov/paas-cf/tools/metrics/pkg/metrics"
)

const NonClusteredIdPattern = "[a-z0-9]+-\\d+"
const ClusteredIdPattern = "[a-z0-9]+-\\d+-\\d+"

type RedisServiceDetails struct {
	ServiceInstance cfclient.ServiceInstance
	Space           cfclient.Space
	Org             cfclient.Org
}

func ElasticCacheInstancesGauge(
	logger lager.Logger,
	ecs *elasticache.ElasticacheService,
	cfAPI cfclient.CloudFoundryClient,
	clusterIdHashingFunction elasticache.ElasticacheClusterIdHashingFunction,
	interval time.Duration,
) m.MetricReadCloser {
	return m.NewMetricPoller(interval, func(w m.MetricWriter) error {
		lsess := logger.Session("elasticache-gauges")

		metrics, err := cacheParameterMetrics(ecs)
		if err != nil {
			return err
		}

		clusterMetrics, err := cacheClusterMetrics(lsess, ecs, cfAPI, clusterIdHashingFunction)
		if err != nil {
			return err
		}
		metrics = append(metrics, clusterMetrics...)

		return w.WriteMetrics(metrics)
	})
}

func cacheParameterMetrics(ecs *elasticache.ElasticacheService) ([]m.Metric, error) {
	cacheParameterGroupCount := 0
	err := iterateCacheParameterGroups(
		ecs.Client,
		func(cacheParameterGroup *awsec.CacheParameterGroup) error {
			if !strings.HasPrefix(*cacheParameterGroup.CacheParameterGroupName, "default.") {
				cacheParameterGroupCount++
			}
			return nil
		},
	)
	if err != nil {
		return nil, err
	}

	return []m.Metric{{
		Kind:  m.Gauge,
		Time:  time.Now(),
		Name:  "aws.elasticache.cache_parameter_group.count",
		Value: float64(cacheParameterGroupCount),
		Unit:  "count",
	}}, nil
}

func cacheClusterMetrics(
	logger lager.Logger,
	ecs *elasticache.ElasticacheService,
	cfAPI cfclient.CloudFoundryClient,
	clusterIdHashingFunction elasticache.ElasticacheClusterIdHashingFunction,
) ([]m.Metric, error) {
	redisServiceDetails, err := fetchRedisServiceInstances(cfAPI)
	if err != nil {
		return nil, err
	}

	userSuppliedClusterIdToDetail := map[string]RedisServiceDetails{}
	for _, details := range redisServiceDetails {
		userSuppliedClusterId := clusterIdHashingFunction(details.ServiceInstance.Guid)
		userSuppliedClusterIdToDetail[userSuppliedClusterId] = details
	}

	metrics := []m.Metric{}
	nodeCount := int64(0)
	err = iterateCacheClusterPages(
		ecs.Client,
		func(cacheCluster *awsec.CacheCluster) error {
			nodeCount = nodeCount + *cacheCluster.NumCacheNodes
			elasticacheClusterId := aws.StringValue(cacheCluster.CacheClusterId)
			userSuppliedClusterId := extractUserSuppliedClusterID(cacheCluster)

			details, ok := userSuppliedClusterIdToDetail[userSuppliedClusterId]
			if !ok {
				e := fmt.Errorf("unable to find details of cluster with user supplied cluster id of %s", userSuppliedClusterId)
				logger.Error("details-not-found", e, lager.Data{
					"cluster_id":               elasticacheClusterId,
					"user_supplied_cluster_id": userSuppliedClusterId,
				})
				return e
			}

			metrics = append(metrics, m.Metric{
				Kind:  m.Gauge,
				Time:  time.Now(),
				Name:  "aws.elasticache.cluster.nodes.count",
				Value: float64(*cacheCluster.NumCacheNodes),
				Unit:  "count",
				Tags: m.MetricTags{
					{Label: "cluster_id", Value: userSuppliedClusterId},
					{Label: "elasticache_cluster_id", Value: elasticacheClusterId},
					{Label: "service_instance_guid", Value: details.ServiceInstance.Guid},
					{Label: "space_name", Value: details.Space.Name},
					{Label: "space_guid", Value: details.Space.Guid},
					{Label: "org_name", Value: details.Org.Name},
					{Label: "org_guid", Value: details.Org.Guid},
				},
			})
			return nil
		},
	)
	if err != nil {
		return nil, err
	}

	metrics = append(metrics, m.Metric{
		Kind:  m.Gauge,
		Time:  time.Now(),
		Name:  "aws.elasticache.node.count",
		Value: float64(nodeCount),
		Unit:  "count",
	})
	return metrics, nil
}

func fetchRedisServiceInstances(cfAPI cfclient.CloudFoundryClient) ([]RedisServiceDetails, error) {
	spacesCache := map[string]cfclient.Space{}
	orgsCache := map[string]cfclient.Org{}

	servicesWithRedisLabel, err := cfAPI.ListServicesByQuery(url.Values{
		"q": []string{"label:redis"},
	})

	if err != nil {
		return []RedisServiceDetails{}, err
	}

	if len(servicesWithRedisLabel) == 0 {
		return nil, fmt.Errorf("could not find service with label=redis")
	}

	redisServiceGuid := servicesWithRedisLabel[0].Guid
	redisServicePlans, err := cfAPI.ListServicePlansByQuery(url.Values{
		"q": []string{"service_guid:" + redisServiceGuid},
	})

	if err != nil {
		return []RedisServiceDetails{}, err
	}

	var serviceDetails []RedisServiceDetails
	for _, plan := range redisServicePlans {
		planInstances, err := cfAPI.ListServiceInstancesByQuery(url.Values{
			"q": []string{"service_plan_guid:" + plan.Guid},
		})

		if err != nil {
			return []RedisServiceDetails{}, err
		}

		for _, instance := range planInstances {
			var space cfclient.Space
			if s, ok := spacesCache[instance.SpaceGuid]; ok {
				space = s
			} else {
				space, err = cfAPI.GetSpaceByGuid(instance.SpaceGuid)
				if err != nil {
					return []RedisServiceDetails{}, err
				}

				spacesCache[space.Guid] = space
			}

			var org cfclient.Org
			if o, ok := orgsCache[space.OrganizationGuid]; ok {
				org = o
			} else {
				org, err = cfAPI.GetOrgByGuid(space.OrganizationGuid)
				if err != nil {
					return []RedisServiceDetails{}, err
				}

				orgsCache[org.Guid] = org
			}

			serviceDetails = append(serviceDetails, RedisServiceDetails{
				ServiceInstance: instance,
				Space:           space,
				Org:             org,
			})
		}
	}

	return serviceDetails, nil
}

// The AWS API documentation says that
// CacheClusterId is the user-supplied name
// of the cache cluster. However, in reality,
// it returns that value, plus some extra
// information.
//
// It is in the format {user-supplied}-{n}[-{m}].
// Typically, our cache cluster names have the
// format "cf-{FNV hash of guid}", but that's
// not guaranteed, so this method only strips
// the last one/two parts.
func extractUserSuppliedClusterID(cluster *awsec.CacheCluster) string {
	clusteredRegex := regexp.MustCompile(ClusteredIdPattern)
	unclusteredRegex := regexp.MustCompile(NonClusteredIdPattern)

	strBytes := []byte(aws.StringValue(cluster.CacheClusterId))

	parts := strings.Split(
		aws.StringValue(cluster.CacheClusterId),
		"-",
	)

	var topIndex int
	if clusteredRegex.Match(strBytes) {
		topIndex = len(parts) - 2 // Minus two because we want to stop before the second-to-last part
		return strings.Join(parts[:topIndex], "-")

	} else if unclusteredRegex.Match(strBytes) {
		topIndex = len(parts) - 1 // Minus one because we want to stop before the last part
		return strings.Join(parts[:topIndex], "-")

	} else {
		// Return the original value if it doesn't match either pattern
		return aws.StringValue(cluster.CacheClusterId)
	}
}

func iterateCacheParameterGroups(client elasticacheiface.ElastiCacheAPI, fn func(*awsec.CacheParameterGroup) error) error {
	errs := []error{}
	err := client.DescribeCacheParameterGroupsPages(
		&awsec.DescribeCacheParameterGroupsInput{},
		func(page *awsec.DescribeCacheParameterGroupsOutput, lastPage bool) bool {
			for _, cacheParameterGroup := range page.CacheParameterGroups {
				err := fn(cacheParameterGroup)
				if err != nil {
					errs = append(errs, err)
				}
			}
			return true
		},
	)
	if err != nil {
		return err
	}
	if len(errs) > 0 {
		return errs[0]
	}
	return nil
}

func iterateCacheClusterPages(client elasticacheiface.ElastiCacheAPI, fn func(cluster *awsec.CacheCluster) error) error {
	errs := []error{}
	err := client.DescribeCacheClustersPages(
		&awsec.DescribeCacheClustersInput{},
		func(page *awsec.DescribeCacheClustersOutput, lastPage bool) bool {
			for _, cacheCluster := range page.CacheClusters {
				err := fn(cacheCluster)
				if err != nil {
					errs = append(errs, err)
				}
			}
			return true
		},
	)
	if err != nil {
		return err
	}
	if len(errs) > 0 {
		return errs[0]
	}
	return nil
}
<|endoftext|>


<commit_before>package options

import (
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/hashicorp/go-version"
)

var TERRAFORM_COMMANDS_WITH_SUBCOMMAND = []string{
	"debug",
	"force-unlock",
	"state",
}

const DEFAULT_MAX_FOLDERS_TO_CHECK = 100

const DEFAULT_PARALLELISM = 50

const TerragruntCacheDir = ".terragrunt-cache"

// TerragruntOptions represents options that configure the behavior of the Terragrunt program
type TerragruntOptions struct {
	// Location of the Terragrunt config file
	TerragruntConfigPath string

	// Location of the terraform binary
	TerraformPath string

	// Current Terraform command being executed by Terragrunt
	TerraformCommand string

	// Version of terraform (obtained by running 'terraform version')
	TerraformVersion *version.Version

	// Whether we should prompt the user for confirmation or always assume "yes"
	NonInteractive bool

	// Whether we should automatically run terraform init if necessary when executing other commands
	AutoInit bool

	// CLI args that are intended for Terraform (i.e. all the CLI args except the --terragrunt ones)
	TerraformCliArgs []string

	// The working directory in which to run Terraform
	WorkingDir string

	// The logger to use for all logging
	Logger *log.Logger

	// Environment variables at runtime
	Env map[string]string

	// Download Terraform configurations from the specified source location into a temporary folder and run
	// Terraform in that temporary folder
	Source string

	// If set to true, delete the contents of the temporary folder before downloading Terraform source code into it
	SourceUpdate bool

	// Download Terraform configurations specified in the Source parameter into this folder
	DownloadDir string

	// The ARN of an IAM Role to assume before running Terraform
	IamRole string

	// If set to true, continue running *-all commands even if a dependency has errors. This is mostly useful for 'output-all <some_variable>'. See https://github.com/gruntwork-io/terragrunt/issues/193
	IgnoreDependencyErrors bool

	// If you want stdout to go somewhere other than os.stdout
	Writer io.Writer

	// If you want stderr to go somewhere other than os.stderr
	ErrWriter io.Writer

	// When searching the directory tree, this is the max folders to check before exiting with an error. This is
	// exposed here primarily so we can set it to a low value at test time.
	MaxFoldersToCheck int

	// Whether we should automatically run terraform init if necessary when executing other commands
	AutoRetry bool

	// Maximum number of times to retry errors matching RetryableErrors
	MaxRetryAttempts int

	// Sleep is the duration in seconds to wait before retrying
	Sleep time.Duration

	// RetryableErrors is an array of regular expressions with RE2 syntax (https://github.com/google/re2/wiki/Syntax) that qualify for retrying
	RetryableErrors []string

	// Unix-style glob of directories to exclude when running *-all commands
	ExcludeDirs []string

	// Unix-style glob of directories to include when running *-all commands
	IncludeDirs []string

	// Parallelism limits the number of commands to run concurrently during *-all commands
	Parallelism int

	// A command that can be used to run Terragrunt with the given options. This is useful for running Terragrunt
	// multiple times (e.g. when spinning up a stack of Terraform modules). The actual command is normally defined
	// in the cli package, which depends on almost all other packages, so we declare it here so that other
	// packages can use the command without a direct reference back to the cli package (which would create a
	// circular dependency).
	RunTerragrunt func(*TerragruntOptions) error
}

// Create a new TerragruntOptions object with reasonable defaults for real usage
func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, error) {
	logger := util.CreateLogger("")

	workingDir, downloadDir, err := DefaultWorkingAndDownloadDirs(terragruntConfigPath)
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return &TerragruntOptions{
		TerragruntConfigPath:   terragruntConfigPath,
		TerraformPath:          "terraform",
		TerraformCommand:       "",
		AutoInit:               true,
		NonInteractive:         false,
		TerraformCliArgs:       []string{},
		WorkingDir:             workingDir,
		Logger:                 logger,
		Env:                    map[string]string{},
		Source:                 "",
		SourceUpdate:           false,
		DownloadDir:            downloadDir,
		IgnoreDependencyErrors: false,
		Writer:                 os.Stdout,
		ErrWriter:              os.Stderr,
		MaxFoldersToCheck:      DEFAULT_MAX_FOLDERS_TO_CHECK,
		AutoRetry:              true,
		MaxRetryAttempts:       DEFAULT_MAX_RETRY_ATTEMPTS,
		Sleep:                  DEFAULT_SLEEP,
		RetryableErrors:        util.CloneStringList(RETRYABLE_ERRORS),
		ExcludeDirs:            []string{},
		IncludeDirs:            []string{},
		Parallelism:            DEFAULT_PARALLELISM,
		RunTerragrunt: func(terragruntOptions *TerragruntOptions) error {
			return errors.WithStackTrace(RunTerragruntCommandNotSet)
		},
	}, nil
}

// Get the default working and download directories for the given Terragrunt config path
func DefaultWorkingAndDownloadDirs(terragruntConfigPath string) (string, string, error) {
	workingDir := filepath.Dir(terragruntConfigPath)

	downloadDir, err := filepath.Abs(filepath.Join(workingDir, TerragruntCacheDir))
	if err != nil {
		return "", "", errors.WithStackTrace(err)
	}

	return workingDir, downloadDir, nil
}

// Create a new TerragruntOptions object with reasonable defaults for test usage
func NewTerragruntOptionsForTest(terragruntConfigPath string) (*TerragruntOptions, error) {
	opts, err := NewTerragruntOptions(terragruntConfigPath)

	if err != nil {
		logger := util.CreateLogger("")
		logger.Printf("error: %v\n", errors.WithStackTrace(err))
		return nil, err
	}

	opts.NonInteractive = true

	return opts, nil
}

// Create a copy of this TerragruntOptions, but with different values for the given variables. This is useful for
// creating a TerragruntOptions that behaves the same way, but is used for a Terraform module in a different folder.
func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) *TerragruntOptions {
	workingDir := filepath.Dir(terragruntConfigPath)

	// Note that we clone lists and maps below as TerragruntOptions may be used and modified concurrently in the code
	// during xxx-all commands (e.g., apply-all, plan-all). See https://github.com/gruntwork-io/terragrunt/issues/367
	// for more info.
	return &TerragruntOptions{
		TerragruntConfigPath:   terragruntConfigPath,
		TerraformPath:          terragruntOptions.TerraformPath,
		TerraformCommand:       terragruntOptions.TerraformCommand,
		TerraformVersion:       terragruntOptions.TerraformVersion,
		AutoInit:               terragruntOptions.AutoInit,
		NonInteractive:         terragruntOptions.NonInteractive,
		TerraformCliArgs:       util.CloneStringList(terragruntOptions.TerraformCliArgs),
		WorkingDir:             workingDir,
		Logger:                 util.CreateLoggerWithWriter(terragruntOptions.ErrWriter, workingDir),
		Env:                    util.CloneStringMap(terragruntOptions.Env),
		Source:                 terragruntOptions.Source,
		SourceUpdate:           terragruntOptions.SourceUpdate,
		DownloadDir:            terragruntOptions.DownloadDir,
		IamRole:                terragruntOptions.IamRole,
		IgnoreDependencyErrors: terragruntOptions.IgnoreDependencyErrors,
		Writer:                 terragruntOptions.Writer,
		ErrWriter:              terragruntOptions.ErrWriter,
		MaxFoldersToCheck:      terragruntOptions.MaxFoldersToCheck,
		AutoRetry:              terragruntOptions.AutoRetry,
		MaxRetryAttempts:       terragruntOptions.MaxRetryAttempts,
		Sleep:                  terragruntOptions.Sleep,
		RetryableErrors:        util.CloneStringList(terragruntOptions.RetryableErrors),
		ExcludeDirs:            terragruntOptions.ExcludeDirs,
		IncludeDirs:            terragruntOptions.IncludeDirs,
		Parallelism:            terragruntOptions.Parallelism,
		RunTerragrunt:          terragruntOptions.RunTerragrunt,
	}
}

// Inserts the given argsToInsert after the terraform command argument, but before the remaining args
func (terragruntOptions *TerragruntOptions) InsertTerraformCliArgs(argsToInsert ...string) {

	commandLength := 1
	if util.ListContainsElement(TERRAFORM_COMMANDS_WITH_SUBCOMMAND, terragruntOptions.TerraformCliArgs[0]) {
		// Since these terraform commands require subcommands which may not always be properly passed by the user,
		// using util.Min to return the minimum to avoid potential out of bounds slice errors.
		commandLength = util.Min(2, len(terragruntOptions.TerraformCliArgs))
	}

	// Options must be inserted after command but before the other args
	// command is either 1 word or 2 words
	var args []string
	args = append(args, terragruntOptions.TerraformCliArgs[:commandLength]...)
	args = append(args, argsToInsert...)
	args = append(args, terragruntOptions.TerraformCliArgs[commandLength:]...)
	terragruntOptions.TerraformCliArgs = args
}

// Appends the given argsToAppend after the current TerraformCliArgs
func (terragruntOptions *TerragruntOptions) AppendTerraformCliArgs(argsToAppend ...string) {
	terragruntOptions.TerraformCliArgs = append(terragruntOptions.TerraformCliArgs, argsToAppend...)
}

// Custom error types

var RunTerragruntCommandNotSet = fmt.Errorf("The RunTerragrunt option has not been set on this TerragruntOptions object")
<commit_msg>change default limit to 10<commit_after>package options

import (
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/hashicorp/go-version"
)

var TERRAFORM_COMMANDS_WITH_SUBCOMMAND = []string{
	"debug",
	"force-unlock",
	"state",
}

const DEFAULT_MAX_FOLDERS_TO_CHECK = 100

const DEFAULT_PARALLELISM = 10

const TerragruntCacheDir = ".terragrunt-cache"

// TerragruntOptions represents options that configure the behavior of the Terragrunt program
type TerragruntOptions struct {
	// Location of the Terragrunt config file
	TerragruntConfigPath string

	// Location of the terraform binary
	TerraformPath string

	// Current Terraform command being executed by Terragrunt
	TerraformCommand string

	// Version of terraform (obtained by running 'terraform version')
	TerraformVersion *version.Version

	// Whether we should prompt the user for confirmation or always assume "yes"
	NonInteractive bool

	// Whether we should automatically run terraform init if necessary when executing other commands
	AutoInit bool

	// CLI args that are intended for Terraform (i.e. all the CLI args except the --terragrunt ones)
	TerraformCliArgs []string

	// The working directory in which to run Terraform
	WorkingDir string

	// The logger to use for all logging
	Logger *log.Logger

	// Environment variables at runtime
	Env map[string]string

	// Download Terraform configurations from the specified source location into a temporary folder and run
	// Terraform in that temporary folder
	Source string

	// If set to true, delete the contents of the temporary folder before downloading Terraform source code into it
	SourceUpdate bool

	// Download Terraform configurations specified in the Source parameter into this folder
	DownloadDir string

	// The ARN of an IAM Role to assume before running Terraform
	IamRole string

	// If set to true, continue running *-all commands even if a dependency has errors. This is mostly useful for 'output-all <some_variable>'. See https://github.com/gruntwork-io/terragrunt/issues/193
	IgnoreDependencyErrors bool

	// If you want stdout to go somewhere other than os.stdout
	Writer io.Writer

	// If you want stderr to go somewhere other than os.stderr
	ErrWriter io.Writer

	// When searching the directory tree, this is the max folders to check before exiting with an error. This is
	// exposed here primarily so we can set it to a low value at test time.
	MaxFoldersToCheck int

	// Whether we should automatically run terraform init if necessary when executing other commands
	AutoRetry bool

	// Maximum number of times to retry errors matching RetryableErrors
	MaxRetryAttempts int

	// Sleep is the duration in seconds to wait before retrying
	Sleep time.Duration

	// RetryableErrors is an array of regular expressions with RE2 syntax (https://github.com/google/re2/wiki/Syntax) that qualify for retrying
	RetryableErrors []string

	// Unix-style glob of directories to exclude when running *-all commands
	ExcludeDirs []string

	// Unix-style glob of directories to include when running *-all commands
	IncludeDirs []string

	// Parallelism limits the number of commands to run concurrently during *-all commands
	Parallelism int

	// A command that can be used to run Terragrunt with the given options. This is useful for running Terragrunt
	// multiple times (e.g. when spinning up a stack of Terraform modules). The actual command is normally defined
	// in the cli package, which depends on almost all other packages, so we declare it here so that other
	// packages can use the command without a direct reference back to the cli package (which would create a
	// circular dependency).
	RunTerragrunt func(*TerragruntOptions) error
}

// Create a new TerragruntOptions object with reasonable defaults for real usage
func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, error) {
	logger := util.CreateLogger("")

	workingDir, downloadDir, err := DefaultWorkingAndDownloadDirs(terragruntConfigPath)
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return &TerragruntOptions{
		TerragruntConfigPath:   terragruntConfigPath,
		TerraformPath:          "terraform",
		TerraformCommand:       "",
		AutoInit:               true,
		NonInteractive:         false,
		TerraformCliArgs:       []string{},
		WorkingDir:             workingDir,
		Logger:                 logger,
		Env:                    map[string]string{},
		Source:                 "",
		SourceUpdate:           false,
		DownloadDir:            downloadDir,
		IgnoreDependencyErrors: false,
		Writer:                 os.Stdout,
		ErrWriter:              os.Stderr,
		MaxFoldersToCheck:      DEFAULT_MAX_FOLDERS_TO_CHECK,
		AutoRetry:              true,
		MaxRetryAttempts:       DEFAULT_MAX_RETRY_ATTEMPTS,
		Sleep:                  DEFAULT_SLEEP,
		RetryableErrors:        util.CloneStringList(RETRYABLE_ERRORS),
		ExcludeDirs:            []string{},
		IncludeDirs:            []string{},
		Parallelism:            DEFAULT_PARALLELISM,
		RunTerragrunt: func(terragruntOptions *TerragruntOptions) error {
			return errors.WithStackTrace(RunTerragruntCommandNotSet)
		},
	}, nil
}

// Get the default working and download directories for the given Terragrunt config path
func DefaultWorkingAndDownloadDirs(terragruntConfigPath string) (string, string, error) {
	workingDir := filepath.Dir(terragruntConfigPath)

	downloadDir, err := filepath.Abs(filepath.Join(workingDir, TerragruntCacheDir))
	if err != nil {
		return "", "", errors.WithStackTrace(err)
	}

	return workingDir, downloadDir, nil
}

// Create a new TerragruntOptions object with reasonable defaults for test usage
func NewTerragruntOptionsForTest(terragruntConfigPath string) (*TerragruntOptions, error) {
	opts, err := NewTerragruntOptions(terragruntConfigPath)

	if err != nil {
		logger := util.CreateLogger("")
		logger.Printf("error: %v\n", errors.WithStackTrace(err))
		return nil, err
	}

	opts.NonInteractive = true

	return opts, nil
}

// Create a copy of this TerragruntOptions, but with different values for the given variables. This is useful for
// creating a TerragruntOptions that behaves the same way, but is used for a Terraform module in a different folder.
func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) *TerragruntOptions {
	workingDir := filepath.Dir(terragruntConfigPath)

	// Note that we clone lists and maps below as TerragruntOptions may be used and modified concurrently in the code
	// during xxx-all commands (e.g., apply-all, plan-all). See https://github.com/gruntwork-io/terragrunt/issues/367
	// for more info.
	return &TerragruntOptions{
		TerragruntConfigPath:   terragruntConfigPath,
		TerraformPath:          terragruntOptions.TerraformPath,
		TerraformCommand:       terragruntOptions.TerraformCommand,
		TerraformVersion:       terragruntOptions.TerraformVersion,
		AutoInit:               terragruntOptions.AutoInit,
		NonInteractive:         terragruntOptions.NonInteractive,
		TerraformCliArgs:       util.CloneStringList(terragruntOptions.TerraformCliArgs),
		WorkingDir:             workingDir,
		Logger:                 util.CreateLoggerWithWriter(terragruntOptions.ErrWriter, workingDir),
		Env:                    util.CloneStringMap(terragruntOptions.Env),
		Source:                 terragruntOptions.Source,
		SourceUpdate:           terragruntOptions.SourceUpdate,
		DownloadDir:            terragruntOptions.DownloadDir,
		IamRole:                terragruntOptions.IamRole,
		IgnoreDependencyErrors: terragruntOptions.IgnoreDependencyErrors,
		Writer:                 terragruntOptions.Writer,
		ErrWriter:              terragruntOptions.ErrWriter,
		MaxFoldersToCheck:      terragruntOptions.MaxFoldersToCheck,
		AutoRetry:              terragruntOptions.AutoRetry,
		MaxRetryAttempts:       terragruntOptions.MaxRetryAttempts,
		Sleep:                  terragruntOptions.Sleep,
		RetryableErrors:        util.CloneStringList(terragruntOptions.RetryableErrors),
		ExcludeDirs:            terragruntOptions.ExcludeDirs,
		IncludeDirs:            terragruntOptions.IncludeDirs,
		Parallelism:            terragruntOptions.Parallelism,
		RunTerragrunt:          terragruntOptions.RunTerragrunt,
	}
}

// Inserts the given argsToInsert after the terraform command argument, but before the remaining args
func (terragruntOptions *TerragruntOptions) InsertTerraformCliArgs(argsToInsert ...string) {

	commandLength := 1
	if util.ListContainsElement(TERRAFORM_COMMANDS_WITH_SUBCOMMAND, terragruntOptions.TerraformCliArgs[0]) {
		// Since these terraform commands require subcommands which may not always be properly passed by the user,
		// using util.Min to return the minimum to avoid potential out of bounds slice errors.
		commandLength = util.Min(2, len(terragruntOptions.TerraformCliArgs))
	}

	// Options must be inserted after command but before the other args
	// command is either 1 word or 2 words
	var args []string
	args = append(args, terragruntOptions.TerraformCliArgs[:commandLength]...)
	args = append(args, argsToInsert...)
	args = append(args, terragruntOptions.TerraformCliArgs[commandLength:]...)
	terragruntOptions.TerraformCliArgs = args
}

// Appends the given argsToAppend after the current TerraformCliArgs
func (terragruntOptions *TerragruntOptions) AppendTerraformCliArgs(argsToAppend ...string) {
	terragruntOptions.TerraformCliArgs = append(terragruntOptions.TerraformCliArgs, argsToAppend...)
}

// Custom error types

var RunTerragruntCommandNotSet = fmt.Errorf("The RunTerragrunt option has not been set on this TerragruntOptions object")
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"net/textproto"
	"os"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
)

// PGPTransformer provides support for converting arbitrary plain messages to PGP/MIME
// messages in a way which allows for bit-perfect reversal of the operation.
type PGPTransformer struct {
	pgpBuffer     *bytes.Buffer
	outBuffer     *bytes.Buffer
	plainBuffer   *bytes.Buffer
	pgpWriter     io.WriteCloser
	asciiWriter   io.WriteCloser
	encryptionKey *openpgp.Entity
	signingKey    *openpgp.Entity
	keepHeaders   []string
}

// NewPGPTransformer returns a new PGPTransformer instance.
func NewPGPTransformer(keepHeaders []string) *PGPTransformer {
	return &PGPTransformer{keepHeaders: keepHeaders}
}

// LoadEncryptionKey loads the keyring from the given path and tries to set up the
// first and only public key found there as the encryption target.
func (w *PGPTransformer) LoadEncryptionKey(path string) error {
	logger.Debugf("loading encryption key from %s", path)
	var err error
	w.encryptionKey, err = w.loadKey(path)
	return err
}

// LoadSigningKey loads the keyring from the given path and tries to so set up
// the first and only private key found there as the signing key, optionally
// decrypting it with the given passphrase first.
func (w *PGPTransformer) LoadSigningKey(path, passphrase string) error {
	logger.Debugf("loading signing key from %s", path)
	var err error
	w.signingKey, err = w.loadKey(path)
	if err != nil {
		return err
	}
	priv := w.signingKey.PrivateKey
	if priv == nil {
		return errors.New("signing key lacks private key")
	}
	if priv.Encrypted {
		err := priv.Decrypt([]byte(passphrase))
		if err != nil {
			return fmt.Errorf("failed to decrypt private key: %s", err)
		}
	}
	return nil
}

// loadKey is the internal method which contains the common key loading and
// parsing functionality.
func (w *PGPTransformer) loadKey(path string) (*openpgp.Entity, error) {
	keyringReader, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer keyringReader.Close()
	keyring, err := openpgp.ReadArmoredKeyRing(keyringReader)
	if err != nil {
		return nil, err
	}
	if len(keyring) != 1 {
		logger.Errorf("encryption key ring contains %d keys; expected 1", len(keyring))
		return nil, errors.New("encryption key ring must contain exactly one key")
	}
	return keyring[0], nil
}

// Reset configures the PGPTransformer to be ready for the first or next message
// transformation.
// Call this method before initially passing the transformer data or before passing
// it any new data after finishing handling of the previous item.
func (w *PGPTransformer) Reset() error {
	w.pgpBuffer = &bytes.Buffer{}
	w.plainBuffer = &bytes.Buffer{}
	var err error
	w.asciiWriter, err = armor.Encode(w.pgpBuffer, "PGP MESSAGE", nil)
	if err != nil {
		return err
	}
	if w.encryptionKey == nil {
		return errors.New("missing encryption key")
	}
	w.pgpWriter, err = openpgp.Encrypt(w.asciiWriter,
		[]*openpgp.Entity{w.encryptionKey}, w.signingKey,
		&openpgp.FileHints{IsBinary: true}, nil)
	return err
}

// Write passes the given data to the underlying PGP encryptor.
func (w *PGPTransformer) Write(data []byte) (int, error) {
	_, err := w.plainBuffer.Write(data)
	if err != nil {
		return 0, err
	}
	return w.pgpWriter.Write(data)
}

// GetBytes returns the encrypted message as a byte array.
func (w *PGPTransformer) GetBytes() ([]byte, error) {
	w.finalizePGP()
	err := w.finalizeMIME()
	if err != nil {
		return nil, err
	}
	return w.outBuffer.Bytes(), nil
}

// finalizePGP ends the PGP encryption process and ascii-encoding process.
func (w *PGPTransformer) finalizePGP() {
	w.pgpWriter.Close()
	w.asciiWriter.Close()
}

// finalizeMIME finally encodes the PGP ascii-armored data in a MIME message.
func (w *PGPTransformer) finalizeMIME() error {
	tmp := make([]byte, 30)
	_, err := io.ReadFull(rand.Reader, tmp)
	if err != nil {
		return err
	}
	boundary := fmt.Sprintf("%x", tmp)
	w.outBuffer = &bytes.Buffer{}
	err = w.writePlainHeaders()
	if err != nil {
		return err
	}
	w.outBuffer.WriteString(
		"MIME-Version: 1.0\n" +
			"Content-Type: multipart/encrypted;\n" +
			" protocol=\"application/pgp-encrypted\";\n" +
			" boundary=\"" + boundary + "\"\n\n" +
			"OpenPGP/MIME\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/pgp-encrypted\n\n" +
			"Version: 1\n\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/octet-stream; name=\"encrypted.asc\"\n" +
			"Content-Disposition: inline; filename=\"encrypted.asc\"\n\n" +
			string(w.pgpBuffer.Bytes()) + "\n--" + boundary + "--")
	return nil
}

// writePlainHeaders generates Message-Id and copies all the plain headers which are
// configured to be copied up from the original message to the output buffer.
func (w *PGPTransformer) writePlainHeaders() error {
	plainReader := bufio.NewReader(w.plainBuffer)
	mimeReader := textproto.NewReader(plainReader)
	headers, err := mimeReader.ReadMIMEHeader()
	if err != nil {
		return err
	}
	msgid := headers.Get("Message-Id")
	if msgid != "" {
		prefix := "lemoncrypt."
		if len(msgid) > 1 && msgid[0] == '<' {
			msgid = msgid[0:1] + prefix + msgid[1:]
		} else {
			msgid += prefix + msgid[1:]
		}
	}
	w.outBuffer.WriteString("Message-Id: " + msgid + "\n")
	for _, key := range w.keepHeaders {
		val := headers.Get(key)
		if val == "" {
			// don't attempt to copy empty headers
			continue
		}
		//FIXME proper line wrapping; not obvious how go does it
		w.outBuffer.WriteString(key + ": " + val + "\n")
	}
	return nil
}
<commit_msg>pgp: refactor<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"net/textproto"
	"os"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
)

// PGPTransformer provides support for converting arbitrary plain messages to PGP/MIME
// messages in a way which allows for bit-perfect reversal of the operation.
type PGPTransformer struct {
	pgpBuffer     *bytes.Buffer
	outBuffer     *bytes.Buffer
	plainBuffer   *bytes.Buffer
	pgpWriter     io.WriteCloser
	asciiWriter   io.WriteCloser
	encryptionKey *openpgp.Entity
	signingKey    *openpgp.Entity
	keepHeaders   []string
}

// NewPGPTransformer returns a new PGPTransformer instance.
func NewPGPTransformer(keepHeaders []string) *PGPTransformer {
	return &PGPTransformer{keepHeaders: keepHeaders}
}

// LoadEncryptionKey loads the keyring from the given path and tries to set up the
// first and only public key found there as the encryption target.
func (w *PGPTransformer) LoadEncryptionKey(path string) error {
	logger.Debugf("loading encryption key from %s", path)
	var err error
	w.encryptionKey, err = w.loadKey(path)
	return err
}

// LoadSigningKey loads the keyring from the given path and tries to so set up
// the first and only private key found there as the signing key, optionally
// decrypting it with the given passphrase first.
func (w *PGPTransformer) LoadSigningKey(path, passphrase string) error {
	logger.Debugf("loading signing key from %s", path)
	var err error
	w.signingKey, err = w.loadKey(path)
	if err != nil {
		return err
	}
	priv := w.signingKey.PrivateKey
	if priv == nil {
		return errors.New("signing key lacks private key")
	}
	if priv.Encrypted {
		err := priv.Decrypt([]byte(passphrase))
		if err != nil {
			return fmt.Errorf("failed to decrypt private key: %s", err)
		}
	}
	return nil
}

// loadKey is the internal method which contains the common key loading and
// parsing functionality.
func (w *PGPTransformer) loadKey(path string) (*openpgp.Entity, error) {
	keyringReader, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer keyringReader.Close()
	keyring, err := openpgp.ReadArmoredKeyRing(keyringReader)
	if err != nil {
		return nil, err
	}
	if len(keyring) != 1 {
		logger.Errorf("encryption key ring contains %d keys; expected 1", len(keyring))
		return nil, errors.New("encryption key ring must contain exactly one key")
	}
	return keyring[0], nil
}

// Reset configures the PGPTransformer to be ready for the first or next message
// transformation.
// Call this method before initially passing the transformer data or before passing
// it any new data after finishing handling of the previous item.
func (w *PGPTransformer) Reset() error {
	w.pgpBuffer = &bytes.Buffer{}
	w.plainBuffer = &bytes.Buffer{}
	var err error
	w.asciiWriter, err = armor.Encode(w.pgpBuffer, "PGP MESSAGE", nil)
	if err != nil {
		return err
	}
	if w.encryptionKey == nil {
		return errors.New("missing encryption key")
	}
	w.pgpWriter, err = openpgp.Encrypt(w.asciiWriter,
		[]*openpgp.Entity{w.encryptionKey}, w.signingKey,
		&openpgp.FileHints{IsBinary: true}, nil)
	return err
}

// Write passes the given data to the underlying PGP encryptor.
func (w *PGPTransformer) Write(data []byte) (int, error) {
	_, err := w.plainBuffer.Write(data)
	if err != nil {
		return 0, err
	}
	return w.pgpWriter.Write(data)
}

// GetBytes returns the encrypted message as a byte array.
func (w *PGPTransformer) GetBytes() ([]byte, error) {
	w.finalizePGP()
	err := w.finalizeMIME()
	if err != nil {
		return nil, err
	}
	return w.outBuffer.Bytes(), nil
}

// finalizePGP ends the PGP encryption process and ascii-encoding process.
func (w *PGPTransformer) finalizePGP() {
	w.pgpWriter.Close()
	w.asciiWriter.Close()
}

// finalizeMIME finally encodes the PGP ascii-armored data in a MIME message.
func (w *PGPTransformer) finalizeMIME() error {
	w.outBuffer = &bytes.Buffer{}
	err := w.writePlainHeaders()
	if err != nil {
		return err
	}
	return w.writeMIMEStructure()
}

// writePlainHeaders generates Message-Id and copies all the plain headers which are
// configured to be copied up from the original message to the output buffer.
func (w *PGPTransformer) writePlainHeaders() error {
	plainReader := bufio.NewReader(w.plainBuffer)
	mimeReader := textproto.NewReader(plainReader)
	headers, err := mimeReader.ReadMIMEHeader()
	if err != nil {
		return err
	}
	msgid := headers.Get("Message-Id")
	if msgid != "" {
		prefix := "lemoncrypt."
		if len(msgid) > 1 && msgid[0] == '<' {
			msgid = msgid[0:1] + prefix + msgid[1:]
		} else {
			msgid += prefix + msgid[1:]
		}
	}
	w.outBuffer.WriteString("Message-Id: " + msgid + "\n")
	for _, key := range w.keepHeaders {
		val := headers.Get(key)
		if val == "" {
			// don't attempt to copy empty headers
			continue
		}
		//FIXME proper line wrapping; not obvious how go does it
		w.outBuffer.WriteString(key + ": " + val + "\n")
	}
	return nil
}

// writeMIMEStructure writes the basic MIME structure and the encrypted content
// to the output buffer.
func (w *PGPTransformer) writeMIMEStructure() error {
	boundary, err := generateBoundary()
	if err != nil {
		return err
	}
	w.outBuffer.WriteString(
		"MIME-Version: 1.0\n" +
			"Content-Type: multipart/encrypted;\n" +
			" protocol=\"application/pgp-encrypted\";\n" +
			" boundary=\"" + boundary + "\"\n\n" +
			"OpenPGP/MIME\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/pgp-encrypted\n\n" +
			"Version: 1\n\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/octet-stream; name=\"encrypted.asc\"\n" +
			"Content-Disposition: inline; filename=\"encrypted.asc\"\n\n" +
			string(w.pgpBuffer.Bytes()) + "\n--" + boundary + "--")
	return nil
}

// generateBoundary creates a random boundary string suitable for
// MIME part separation.
func generateBoundary() (string, error) {
	tmp := make([]byte, 30)
	_, err := io.ReadFull(rand.Reader, tmp)
	if err != nil {
		return "", err
	}
	boundary := fmt.Sprintf("%x", tmp)
	return boundary, nil
}
<|endoftext|>


<commit_before>package nat

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	ma "github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-multiaddr-net"

	nat "github.com/fd/go-nat"
	goprocess "github.com/jbenet/goprocess"
	periodic "github.com/jbenet/goprocess/periodic"
	notifier "thirdparty/notifier"
	logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log"
)

var (
	// ErrNoMapping signals no mapping exists for an address
	ErrNoMapping = errors.New("mapping not established")
)

var log = logging.Logger("nat")

// MappingDuration is a default port mapping duration.
// Port mappings are renewed every (MappingDuration / 3)
const MappingDuration = time.Second * 60

// CacheTime is the time a mapping will cache an external address for
const CacheTime = time.Second * 15

// DiscoverNAT looks for a NAT device in the network and
// returns an object that can manage port mappings.
func DiscoverNAT() *NAT {
	nat, err := nat.DiscoverGateway()
	if err != nil {
		log.Debug("DiscoverGateway error:", err)
		return nil
	}
	addr, err := nat.GetDeviceAddress()
	if err != nil {
		log.Debug("DiscoverGateway address error:", err)
	} else {
		log.Debug("DiscoverGateway address:", addr)
	}
	return newNAT(nat)
}

// NAT is an object that manages address port mappings in
// NATs (Network Address Translators). It is a long-running
// service that will periodically renew port mappings,
// and keep an up-to-date list of all the external addresses.
type NAT struct {
	nat  nat.NAT
	proc goprocess.Process // manages nat mappings lifecycle

	mappingmu sync.RWMutex // guards mappings
	mappings  map[*mapping]struct{}

	Notifier
}

func newNAT(realNAT nat.NAT) *NAT {
	return &NAT{
		nat:      realNAT,
		proc:     goprocess.WithParent(goprocess.Background()),
		mappings: make(map[*mapping]struct{}),
	}
}

// Close shuts down all port mappings. NAT can no longer be used.
func (nat *NAT) Close() error {
	return nat.proc.Close()
}

// Process returns the nat's life-cycle manager, for making it listen
// to close signals.
func (nat *NAT) Process() goprocess.Process {
	return nat.proc
}

// Notifier is an object that assists NAT in notifying listeners.
// It is implemented using thirdparty/notifier
type Notifier struct {
	n notifier.Notifier
}

func (n *Notifier) notifyAll(notify func(n Notifiee)) {
	n.n.NotifyAll(func(n notifier.Notifiee) {
		notify(n.(Notifiee))
	})
}

// Notify signs up notifiee to listen to NAT events.
func (n *Notifier) Notify(notifiee Notifiee) {
	n.n.Notify(n)
}

// StopNotify stops signaling events to notifiee.
func (n *Notifier) StopNotify(notifiee Notifiee) {
	n.n.StopNotify(notifiee)
}

// Notifiee is an interface objects must implement to listen to NAT events.
type Notifiee interface {

	// Called every time a successful mapping happens
	// Warning: the port mapping may have changed. If that is the
	// case, both MappingSuccess and MappingChanged are called.
	MappingSuccess(nat *NAT, m Mapping)

	// Called when mapping a port succeeds, but the mapping is
	// with a different port than an earlier success.
	MappingChanged(nat *NAT, m Mapping, oldport int, newport int)

	// Called when a port mapping fails. NAT will continue attempting after
	// the next period. To stop trying, use: mapping.Close(). After this failure,
	// mapping.ExternalPort() will be zero, and nat.ExternalAddrs() will not
	// return the address for this mapping. With luck, the next attempt will
	// succeed, without the client needing to do anything.
	MappingFailed(nat *NAT, m Mapping, oldport int, err error)
}

// Mapping represents a port mapping in a NAT.
type Mapping interface {
	// NAT returns the NAT object this Mapping belongs to.
	NAT() *NAT

	// Protocol returns the protocol of this port mapping. This is either
	// "tcp" or "udp" as no other protocols are likely to be NAT-supported.
	Protocol() string

	// InternalPort returns the internal device port. Mapping will continue to
	// try to map InternalPort() to an external facing port.
	InternalPort() int

	// ExternalPort returns the external facing port. If the mapping is not
	// established, port will be 0
	ExternalPort() int

	// InternalAddr returns the internal address.
	InternalAddr() ma.Multiaddr

	// ExternalAddr returns the external facing address. If the mapping is not
	// established, addr will be nil, and and ErrNoMapping will be returned.
	ExternalAddr() (addr ma.Multiaddr, err error)

	// Close closes the port mapping
	Close() error
}

// keeps republishing
type mapping struct {
	sync.Mutex // guards all fields

	nat     *NAT
	proto   string
	intport int
	extport int
	intaddr ma.Multiaddr
	proc    goprocess.Process

	cached    ma.Multiaddr
	cacheTime time.Time
}

func (m *mapping) NAT() *NAT {
	m.Lock()
	defer m.Unlock()
	return m.nat
}

func (m *mapping) Protocol() string {
	m.Lock()
	defer m.Unlock()
	return m.proto
}

func (m *mapping) InternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.intport
}

func (m *mapping) ExternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.extport
}

func (m *mapping) setExternalPort(p int) {
	m.Lock()
	defer m.Unlock()
	m.extport = p
}

func (m *mapping) InternalAddr() ma.Multiaddr {
	m.Lock()
	defer m.Unlock()
	return m.intaddr
}

func (m *mapping) ExternalAddr() (ma.Multiaddr, error) {
	if time.Now().Sub(m.cacheTime) < CacheTime {
		return m.cached, nil
	}

	if m.ExternalPort() == 0 { // dont even try right now.
		return nil, ErrNoMapping
	}

	ip, err := m.nat.nat.GetExternalAddress()
	if err != nil {
		return nil, err
	}

	ipmaddr, err := manet.FromIP(ip)
	if err != nil {
		return nil, fmt.Errorf("error parsing ip")
	}

	// call m.ExternalPort again, as mapping may have changed under our feet. (tocttou)
	extport := m.ExternalPort()
	if extport == 0 {
		return nil, ErrNoMapping
	}

	tcp, err := ma.NewMultiaddr(fmt.Sprintf("/%s/%d", m.Protocol(), extport))
	if err != nil {
		return nil, err
	}

	maddr2 := ipmaddr.Encapsulate(tcp)

	m.cached = maddr2
	m.cacheTime = time.Now()
	return maddr2, nil
}

func (m *mapping) Close() error {
	return m.proc.Close()
}

// Mappings returns a slice of all NAT mappings
func (nat *NAT) Mappings() []Mapping {
	nat.mappingmu.Lock()
	maps2 := make([]Mapping, 0, len(nat.mappings))
	for m := range nat.mappings {
		maps2 = append(maps2, m)
	}
	nat.mappingmu.Unlock()
	return maps2
}

func (nat *NAT) addMapping(m *mapping) {
	// make mapping automatically close when nat is closed.
	nat.proc.AddChild(m.proc)

	nat.mappingmu.Lock()
	nat.mappings[m] = struct{}{}
	nat.mappingmu.Unlock()
}

func (nat *NAT) rmMapping(m *mapping) {
	nat.mappingmu.Lock()
	delete(nat.mappings, m)
	nat.mappingmu.Unlock()
}

// NewMapping attemps to construct a mapping on protocol and internal port
// It will also periodically renew the mapping until the returned Mapping
// -- or its parent NAT -- is Closed.
//
// May not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) NewMapping(maddr ma.Multiaddr) (Mapping, error) {
	if nat == nil {
		return nil, fmt.Errorf("no nat available")
	}

	network, addr, err := manet.DialArgs(maddr)
	if err != nil {
		return nil, fmt.Errorf("DialArgs failed on addr:", maddr.String())
	}

	switch network {
	case "tcp", "tcp4", "tcp6":
		network = "tcp"
	case "udp", "udp4", "udp6":
		network = "udp"
	default:
		return nil, fmt.Errorf("transport not supported by NAT: %s", network)
	}

	intports := strings.Split(addr, ":")[1]
	intport, err := strconv.Atoi(intports)
	if err != nil {
		return nil, err
	}

	m := &mapping{
		nat:     nat,
		proto:   network,
		intport: intport,
		intaddr: maddr,
	}
	m.proc = goprocess.WithTeardown(func() error {
		nat.rmMapping(m)
		return nil
	})
	nat.addMapping(m)

	m.proc.AddChild(periodic.Every(MappingDuration/3, func(worker goprocess.Process) {
		nat.establishMapping(m)
	}))

	// do it once synchronously, so first mapping is done right away, and before exiting,
	// allowing users -- in the optimistic case -- to use results right after.
	nat.establishMapping(m)
	return m, nil
}

func (nat *NAT) establishMapping(m *mapping) {
	oldport := m.ExternalPort()
	log.Debugf("Attempting port map: %s/%d", m.Protocol(), m.InternalPort())
	newport, err := nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), "http", MappingDuration)

	failure := func() {
		m.setExternalPort(0) // clear mapping
		// TODO: log.Event
		log.Debugf("failed to establish port mapping: %s", err)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingFailed(nat, m, oldport, err)
		})

		// we do not close if the mapping failed,
		// because it may work again next time.
	}

	if err != nil || newport == 0 {
		failure()
		return
	}

	m.setExternalPort(newport)
	ext, err := m.ExternalAddr()
	if err != nil {
		log.Debugf("NAT Mapping addr error: %s %s", m.InternalAddr(), err)
		failure()
		return
	}

	log.Debugf("NAT Mapping: %s --> %s", m.InternalAddr(), ext)
	if oldport != 0 && newport != oldport {
		log.Debugf("failed to renew same port mapping: ch %d -> %d", oldport, newport)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingChanged(nat, m, oldport, newport)
		})
	}

	nat.Notifier.notifyAll(func(n Notifiee) {
		n.MappingSuccess(nat, m)
	})
}

// PortMapAddrs attempts to open (and continue to keep open)
// port mappings for given addrs. This function blocks until
// all  addresses have been tried. This allows clients to
// retrieve results immediately after:
//
//   nat.PortMapAddrs(addrs)
//   mapped := nat.ExternalAddrs()
//
// Some may not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) PortMapAddrs(addrs []ma.Multiaddr) {
	// spin off addr mappings independently.
	var wg sync.WaitGroup
	for _, addr := range addrs {
		// do all of them concurrently
		wg.Add(1)
		go func() {
			defer wg.Done()
			nat.NewMapping(addr)
		}()
	}
	wg.Wait()
}

// MappedAddrs returns address mappings NAT believes have been
// successfully established. Unsuccessful mappings are nil. This is:
//
// 		map[internalAddr]externalAddr
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) MappedAddrs() map[ma.Multiaddr]ma.Multiaddr {

	mappings := nat.Mappings()
	addrmap := make(map[ma.Multiaddr]ma.Multiaddr, len(mappings))

	for _, m := range mappings {
		i := m.InternalAddr()
		e, err := m.ExternalAddr()
		if err != nil {
			addrmap[i] = nil
		} else {
			addrmap[i] = e
		}
	}
	return addrmap
}

// ExternalAddrs returns a list of addresses that NAT believes have
// been successfully established. Unsuccessful mappings are omitted,
// so nat.ExternalAddrs() may return less addresses than nat.InternalAddrs().
// To see which addresses are mapped, use nat.MappedAddrs().
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) ExternalAddrs() []ma.Multiaddr {
	mappings := nat.Mappings()
	addrs := make([]ma.Multiaddr, 0, len(mappings))
	for _, m := range mappings {
		a, err := m.ExternalAddr()
		if err != nil {
			continue // this mapping not currently successful.
		}
		addrs = append(addrs, a)
	}
	return addrs
}
<commit_msg>more vendoring<commit_after>package nat

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr"
	manet "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net"

	logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log"
	nat "github.com/fd/go-nat"
	goprocess "github.com/jbenet/goprocess"
	periodic "github.com/jbenet/goprocess/periodic"
	notifier "thirdparty/notifier"
)

var (
	// ErrNoMapping signals no mapping exists for an address
	ErrNoMapping = errors.New("mapping not established")
)

var log = logging.Logger("nat")

// MappingDuration is a default port mapping duration.
// Port mappings are renewed every (MappingDuration / 3)
const MappingDuration = time.Second * 60

// CacheTime is the time a mapping will cache an external address for
const CacheTime = time.Second * 15

// DiscoverNAT looks for a NAT device in the network and
// returns an object that can manage port mappings.
func DiscoverNAT() *NAT {
	nat, err := nat.DiscoverGateway()
	if err != nil {
		log.Debug("DiscoverGateway error:", err)
		return nil
	}
	addr, err := nat.GetDeviceAddress()
	if err != nil {
		log.Debug("DiscoverGateway address error:", err)
	} else {
		log.Debug("DiscoverGateway address:", addr)
	}
	return newNAT(nat)
}

// NAT is an object that manages address port mappings in
// NATs (Network Address Translators). It is a long-running
// service that will periodically renew port mappings,
// and keep an up-to-date list of all the external addresses.
type NAT struct {
	nat  nat.NAT
	proc goprocess.Process // manages nat mappings lifecycle

	mappingmu sync.RWMutex // guards mappings
	mappings  map[*mapping]struct{}

	Notifier
}

func newNAT(realNAT nat.NAT) *NAT {
	return &NAT{
		nat:      realNAT,
		proc:     goprocess.WithParent(goprocess.Background()),
		mappings: make(map[*mapping]struct{}),
	}
}

// Close shuts down all port mappings. NAT can no longer be used.
func (nat *NAT) Close() error {
	return nat.proc.Close()
}

// Process returns the nat's life-cycle manager, for making it listen
// to close signals.
func (nat *NAT) Process() goprocess.Process {
	return nat.proc
}

// Notifier is an object that assists NAT in notifying listeners.
// It is implemented using thirdparty/notifier
type Notifier struct {
	n notifier.Notifier
}

func (n *Notifier) notifyAll(notify func(n Notifiee)) {
	n.n.NotifyAll(func(n notifier.Notifiee) {
		notify(n.(Notifiee))
	})
}

// Notify signs up notifiee to listen to NAT events.
func (n *Notifier) Notify(notifiee Notifiee) {
	n.n.Notify(n)
}

// StopNotify stops signaling events to notifiee.
func (n *Notifier) StopNotify(notifiee Notifiee) {
	n.n.StopNotify(notifiee)
}

// Notifiee is an interface objects must implement to listen to NAT events.
type Notifiee interface {

	// Called every time a successful mapping happens
	// Warning: the port mapping may have changed. If that is the
	// case, both MappingSuccess and MappingChanged are called.
	MappingSuccess(nat *NAT, m Mapping)

	// Called when mapping a port succeeds, but the mapping is
	// with a different port than an earlier success.
	MappingChanged(nat *NAT, m Mapping, oldport int, newport int)

	// Called when a port mapping fails. NAT will continue attempting after
	// the next period. To stop trying, use: mapping.Close(). After this failure,
	// mapping.ExternalPort() will be zero, and nat.ExternalAddrs() will not
	// return the address for this mapping. With luck, the next attempt will
	// succeed, without the client needing to do anything.
	MappingFailed(nat *NAT, m Mapping, oldport int, err error)
}

// Mapping represents a port mapping in a NAT.
type Mapping interface {
	// NAT returns the NAT object this Mapping belongs to.
	NAT() *NAT

	// Protocol returns the protocol of this port mapping. This is either
	// "tcp" or "udp" as no other protocols are likely to be NAT-supported.
	Protocol() string

	// InternalPort returns the internal device port. Mapping will continue to
	// try to map InternalPort() to an external facing port.
	InternalPort() int

	// ExternalPort returns the external facing port. If the mapping is not
	// established, port will be 0
	ExternalPort() int

	// InternalAddr returns the internal address.
	InternalAddr() ma.Multiaddr

	// ExternalAddr returns the external facing address. If the mapping is not
	// established, addr will be nil, and and ErrNoMapping will be returned.
	ExternalAddr() (addr ma.Multiaddr, err error)

	// Close closes the port mapping
	Close() error
}

// keeps republishing
type mapping struct {
	sync.Mutex // guards all fields

	nat     *NAT
	proto   string
	intport int
	extport int
	intaddr ma.Multiaddr
	proc    goprocess.Process

	cached    ma.Multiaddr
	cacheTime time.Time
}

func (m *mapping) NAT() *NAT {
	m.Lock()
	defer m.Unlock()
	return m.nat
}

func (m *mapping) Protocol() string {
	m.Lock()
	defer m.Unlock()
	return m.proto
}

func (m *mapping) InternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.intport
}

func (m *mapping) ExternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.extport
}

func (m *mapping) setExternalPort(p int) {
	m.Lock()
	defer m.Unlock()
	m.extport = p
}

func (m *mapping) InternalAddr() ma.Multiaddr {
	m.Lock()
	defer m.Unlock()
	return m.intaddr
}

func (m *mapping) ExternalAddr() (ma.Multiaddr, error) {
	if time.Now().Sub(m.cacheTime) < CacheTime {
		return m.cached, nil
	}

	if m.ExternalPort() == 0 { // dont even try right now.
		return nil, ErrNoMapping
	}

	ip, err := m.nat.nat.GetExternalAddress()
	if err != nil {
		return nil, err
	}

	ipmaddr, err := manet.FromIP(ip)
	if err != nil {
		return nil, fmt.Errorf("error parsing ip")
	}

	// call m.ExternalPort again, as mapping may have changed under our feet. (tocttou)
	extport := m.ExternalPort()
	if extport == 0 {
		return nil, ErrNoMapping
	}

	tcp, err := ma.NewMultiaddr(fmt.Sprintf("/%s/%d", m.Protocol(), extport))
	if err != nil {
		return nil, err
	}

	maddr2 := ipmaddr.Encapsulate(tcp)

	m.cached = maddr2
	m.cacheTime = time.Now()
	return maddr2, nil
}

func (m *mapping) Close() error {
	return m.proc.Close()
}

// Mappings returns a slice of all NAT mappings
func (nat *NAT) Mappings() []Mapping {
	nat.mappingmu.Lock()
	maps2 := make([]Mapping, 0, len(nat.mappings))
	for m := range nat.mappings {
		maps2 = append(maps2, m)
	}
	nat.mappingmu.Unlock()
	return maps2
}

func (nat *NAT) addMapping(m *mapping) {
	// make mapping automatically close when nat is closed.
	nat.proc.AddChild(m.proc)

	nat.mappingmu.Lock()
	nat.mappings[m] = struct{}{}
	nat.mappingmu.Unlock()
}

func (nat *NAT) rmMapping(m *mapping) {
	nat.mappingmu.Lock()
	delete(nat.mappings, m)
	nat.mappingmu.Unlock()
}

// NewMapping attemps to construct a mapping on protocol and internal port
// It will also periodically renew the mapping until the returned Mapping
// -- or its parent NAT -- is Closed.
//
// May not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) NewMapping(maddr ma.Multiaddr) (Mapping, error) {
	if nat == nil {
		return nil, fmt.Errorf("no nat available")
	}

	network, addr, err := manet.DialArgs(maddr)
	if err != nil {
		return nil, fmt.Errorf("DialArgs failed on addr:", maddr.String())
	}

	switch network {
	case "tcp", "tcp4", "tcp6":
		network = "tcp"
	case "udp", "udp4", "udp6":
		network = "udp"
	default:
		return nil, fmt.Errorf("transport not supported by NAT: %s", network)
	}

	intports := strings.Split(addr, ":")[1]
	intport, err := strconv.Atoi(intports)
	if err != nil {
		return nil, err
	}

	m := &mapping{
		nat:     nat,
		proto:   network,
		intport: intport,
		intaddr: maddr,
	}
	m.proc = goprocess.WithTeardown(func() error {
		nat.rmMapping(m)
		return nil
	})
	nat.addMapping(m)

	m.proc.AddChild(periodic.Every(MappingDuration/3, func(worker goprocess.Process) {
		nat.establishMapping(m)
	}))

	// do it once synchronously, so first mapping is done right away, and before exiting,
	// allowing users -- in the optimistic case -- to use results right after.
	nat.establishMapping(m)
	return m, nil
}

func (nat *NAT) establishMapping(m *mapping) {
	oldport := m.ExternalPort()
	log.Debugf("Attempting port map: %s/%d", m.Protocol(), m.InternalPort())
	newport, err := nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), "http", MappingDuration)

	failure := func() {
		m.setExternalPort(0) // clear mapping
		// TODO: log.Event
		log.Debugf("failed to establish port mapping: %s", err)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingFailed(nat, m, oldport, err)
		})

		// we do not close if the mapping failed,
		// because it may work again next time.
	}

	if err != nil || newport == 0 {
		failure()
		return
	}

	m.setExternalPort(newport)
	ext, err := m.ExternalAddr()
	if err != nil {
		log.Debugf("NAT Mapping addr error: %s %s", m.InternalAddr(), err)
		failure()
		return
	}

	log.Debugf("NAT Mapping: %s --> %s", m.InternalAddr(), ext)
	if oldport != 0 && newport != oldport {
		log.Debugf("failed to renew same port mapping: ch %d -> %d", oldport, newport)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingChanged(nat, m, oldport, newport)
		})
	}

	nat.Notifier.notifyAll(func(n Notifiee) {
		n.MappingSuccess(nat, m)
	})
}

// PortMapAddrs attempts to open (and continue to keep open)
// port mappings for given addrs. This function blocks until
// all  addresses have been tried. This allows clients to
// retrieve results immediately after:
//
//   nat.PortMapAddrs(addrs)
//   mapped := nat.ExternalAddrs()
//
// Some may not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) PortMapAddrs(addrs []ma.Multiaddr) {
	// spin off addr mappings independently.
	var wg sync.WaitGroup
	for _, addr := range addrs {
		// do all of them concurrently
		wg.Add(1)
		go func() {
			defer wg.Done()
			nat.NewMapping(addr)
		}()
	}
	wg.Wait()
}

// MappedAddrs returns address mappings NAT believes have been
// successfully established. Unsuccessful mappings are nil. This is:
//
// 		map[internalAddr]externalAddr
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) MappedAddrs() map[ma.Multiaddr]ma.Multiaddr {

	mappings := nat.Mappings()
	addrmap := make(map[ma.Multiaddr]ma.Multiaddr, len(mappings))

	for _, m := range mappings {
		i := m.InternalAddr()
		e, err := m.ExternalAddr()
		if err != nil {
			addrmap[i] = nil
		} else {
			addrmap[i] = e
		}
	}
	return addrmap
}

// ExternalAddrs returns a list of addresses that NAT believes have
// been successfully established. Unsuccessful mappings are omitted,
// so nat.ExternalAddrs() may return less addresses than nat.InternalAddrs().
// To see which addresses are mapped, use nat.MappedAddrs().
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) ExternalAddrs() []ma.Multiaddr {
	mappings := nat.Mappings()
	addrs := make([]ma.Multiaddr, 0, len(mappings))
	for _, m := range mappings {
		a, err := m.ExternalAddr()
		if err != nil {
			continue // this mapping not currently successful.
		}
		addrs = append(addrs, a)
	}
	return addrs
}
<|endoftext|>