<commit_before>package main

import (
	"context"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"strings"
	"time"

	"cloud.google.com/go/logging"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/cloudfoundry"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/config"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/heartbeat"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/nozzle"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/stackdriver"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/version"
	"github.com/cloudfoundry/lager"
)

func main() {
	a := newApp()

	ctx, cancel := context.WithCancel(context.Background())

	if a.c.DebugNozzle {
		defer handleFatalError(a, cancel)

		go func() {
			a.logger.Info("pprof", lager.Data{
				"http.ListenAndServe": http.ListenAndServe("localhost:6060", nil),
			})
		}()
	}

	producer := a.newProducer()
	consumer, err := a.newConsumer(ctx)
	if err != nil {
		a.logger.Fatal("construction", err)
	}

	errs, fhErrs := consumer.Start(producer)
	defer consumer.Stop()

	go func() {
		for err := range errs {
			a.logger.Error("nozzle", err)
		}
	}()

	if fatalErr := <-fhErrs; fatalErr != nil {
		cancel()
		t := time.NewTimer(5 * time.Second)
		for {
			select {
			case <-time.Tick(100 * time.Millisecond):
				if a.bufferEmpty() {
					a.logger.Fatal("firehose", fatalErr, lager.Data{"cleanup": "The metrics buffer was successfully flushed before shutdown"})
				}
			case <-t.C:
				a.logger.Fatal("firehose", fatalErr, lager.Data{"cleanup": "The metrics buffer could not be flushed before shutdown"})
			}
		}
	}
}

func handleFatalError(a *app, cancel context.CancelFunc) {
	if e := recover(); e != nil {
		// Cancel the context
		cancel()

		stack := make([]byte, 1<<16)
		stackSize := runtime.Stack(stack, true)
		stackTrace := string(stack[:stackSize])

		payload := map[string]interface{}{
			"serviceContext": map[string]interface{}{
				"service": version.Name,
				"version": version.Release(),
			},
			"message": stackTrace,
		}

		log := &stackdriver.Log{
			Payload:  payload,
			Labels:   map[string]string{},
			Severity: logging.Error,
		}

		// Purposefully get a new log adapter here since there
		// were issues re-using the one that the nozzle uses.
		logAdapter, _ := a.newLogAdapter()
		logAdapter.PostLog(log)
		logAdapter.Flush()

		// Re-throw the error, we want to ensure it's logged directly to
		// stackdriver but we are not in a recoverable state.
		panic(e)
	}
}

func newApp() *app {
	logger := lager.NewLogger("stackdriver-nozzle")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))
	logger.Info("version", lager.Data{"name": version.Name, "release": version.Release(), "user_agent": version.UserAgent()})

	c, err := config.NewConfig()
	if err != nil {
		logger.Fatal("config", err)
	}

	logger.Info("arguments", c.ToData())

	metricClient, err := stackdriver.NewMetricClient()
	if err != nil {
		logger.Fatal("metricClient", err)
	}

	// Create a metricAdapter that will be used by the heartbeater
	// to send heartbeat metrics to Stackdriver. This metricAdapter
	// has its own heartbeater (with its own trigger) that writes to a logger.
	trigger := time.NewTicker(time.Duration(c.HeartbeatRate) * time.Second).C
	adapterHeartbeater := heartbeat.NewHeartbeater(logger, trigger)
	adapterHeartbeater.Start()
	metricAdapter, err := stackdriver.NewMetricAdapter(c.ProjectID, metricClient, adapterHeartbeater)
	if err != nil {
		logger.Error("metricAdapter", err)
	}

	// Create a heartbeater that will write heartbeat events to Stackdriver
	// logging and monitoring. It uses the metricAdapter created previously
	// to write to Stackdriver.
	metricHandler := heartbeat.NewMetricHandler(metricAdapter, logger, c.NozzleId, c.NozzleName, c.NozzleZone)
	trigger2 := time.NewTicker(time.Duration(c.HeartbeatRate) * time.Second).C
	heartbeater := heartbeat.NewLoggerMetricHeartbeater(metricHandler, logger, trigger2)

	cfConfig := &cfclient.Config{
		ApiAddress:        c.APIEndpoint,
		Username:          c.Username,
		Password:          c.Password,
		SkipSslValidation: c.SkipSSL}
	cfClient, err := cfclient.NewClient(cfConfig)
	if err != nil {
		logger.Error("cfClient", err)
	}

	var appInfoRepository cloudfoundry.AppInfoRepository
	if c.ResolveAppMetadata {
		appInfoRepository = cloudfoundry.NewAppInfoRepository(cfClient)
	} else {
		appInfoRepository = cloudfoundry.NullAppInfoRepository()
	}
	labelMaker := nozzle.NewLabelMaker(appInfoRepository)

	return &app{
		logger:      logger,
		c:           c,
		cfConfig:    cfConfig,
		cfClient:    cfClient,
		labelMaker:  labelMaker,
		heartbeater: heartbeater,
	}
}

type app struct {
	logger      lager.Logger
	c           *config.Config
	cfConfig    *cfclient.Config
	cfClient    *cfclient.Client
	labelMaker  nozzle.LabelMaker
	heartbeater heartbeat.Heartbeater
	bufferEmpty func() bool
}

func (a *app) newProducer() cloudfoundry.Firehose {
	return cloudfoundry.NewFirehose(a.cfConfig, a.cfClient, a.c.SubscriptionID)
}

func (a *app) newConsumer(ctx context.Context) (*nozzle.Nozzle, error) {
	loggingEvents := strings.Split(a.c.LoggingEvents, ",")
	metricEvents := strings.Split(a.c.MonitoringEvents, ",")

	logSink, err := nozzle.NewFilterSink(loggingEvents, a.newLogSink())
	if err != nil {
		return nil, err
	}

	metricSink, err := nozzle.NewFilterSink(metricEvents, a.newMetricSink(ctx))
	if err != nil {
		return nil, err
	}

	return &nozzle.Nozzle{
		LogSink:     logSink,
		MetricSink:  metricSink,
		Heartbeater: a.heartbeater,
	}, nil
}

func (a *app) newLogSink() nozzle.Sink {
	logAdapter, logErrs := a.newLogAdapter()
	go func() {
		err := <-logErrs
		a.logger.Error("logAdapter", err)
	}()

	return nozzle.NewLogSink(a.labelMaker, logAdapter, a.c.NewlineToken)
}

func (a *app) newLogAdapter() (stackdriver.LogAdapter, <-chan error) {
	return stackdriver.NewLogAdapter(
		a.c.ProjectID,
		a.c.BatchCount,
		time.Duration(a.c.BatchDuration)*time.Second,
		a.heartbeater,
	)
}

func (a *app) newMetricSink(ctx context.Context) nozzle.Sink {
	metricClient, err := stackdriver.NewMetricClient()
	if err != nil {
		a.logger.Fatal("metricClient", err)
	}

	metricAdapter, err := stackdriver.NewMetricAdapter(a.c.ProjectID, metricClient, a.heartbeater)
	if err != nil {
		a.logger.Error("metricAdapter", err)
	}

	metricBuffer, errs := stackdriver.NewAutoCulledMetricsBuffer(ctx, a.logger, time.Duration(a.c.MetricsBufferDuration)*time.Second, a.c.MetricsBufferSize, metricAdapter)
	a.bufferEmpty = metricBuffer.IsEmpty
	go func() {
		for err = range errs {
			a.logger.Error("metricsBuffer", err)
		}
	}()

	return nozzle.NewMetricSink(a.labelMaker, metricBuffer, nozzle.NewUnitParser())
}
<commit_msg>main: show panic error as part of log to stackdriver<commit_after>package main

import (
	"context"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"strings"
	"time"

	"cloud.google.com/go/logging"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/cloudfoundry"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/config"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/heartbeat"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/nozzle"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/stackdriver"
	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/version"
	"github.com/cloudfoundry/lager"
)

func main() {
	a := newApp()

	ctx, cancel := context.WithCancel(context.Background())

	if a.c.DebugNozzle {
		defer handleFatalError(a, cancel)

		go func() {
			a.logger.Info("pprof", lager.Data{
				"http.ListenAndServe": http.ListenAndServe("localhost:6060", nil),
			})
		}()
	}

	producer := a.newProducer()
	consumer, err := a.newConsumer(ctx)
	if err != nil {
		a.logger.Fatal("construction", err)
	}

	errs, fhErrs := consumer.Start(producer)
	defer consumer.Stop()

	go func() {
		for err := range errs {
			a.logger.Error("nozzle", err)
		}
	}()

	if fatalErr := <-fhErrs; fatalErr != nil {
		cancel()
		t := time.NewTimer(5 * time.Second)
		for {
			select {
			case <-time.Tick(100 * time.Millisecond):
				if a.bufferEmpty() {
					a.logger.Fatal("firehose", fatalErr, lager.Data{"cleanup": "The metrics buffer was successfully flushed before shutdown"})
				}
			case <-t.C:
				a.logger.Fatal("firehose", fatalErr, lager.Data{"cleanup": "The metrics buffer could not be flushed before shutdown"})
			}
		}
	}
}

func handleFatalError(a *app, cancel context.CancelFunc) {
	if e := recover(); e != nil {
		// Cancel the context
		cancel()

		stack := make([]byte, 1<<16)
		stackSize := runtime.Stack(stack, true)
		stackTrace := string(stack[:stackSize])

		payload := map[string]interface{}{
			"serviceContext": map[string]interface{}{
				"service": version.Name,
				"version": version.Release(),
			},
			"message": fmt.Sprintf("%v\n%v", e, stackTrace),
		}

		log := &stackdriver.Log{
			Payload:  payload,
			Labels:   map[string]string{},
			Severity: logging.Error,
		}

		// Purposefully get a new log adapter here since there
		// were issues re-using the one that the nozzle uses.
		logAdapter, _ := a.newLogAdapter()
		logAdapter.PostLog(log)
		logAdapter.Flush()

		// Re-throw the error, we want to ensure it's logged directly to
		// stackdriver but we are not in a recoverable state.
		panic(e)
	}
}

func newApp() *app {
	logger := lager.NewLogger("stackdriver-nozzle")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))
	logger.Info("version", lager.Data{"name": version.Name, "release": version.Release(), "user_agent": version.UserAgent()})

	c, err := config.NewConfig()
	if err != nil {
		logger.Fatal("config", err)
	}

	logger.Info("arguments", c.ToData())

	metricClient, err := stackdriver.NewMetricClient()
	if err != nil {
		logger.Fatal("metricClient", err)
	}

	// Create a metricAdapter that will be used by the heartbeater
	// to send heartbeat metrics to Stackdriver. This metricAdapter
	// has its own heartbeater (with its own trigger) that writes to a logger.
	trigger := time.NewTicker(time.Duration(c.HeartbeatRate) * time.Second).C
	adapterHeartbeater := heartbeat.NewHeartbeater(logger, trigger)
	adapterHeartbeater.Start()
	metricAdapter, err := stackdriver.NewMetricAdapter(c.ProjectID, metricClient, adapterHeartbeater)
	if err != nil {
		logger.Error("metricAdapter", err)
	}

	// Create a heartbeater that will write heartbeat events to Stackdriver
	// logging and monitoring. It uses the metricAdapter created previously
	// to write to Stackdriver.
	metricHandler := heartbeat.NewMetricHandler(metricAdapter, logger, c.NozzleId, c.NozzleName, c.NozzleZone)
	trigger2 := time.NewTicker(time.Duration(c.HeartbeatRate) * time.Second).C
	heartbeater := heartbeat.NewLoggerMetricHeartbeater(metricHandler, logger, trigger2)

	cfConfig := &cfclient.Config{
		ApiAddress:        c.APIEndpoint,
		Username:          c.Username,
		Password:          c.Password,
		SkipSslValidation: c.SkipSSL}
	cfClient, err := cfclient.NewClient(cfConfig)
	if err != nil {
		logger.Error("cfClient", err)
	}

	var appInfoRepository cloudfoundry.AppInfoRepository
	if c.ResolveAppMetadata {
		appInfoRepository = cloudfoundry.NewAppInfoRepository(cfClient)
	} else {
		appInfoRepository = cloudfoundry.NullAppInfoRepository()
	}
	labelMaker := nozzle.NewLabelMaker(appInfoRepository)

	return &app{
		logger:      logger,
		c:           c,
		cfConfig:    cfConfig,
		cfClient:    cfClient,
		labelMaker:  labelMaker,
		heartbeater: heartbeater,
	}
}

type app struct {
	logger      lager.Logger
	c           *config.Config
	cfConfig    *cfclient.Config
	cfClient    *cfclient.Client
	labelMaker  nozzle.LabelMaker
	heartbeater heartbeat.Heartbeater
	bufferEmpty func() bool
}

func (a *app) newProducer() cloudfoundry.Firehose {
	return cloudfoundry.NewFirehose(a.cfConfig, a.cfClient, a.c.SubscriptionID)
}

func (a *app) newConsumer(ctx context.Context) (*nozzle.Nozzle, error) {
	loggingEvents := strings.Split(a.c.LoggingEvents, ",")
	metricEvents := strings.Split(a.c.MonitoringEvents, ",")

	logSink, err := nozzle.NewFilterSink(loggingEvents, a.newLogSink())
	if err != nil {
		return nil, err
	}

	metricSink, err := nozzle.NewFilterSink(metricEvents, a.newMetricSink(ctx))
	if err != nil {
		return nil, err
	}

	return &nozzle.Nozzle{
		LogSink:     logSink,
		MetricSink:  metricSink,
		Heartbeater: a.heartbeater,
	}, nil
}

func (a *app) newLogSink() nozzle.Sink {
	logAdapter, logErrs := a.newLogAdapter()
	go func() {
		err := <-logErrs
		a.logger.Error("logAdapter", err)
	}()

	return nozzle.NewLogSink(a.labelMaker, logAdapter, a.c.NewlineToken)
}

func (a *app) newLogAdapter() (stackdriver.LogAdapter, <-chan error) {
	return stackdriver.NewLogAdapter(
		a.c.ProjectID,
		a.c.BatchCount,
		time.Duration(a.c.BatchDuration)*time.Second,
		a.heartbeater,
	)
}

func (a *app) newMetricSink(ctx context.Context) nozzle.Sink {
	metricClient, err := stackdriver.NewMetricClient()
	if err != nil {
		a.logger.Fatal("metricClient", err)
	}

	metricAdapter, err := stackdriver.NewMetricAdapter(a.c.ProjectID, metricClient, a.heartbeater)
	if err != nil {
		a.logger.Error("metricAdapter", err)
	}

	metricBuffer, errs := stackdriver.NewAutoCulledMetricsBuffer(ctx, a.logger, time.Duration(a.c.MetricsBufferDuration)*time.Second, a.c.MetricsBufferSize, metricAdapter)
	a.bufferEmpty = metricBuffer.IsEmpty
	go func() {
		for err = range errs {
			a.logger.Error("metricsBuffer", err)
		}
	}()

	return nozzle.NewMetricSink(a.labelMaker, metricBuffer, nozzle.NewUnitParser())
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"errors"
	"fmt"
	"net/url"

	"github.com/golang/glog"
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/pagination"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/cloudprovider"
)

type Instances struct {
	compute *gophercloud.ServiceClient
}

// Instances returns an implementation of Instances for OpenStack.
func (os *OpenStack) Instances() (cloudprovider.Instances, bool) {
	glog.V(4).Info("openstack.Instances() called")

	compute, err := os.NewComputeV2()
	if err != nil {
		return nil, false
	}

	glog.V(1).Info("Claiming to support Instances")

	return &Instances{compute}, true
}

func (i *Instances) List(name_filter string) ([]types.NodeName, error) {
	glog.V(4).Infof("openstack List(%v) called", name_filter)

	opts := servers.ListOpts{
		Name:   name_filter,
		Status: "ACTIVE",
	}
	pager := servers.List(i.compute, opts)

	ret := make([]types.NodeName, 0)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		sList, err := servers.ExtractServers(page)
		if err != nil {
			return false, err
		}
		for i := range sList {
			ret = append(ret, mapServerToNodeName(&sList[i]))
		}
		return true, nil
	})
	if err != nil {
		return nil, err
	}

	glog.V(3).Infof("Found %v instances matching %v: %v",
		len(ret), name_filter, ret)

	return ret, nil
}

// Implementation of Instances.CurrentNodeName
// Note this is *not* necessarily the same as hostname.
func (i *Instances) CurrentNodeName(hostname string) (types.NodeName, error) {
	md, err := getMetadata()
	if err != nil {
		return "", err
	}
	return types.NodeName(md.Name), nil
}

func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error {
	return errors.New("unimplemented")
}

func (i *Instances) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
	glog.V(4).Infof("NodeAddresses(%v) called", name)

	addrs, err := getAddressesByName(i.compute, name)
	if err != nil {
		return nil, err
	}

	glog.V(4).Infof("NodeAddresses(%v) => %v", name, addrs)
	return addrs, nil
}

// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
	instanceID, err := instanceIDFromProviderID(providerID)

	if err != nil {
		return []v1.NodeAddress{}, err
	}

	server, err := servers.Get(i.compute, instanceID).Extract()

	if err != nil {
		return []v1.NodeAddress{}, err
	}

	addresses, err := nodeAddresses(server)
	if err != nil {
		return []v1.NodeAddress{}, err
	}

	return addresses, nil
}

// ExternalID returns the cloud provider ID of the specified instance (deprecated).
func (i *Instances) ExternalID(name types.NodeName) (string, error) {
	srv, err := getServerByName(i.compute, name)
	if err != nil {
		if err == ErrNotFound {
			return "", cloudprovider.InstanceNotFound
		}
		return "", err
	}
	return srv.ID, nil
}

// InstanceID returns the kubelet's cloud provider ID.
func (os *OpenStack) InstanceID() (string, error) {
	return os.localInstanceID, nil
}

// InstanceID returns the cloud provider ID of the specified instance.
func (i *Instances) InstanceID(name types.NodeName) (string, error) {
	srv, err := getServerByName(i.compute, name)
	if err != nil {
		return "", err
	}
	// In the future it is possible to also return an endpoint as:
	// <endpoint>/<instanceid>
	return "/" + srv.ID, nil
}

// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) {
	instanceID, err := instanceIDFromProviderID(providerID)

	if err != nil {
		return "", err
	}

	server, err := servers.Get(i.compute, instanceID).Extract()

	if err != nil {
		return "", err
	}

	return srvInstanceType(server)
}

// InstanceType returns the type of the specified instance.
func (i *Instances) InstanceType(name types.NodeName) (string, error) {
	srv, err := getServerByName(i.compute, name)

	if err != nil {
		return "", err
	}

	return srvInstanceType(srv)
}

func srvInstanceType(srv *servers.Server) (string, error) {
	val, ok := srv.Flavor["name"]

	if !ok {
		return "", fmt.Errorf("flavor name not present in server info")
	}

	flavor, ok := val.(string)

	if !ok {
		return "", fmt.Errorf("flavor name is not a string")
	}

	return flavor, nil
}

func instanceIDFromProviderID(providerID string) (instanceID string, err error) {
	parsedID, err := url.Parse(providerID)
	if err != nil {
		return "", err
	}
	if parsedID.Scheme != ProviderName {
		return "", fmt.Errorf("unrecognized provider %q", parsedID.Scheme)
	}

	return parsedID.Host, nil
}
<commit_msg>Tolerate Flavor information for computing instance type<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"errors"
	"fmt"
	"net/url"

	"github.com/golang/glog"
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/pagination"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/cloudprovider"
)

type Instances struct {
	compute *gophercloud.ServiceClient
}

// Instances returns an implementation of Instances for OpenStack.
func (os *OpenStack) Instances() (cloudprovider.Instances, bool) {
	glog.V(4).Info("openstack.Instances() called")

	compute, err := os.NewComputeV2()
	if err != nil {
		return nil, false
	}

	glog.V(1).Info("Claiming to support Instances")

	return &Instances{compute}, true
}

func (i *Instances) List(name_filter string) ([]types.NodeName, error) {
	glog.V(4).Infof("openstack List(%v) called", name_filter)

	opts := servers.ListOpts{
		Name:   name_filter,
		Status: "ACTIVE",
	}
	pager := servers.List(i.compute, opts)

	ret := make([]types.NodeName, 0)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		sList, err := servers.ExtractServers(page)
		if err != nil {
			return false, err
		}
		for i := range sList {
			ret = append(ret, mapServerToNodeName(&sList[i]))
		}
		return true, nil
	})
	if err != nil {
		return nil, err
	}

	glog.V(3).Infof("Found %v instances matching %v: %v",
		len(ret), name_filter, ret)

	return ret, nil
}

// Implementation of Instances.CurrentNodeName
// Note this is *not* necessarily the same as hostname.
func (i *Instances) CurrentNodeName(hostname string) (types.NodeName, error) {
	md, err := getMetadata()
	if err != nil {
		return "", err
	}
	return types.NodeName(md.Name), nil
}

func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error {
	return errors.New("unimplemented")
}

func (i *Instances) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
	glog.V(4).Infof("NodeAddresses(%v) called", name)

	addrs, err := getAddressesByName(i.compute, name)
	if err != nil {
		return nil, err
	}

	glog.V(4).Infof("NodeAddresses(%v) => %v", name, addrs)
	return addrs, nil
}

// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
	instanceID, err := instanceIDFromProviderID(providerID)

	if err != nil {
		return []v1.NodeAddress{}, err
	}

	server, err := servers.Get(i.compute, instanceID).Extract()

	if err != nil {
		return []v1.NodeAddress{}, err
	}

	addresses, err := nodeAddresses(server)
	if err != nil {
		return []v1.NodeAddress{}, err
	}

	return addresses, nil
}

// ExternalID returns the cloud provider ID of the specified instance (deprecated).
func (i *Instances) ExternalID(name types.NodeName) (string, error) {
	srv, err := getServerByName(i.compute, name)
	if err != nil {
		if err == ErrNotFound {
			return "", cloudprovider.InstanceNotFound
		}
		return "", err
	}
	return srv.ID, nil
}

// InstanceID returns the kubelet's cloud provider ID.
func (os *OpenStack) InstanceID() (string, error) {
	return os.localInstanceID, nil
}

// InstanceID returns the cloud provider ID of the specified instance.
func (i *Instances) InstanceID(name types.NodeName) (string, error) {
	srv, err := getServerByName(i.compute, name)
	if err != nil {
		return "", err
	}
	// In the future it is possible to also return an endpoint as:
	// <endpoint>/<instanceid>
	return "/" + srv.ID, nil
}

// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) {
	instanceID, err := instanceIDFromProviderID(providerID)

	if err != nil {
		return "", err
	}

	server, err := servers.Get(i.compute, instanceID).Extract()

	if err != nil {
		return "", err
	}

	return srvInstanceType(server)
}

// InstanceType returns the type of the specified instance.
func (i *Instances) InstanceType(name types.NodeName) (string, error) {
	srv, err := getServerByName(i.compute, name)

	if err != nil {
		return "", err
	}

	return srvInstanceType(srv)
}

func srvInstanceType(srv *servers.Server) (string, error) {
	keys := []string{"name", "id", "original_name"}
	for _, key := range keys {
		val, found := srv.Flavor[key]
		if found {
			flavor, ok := val.(string)
			if ok {
				return flavor, nil
			}
		}
	}
	return "", fmt.Errorf("flavor name/id not found")
}

func instanceIDFromProviderID(providerID string) (instanceID string, err error) {
	parsedID, err := url.Parse(providerID)
	if err != nil {
		return "", err
	}
	if parsedID.Scheme != ProviderName {
		return "", fmt.Errorf("unrecognized provider %q", parsedID.Scheme)
	}

	return parsedID.Host, nil
}
<|endoftext|>


<commit_before>package sql

import (
	"database/sql"
	"gnd.la/orm/driver"
	"reflect"
)

type Iter struct {
	model  driver.Model
	driver *Driver
	rows   *sql.Rows
	err    error
}

func (i *Iter) Next(out ...interface{}) bool {
	if i.err == nil && i.rows != nil && i.rows.Next() {
		var vals []reflect.Value
		var fields []*driver.Fields
		var values []interface{}
		var scanners [][]scanner
		model := i.model
		for _, v := range out {
			val, vfields, vvalues, vscanners, err := i.driver.outValues(model, v)
			if err != nil {
				i.err = err
				return false
			}
			vals = append(vals, val)
			fields = append(fields, vfields)
			values = append(values, vvalues...)
			scanners = append(scanners, vscanners)
			if j := model.Join(); j != nil {
				model = j.Model()
			}
		}
		i.err = i.rows.Scan(values...)
		for ii, f := range fields {
			if f == nil {
			    // This model was skipped
			    continue
			}
			vscanners := scanners[ii]
			nilVal := true
			for _, p := range f.Pointers {
				isNil := true
				for jj, v := range f.Indexes {
					if f.IsSubfield(v, p) && !vscanners[jj].IsNil() {
						isNil = false
						nilVal = false
						break
					}
				}
				if isNil {
					fval := i.driver.fieldByIndex(vals[ii], p, false)
					fval.Set(reflect.Zero(fval.Type()))
				}
			}
			if nilVal {
				for _, v := range vscanners {
					if !v.IsNil() {
						nilVal = false
						break
					}
				}
			}
			if nilVal {
				val := reflect.ValueOf(out[ii]).Elem()
				val.Set(reflect.Zero(val.Type()))
			}
		}
		for _, s := range scanners {
			for _, v := range s {
				v.Put()
			}
		}
		return i.err == nil
	}
	return false
}

func (i *Iter) Err() error {
	return i.err
}

func (i *Iter) Close() error {
	if i.rows != nil {
		return i.rows.Close()
	}
	return nil
}

func NewIter(m driver.Model, d driver.Driver, r *sql.Rows, err error) driver.Iter {
	// TODO: Check for errors here?
	drv, _ := d.(*Driver)
	return &Iter{
		model:  m,
		driver: drv,
		rows:   r,
		err:    err,
	}
}
<commit_msg>Skip over skipped models when iterating results<commit_after>package sql

import (
	"database/sql"
	"gnd.la/orm/driver"
	"reflect"
)

type Iter struct {
	model  driver.Model
	driver *Driver
	rows   *sql.Rows
	err    error
}

func (i *Iter) Next(out ...interface{}) bool {
	if i.err == nil && i.rows != nil && i.rows.Next() {
		var vals []reflect.Value
		var fields []*driver.Fields
		var values []interface{}
		var scanners [][]scanner
		model := i.model
		for model.Skip() {
		    model = model.Join().Model()
		}
		for _, v := range out {
			val, vfields, vvalues, vscanners, err := i.driver.outValues(model, v)
			if err != nil {
				i.err = err
				return false
			}
			vals = append(vals, val)
			fields = append(fields, vfields)
			values = append(values, vvalues...)
			scanners = append(scanners, vscanners)
			if j := model.Join(); j != nil {
				model = j.Model()
			}
		}
		i.err = i.rows.Scan(values...)
		for ii, f := range fields {
			if f == nil {
			    // This model was skipped
			    continue
			}
			vscanners := scanners[ii]
			nilVal := true
			for _, p := range f.Pointers {
				isNil := true
				for jj, v := range f.Indexes {
					if f.IsSubfield(v, p) && !vscanners[jj].IsNil() {
						isNil = false
						nilVal = false
						break
					}
				}
				if isNil {
					fval := i.driver.fieldByIndex(vals[ii], p, false)
					fval.Set(reflect.Zero(fval.Type()))
				}
			}
			if nilVal {
				for _, v := range vscanners {
					if !v.IsNil() {
						nilVal = false
						break
					}
				}
			}
			if nilVal {
				val := reflect.ValueOf(out[ii]).Elem()
				val.Set(reflect.Zero(val.Type()))
			}
		}
		for _, s := range scanners {
			for _, v := range s {
				v.Put()
			}
		}
		return i.err == nil
	}
	return false
}

func (i *Iter) Err() error {
	return i.err
}

func (i *Iter) Close() error {
	if i.rows != nil {
		return i.rows.Close()
	}
	return nil
}

func NewIter(m driver.Model, d driver.Driver, r *sql.Rows, err error) driver.Iter {
	// TODO: Check for errors here?
	drv, _ := d.(*Driver)
	return &Iter{
		model:  m,
		driver: drv,
		rows:   r,
		err:    err,
	}
}
<|endoftext|>


<commit_before>package osl

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/reexec"
	"github.com/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

const prefix = "/var/run/docker/netns"

var (
	once             sync.Once
	garbagePathMap   = make(map[string]bool)
	gpmLock          sync.Mutex
	gpmWg            sync.WaitGroup
	gpmCleanupPeriod = 60 * time.Second
	gpmChan          = make(chan chan struct{})
	nsOnce           sync.Once
	initNs           netns.NsHandle
)

// The networkNamespace type is the linux implementation of the Sandbox
// interface. It represents a linux network namespace, and moves an interface
// into it when called on method AddInterface or sets the gateway etc.
type networkNamespace struct {
	path         string
	iFaces       []*nwIface
	gw           net.IP
	gwv6         net.IP
	staticRoutes []*types.StaticRoute
	neighbors    []*neigh
	nextIfIndex  int
	sync.Mutex
}

func init() {
	reexec.Register("netns-create", reexecCreateNamespace)
}

func createBasePath() {
	err := os.MkdirAll(prefix, 0644)
	if err != nil {
		panic("Could not create net namespace path directory")
	}

	// Start the garbage collection go routine
	go removeUnusedPaths()
}

func removeUnusedPaths() {
	gpmLock.Lock()
	period := gpmCleanupPeriod
	gpmLock.Unlock()

	ticker := time.NewTicker(period)
	for {
		var (
			gc   chan struct{}
			gcOk bool
		)

		select {
		case <-ticker.C:
		case gc, gcOk = <-gpmChan:
		}

		gpmLock.Lock()
		pathList := make([]string, 0, len(garbagePathMap))
		for path := range garbagePathMap {
			pathList = append(pathList, path)
		}
		garbagePathMap = make(map[string]bool)
		gpmWg.Add(1)
		gpmLock.Unlock()

		for _, path := range pathList {
			os.Remove(path)
		}

		gpmWg.Done()
		if gcOk {
			close(gc)
		}
	}
}

func addToGarbagePaths(path string) {
	gpmLock.Lock()
	garbagePathMap[path] = true
	gpmLock.Unlock()
}

func removeFromGarbagePaths(path string) {
	gpmLock.Lock()
	delete(garbagePathMap, path)
	gpmLock.Unlock()
}

// GC triggers garbage collection of namespace path right away
// and waits for it.
func GC() {
	gpmLock.Lock()
	if len(garbagePathMap) == 0 {
		// No need for GC if map is empty
		gpmLock.Unlock()
		return
	}
	gpmLock.Unlock()

	// if content exists in the garbage paths
	// we can trigger GC to run, providing a
	// channel to be notified on completion
	waitGC := make(chan struct{})
	gpmChan <- waitGC
	// wait for GC completion
	<-waitGC
}

// GenerateKey generates a sandbox key based on the passed
// container id.
func GenerateKey(containerID string) string {
	maxLen := 12
	if len(containerID) < maxLen {
		maxLen = len(containerID)
	}

	return prefix + "/" + containerID[:maxLen]
}

// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
	err := createNetworkNamespace(key, osCreate)
	if err != nil {
		return nil, err
	}

	return &networkNamespace{path: key}, nil
}

func (n *networkNamespace) InterfaceOptions() IfaceOptionSetter {
	return n
}

func (n *networkNamespace) NeighborOptions() NeighborOptionSetter {
	return n
}

func reexecCreateNamespace() {
	if len(os.Args) < 2 {
		log.Fatal("no namespace path provided")
	}

	if err := syscall.Mount("/proc/self/ns/net", os.Args[1], "bind", syscall.MS_BIND, ""); err != nil {
		log.Fatal(err)
	}

	if err := loopbackUp(); err != nil {
		log.Fatal(err)
	}
}

func createNetworkNamespace(path string, osCreate bool) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	origns, err := netns.Get()
	if err != nil {
		return err
	}
	defer origns.Close()

	if err := createNamespaceFile(path); err != nil {
		return err
	}

	cmd := &exec.Cmd{
		Path:   reexec.Self(),
		Args:   append([]string{"netns-create"}, path),
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	if osCreate {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
	}
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("namespace creation reexec command failed: %v", err)
	}

	return nil
}

func unmountNamespaceFile(path string) {
	if _, err := os.Stat(path); err == nil {
		syscall.Unmount(path, syscall.MNT_DETACH)
	}
}

func createNamespaceFile(path string) (err error) {
	var f *os.File

	once.Do(createBasePath)
	// Remove it from garbage collection list if present
	removeFromGarbagePaths(path)

	// If the path is there unmount it first
	unmountNamespaceFile(path)

	// wait for garbage collection to complete if it is in progress
	// before trying to create the file.
	gpmWg.Wait()

	if f, err = os.Create(path); err == nil {
		f.Close()
	}

	return err
}

func loopbackUp() error {
	iface, err := netlink.LinkByName("lo")
	if err != nil {
		return err
	}
	return netlink.LinkSetUp(iface)
}

func (n *networkNamespace) InvokeFunc(f func()) error {
	return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error {
		f()
		return nil
	})
}

func getLink() string {
	l, err := os.Readlink(fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid()))
	if err != nil {
		return fmt.Sprintf("(nil: %v)", err)
	}

	return l
}

func nsInit() {
	var err error

	if initNs, err = netns.Get(); err != nil {
		log.Errorf("could not get initial namespace: %v", err)
	}
}

// InitOSContext initializes OS context while configuring network resources
func InitOSContext() func() {
	runtime.LockOSThread()
	nsOnce.Do(nsInit)
	if err := netns.Set(initNs); err != nil {
		log.Errorf("failed to set to initial namespace, link %s, initns fd %d: %v",
			getLink(), initNs, err)
	}

	return runtime.UnlockOSThread
}

func nsInvoke(path string, prefunc func(nsFD int) error, postfunc func(callerFD int) error) error {
	defer InitOSContext()()

	f, err := os.OpenFile(path, os.O_RDONLY, 0)
	if err != nil {
		return fmt.Errorf("failed get network namespace %q: %v", path, err)
	}
	defer f.Close()

	nsFD := f.Fd()

	// Invoked before the namespace switch happens but after the namespace file
	// handle is obtained.
	if err := prefunc(int(nsFD)); err != nil {
		return fmt.Errorf("failed in prefunc: %v", err)
	}

	if err = netns.Set(netns.NsHandle(nsFD)); err != nil {
		return err
	}
	defer netns.Set(initNs)

	// Invoked after the namespace switch.
	return postfunc(int(initNs))
}

func (n *networkNamespace) nsPath() string {
	n.Lock()
	defer n.Unlock()

	return n.path
}

func (n *networkNamespace) Info() Info {
	return n
}

func (n *networkNamespace) Key() string {
	return n.path
}

func (n *networkNamespace) Destroy() error {
	// Assuming no running process is executing in this network namespace,
	// unmounting is sufficient to destroy it.
	if err := syscall.Unmount(n.path, syscall.MNT_DETACH); err != nil {
		return err
	}

	// Stash it into the garbage collection list
	addToGarbagePaths(n.path)
	return nil
}
<commit_msg>Do not print the word `nil` in a log.<commit_after>package osl

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/reexec"
	"github.com/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

const prefix = "/var/run/docker/netns"

var (
	once             sync.Once
	garbagePathMap   = make(map[string]bool)
	gpmLock          sync.Mutex
	gpmWg            sync.WaitGroup
	gpmCleanupPeriod = 60 * time.Second
	gpmChan          = make(chan chan struct{})
	nsOnce           sync.Once
	initNs           netns.NsHandle
)

// The networkNamespace type is the linux implementation of the Sandbox
// interface. It represents a linux network namespace, and moves an interface
// into it when called on method AddInterface or sets the gateway etc.
type networkNamespace struct {
	path         string
	iFaces       []*nwIface
	gw           net.IP
	gwv6         net.IP
	staticRoutes []*types.StaticRoute
	neighbors    []*neigh
	nextIfIndex  int
	sync.Mutex
}

func init() {
	reexec.Register("netns-create", reexecCreateNamespace)
}

func createBasePath() {
	err := os.MkdirAll(prefix, 0644)
	if err != nil {
		panic("Could not create net namespace path directory")
	}

	// Start the garbage collection go routine
	go removeUnusedPaths()
}

func removeUnusedPaths() {
	gpmLock.Lock()
	period := gpmCleanupPeriod
	gpmLock.Unlock()

	ticker := time.NewTicker(period)
	for {
		var (
			gc   chan struct{}
			gcOk bool
		)

		select {
		case <-ticker.C:
		case gc, gcOk = <-gpmChan:
		}

		gpmLock.Lock()
		pathList := make([]string, 0, len(garbagePathMap))
		for path := range garbagePathMap {
			pathList = append(pathList, path)
		}
		garbagePathMap = make(map[string]bool)
		gpmWg.Add(1)
		gpmLock.Unlock()

		for _, path := range pathList {
			os.Remove(path)
		}

		gpmWg.Done()
		if gcOk {
			close(gc)
		}
	}
}

func addToGarbagePaths(path string) {
	gpmLock.Lock()
	garbagePathMap[path] = true
	gpmLock.Unlock()
}

func removeFromGarbagePaths(path string) {
	gpmLock.Lock()
	delete(garbagePathMap, path)
	gpmLock.Unlock()
}

// GC triggers garbage collection of namespace path right away
// and waits for it.
func GC() {
	gpmLock.Lock()
	if len(garbagePathMap) == 0 {
		// No need for GC if map is empty
		gpmLock.Unlock()
		return
	}
	gpmLock.Unlock()

	// if content exists in the garbage paths
	// we can trigger GC to run, providing a
	// channel to be notified on completion
	waitGC := make(chan struct{})
	gpmChan <- waitGC
	// wait for GC completion
	<-waitGC
}

// GenerateKey generates a sandbox key based on the passed
// container id.
func GenerateKey(containerID string) string {
	maxLen := 12
	if len(containerID) < maxLen {
		maxLen = len(containerID)
	}

	return prefix + "/" + containerID[:maxLen]
}

// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
	err := createNetworkNamespace(key, osCreate)
	if err != nil {
		return nil, err
	}

	return &networkNamespace{path: key}, nil
}

func (n *networkNamespace) InterfaceOptions() IfaceOptionSetter {
	return n
}

func (n *networkNamespace) NeighborOptions() NeighborOptionSetter {
	return n
}

func reexecCreateNamespace() {
	if len(os.Args) < 2 {
		log.Fatal("no namespace path provided")
	}

	if err := syscall.Mount("/proc/self/ns/net", os.Args[1], "bind", syscall.MS_BIND, ""); err != nil {
		log.Fatal(err)
	}

	if err := loopbackUp(); err != nil {
		log.Fatal(err)
	}
}

func createNetworkNamespace(path string, osCreate bool) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	origns, err := netns.Get()
	if err != nil {
		return err
	}
	defer origns.Close()

	if err := createNamespaceFile(path); err != nil {
		return err
	}

	cmd := &exec.Cmd{
		Path:   reexec.Self(),
		Args:   append([]string{"netns-create"}, path),
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	if osCreate {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
	}
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("namespace creation reexec command failed: %v", err)
	}

	return nil
}

func unmountNamespaceFile(path string) {
	if _, err := os.Stat(path); err == nil {
		syscall.Unmount(path, syscall.MNT_DETACH)
	}
}

func createNamespaceFile(path string) (err error) {
	var f *os.File

	once.Do(createBasePath)
	// Remove it from garbage collection list if present
	removeFromGarbagePaths(path)

	// If the path is there unmount it first
	unmountNamespaceFile(path)

	// wait for garbage collection to complete if it is in progress
	// before trying to create the file.
	gpmWg.Wait()

	if f, err = os.Create(path); err == nil {
		f.Close()
	}

	return err
}

func loopbackUp() error {
	iface, err := netlink.LinkByName("lo")
	if err != nil {
		return err
	}
	return netlink.LinkSetUp(iface)
}

func (n *networkNamespace) InvokeFunc(f func()) error {
	return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error {
		f()
		return nil
	})
}

func getLink() (string, error) {
	return os.Readlink(fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid()))
}

func nsInit() {
	var err error

	if initNs, err = netns.Get(); err != nil {
		log.Errorf("could not get initial namespace: %v", err)
	}
}

// InitOSContext initializes OS context while configuring network resources
func InitOSContext() func() {
	runtime.LockOSThread()
	nsOnce.Do(nsInit)
	if err := netns.Set(initNs); err != nil {
		linkInfo, linkErr := getLink()
		if linkErr != nil {
			linkInfo = linkErr.Error()
		}

		log.Errorf("failed to set to initial namespace, %v, initns fd %d: %v",
			linkInfo, initNs, err)
	}

	return runtime.UnlockOSThread
}

func nsInvoke(path string, prefunc func(nsFD int) error, postfunc func(callerFD int) error) error {
	defer InitOSContext()()

	f, err := os.OpenFile(path, os.O_RDONLY, 0)
	if err != nil {
		return fmt.Errorf("failed get network namespace %q: %v", path, err)
	}
	defer f.Close()

	nsFD := f.Fd()

	// Invoked before the namespace switch happens but after the namespace file
	// handle is obtained.
	if err := prefunc(int(nsFD)); err != nil {
		return fmt.Errorf("failed in prefunc: %v", err)
	}

	if err = netns.Set(netns.NsHandle(nsFD)); err != nil {
		return err
	}
	defer netns.Set(initNs)

	// Invoked after the namespace switch.
	return postfunc(int(initNs))
}

func (n *networkNamespace) nsPath() string {
	n.Lock()
	defer n.Unlock()

	return n.path
}

func (n *networkNamespace) Info() Info {
	return n
}

func (n *networkNamespace) Key() string {
	return n.path
}

func (n *networkNamespace) Destroy() error {
	// Assuming no running process is executing in this network namespace,
	// unmounting is sufficient to destroy it.
	if err := syscall.Unmount(n.path, syscall.MNT_DETACH); err != nil {
		return err
	}

	// Stash it into the garbage collection list
	addToGarbagePaths(n.path)
	return nil
}
<|endoftext|>


<commit_before>package vm

import (
	"crypto/md5"
	"errors"
	"io/ioutil"
	"log"
	"strings"
	"sync"

	fsnotify "gopkg.in/fsnotify.v0"

	"github.com/dop251/goja"
	"github.com/dop251/goja_nodejs/console"
	"github.com/dop251/goja_nodejs/require"
	"github.com/orktes/huealexa/hueserver"
)

type VM struct {
	*goja.Runtime
	sync.Mutex
	server  *hueserver.Server
	watcher *fsnotify.Watcher
	dataDir string
	md5     [md5.Size]byte
}

func (vm *VM) RunString(str string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunString(str)
}

func (vm *VM) RunScript(name, value string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunScript(name, value)
}

func (vm *VM) srcLoader(pathname string) ([]byte, error) {
	if !strings.HasSuffix(pathname, ".js") {
		pathname += ".js"
	}

	asset, err := Asset(pathname)
	if err == nil {
		return asset, nil
	}

	return nil, errors.New("Package " + pathname + " not found")
}

func (vm *VM) register() {
	registry := require.NewRegistryWithLoader(vm.srcLoader)
	registry.Enable(vm.Runtime)
	console.Enable(vm.Runtime)
	vm.initServer()
	vm.initAlexa()
	vm.initFS()
	vm.initPath()
	vm.initAsync()
	vm.initHomeKit()
	vm.initTimers()
	vm.initUUID()
	vm.initProcess()
	vm.initSSDP()
	vm.initZWay()
	vm.initWebSocket()
}

func (vm *VM) startWatch(path string) error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	vm.watcher = watcher

	// Process events
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if ev.IsModify() {
					log.Printf("%s changed. Reinitializing VM if needed.", path)
					vm.initWithPath(path)
				}
			case err := <-watcher.Error:
				log.Println("error:", err)
			}
		}
	}()

	return watcher.Watch(path)
}

func (vm *VM) init(path, value string) (err error) {
	md5 := md5.Sum([]byte(value))

	if vm.md5 == md5 {
		log.Print("Content didn't change. Doing nothing.\n")
		return
	}

	vm.md5 = md5

	if vm.Runtime != nil {
		vm.Runtime.Interrupt(errors.New("Interreupted due to update"))
	}

	log.Printf("Initializing VM with %s\n", path)
	vm.Runtime = goja.New()
	// Set env
	vm.Set("env", map[string]interface{}{
		"data_dir":      vm.dataDir,
		"huealexa_uuid": vm.server.UUID,
	})

	vm.register()

	_, err = vm.RunScript(path, value)

	log.Printf("Done initializing %s\n", path)

	return err
}

func (vm *VM) initWithPath(path string) (err error) {
	script, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	return vm.init(path, string(script))
}

func (vm *VM) Close() {
	vm.watcher.Close()
}

func NewVM(path string, dataDir string, server *hueserver.Server) (*VM, error) {
	vm := &VM{dataDir: dataDir, server: server}
	vm.startWatch(path)
	err := vm.initWithPath(path)
	return vm, err
}
<commit_msg>Support requiring local files<commit_after>package vm

import (
	"crypto/md5"
	"errors"
	"io/ioutil"
	"log"
	"path"
	"strings"
	"sync"

	fsnotify "gopkg.in/fsnotify.v0"

	"github.com/dop251/goja"
	"github.com/dop251/goja_nodejs/console"
	"github.com/dop251/goja_nodejs/require"
	"github.com/orktes/huealexa/hueserver"
)

type VM struct {
	*goja.Runtime
	sync.Mutex
	server  *hueserver.Server
	watcher *fsnotify.Watcher
	dataDir string
	md5Map  map[string][md5.Size]byte
	path    string
}

func (vm *VM) RunString(str string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunString(str)
}

func (vm *VM) RunScript(name, value string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunScript(name, value)
}

func (vm *VM) srcLoader(pathname string) ([]byte, error) {
	if !strings.HasSuffix(pathname, ".js") {
		pathname += ".js"
	}

	asset, err := Asset(pathname)
	if err == nil {
		return asset, nil
	}

	pathToFile := path.Join(path.Dir(vm.path), pathname)
	data, err := ioutil.ReadFile(pathToFile)
	if err != nil {
		return nil, err
	}

	md5 := md5.Sum(data)
	vm.md5Map[pathToFile] = md5

	return data, vm.watcher.Watch(pathToFile)
}

func (vm *VM) register() {
	registry := require.NewRegistryWithLoader(vm.srcLoader)
	registry.Enable(vm.Runtime)
	console.Enable(vm.Runtime)
	vm.initServer()
	vm.initAlexa()
	vm.initFS()
	vm.initPath()
	vm.initAsync()
	vm.initHomeKit()
	vm.initTimers()
	vm.initUUID()
	vm.initProcess()
	vm.initSSDP()
	vm.initZWay()
	vm.initWebSocket()
}

func (vm *VM) startWatch() error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	vm.watcher = watcher

	// Process events
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if ev.IsModify() {
					log.Printf("%s changed. Reinitializing VM if needed.", ev.Name)

					data, err := ioutil.ReadFile(ev.Name)
					if err != nil {
						log.Printf("Unable to read file %s: %s", ev.Name, err.Error())
						continue
					}

					md5 := md5.Sum(data)

					if vm.md5Map[ev.Name] == md5 {
						log.Print("Content didn't change. Doing nothing.\n")
						continue
					}

					vm.md5Map[ev.Name] = md5
					vm.init()
				}
			case err := <-watcher.Error:
				log.Println("error:", err)
			}
		}
	}()

	return watcher.Watch(vm.path)
}

func (vm *VM) init() (err error) {
	value, err := ioutil.ReadFile(vm.path)
	if err != nil {
		return
	}

	md5 := md5.Sum(value)
	vm.md5Map[vm.path] = md5

	if vm.Runtime != nil {
		vm.Runtime.Interrupt(errors.New("Interreupted due to update"))
	}

	log.Printf("Initializing VM with %s\n", vm.path)
	vm.Runtime = goja.New()
	// Set env
	vm.Set("env", map[string]interface{}{
		"data_dir":      vm.dataDir,
		"huealexa_uuid": vm.server.UUID,
	})

	vm.register()

	_, err = vm.RunScript(vm.path, string(value))

	log.Printf("Done initializing %s\n", vm.path)

	return err
}

func (vm *VM) Close() {
	vm.watcher.Close()
}

func NewVM(path string, dataDir string, server *hueserver.Server) (*VM, error) {
	vm := &VM{
		dataDir: dataDir,
		server:  server,
		path:    path,
		md5Map:  map[string][md5.Size]byte{},
	}
	vm.startWatch()
	err := vm.init()
	return vm, err
}
<|endoftext|>


<commit_before>/* 	vote.go
_________________________________
Parses votes and executes them for Kylixor Discord Bot
Andrew Langhill
kylixor.com
*/

package main

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/bwmarrin/discordgo"
)

// "0⃣"
var numBlocks = []string{"1⃣", "2⃣", "3⃣", "4⃣", "5⃣", "6⃣", "7⃣", "8⃣", "9⃣"}

// StartVote - begin a vote with variable vote options
func StartVote(s *discordgo.Session, m *discordgo.MessageCreate, data string, quote bool) {
	// Parse the incoming command into an array of options
	array := strings.SplitAfter(data, "|")
	options := len(array)

	if options > 9 {
		s.ChannelMessageSend(m.ChannelID, "9 options max, since Discord only has 9 number emotes")
		return
	}

	s.ChannelMessageSend(m.ChannelID, "```\nStarting vote...react now!\n```")

	// Send vote text
	voteMsg := VotePrint(s, m, data, quote)

	// Insert the vote into the vote table
	k.kdb.CreateVote(voteMsg.ID, m.GuildID, m.Author.ID, options, quote, data)

	// Upsert the vote in the watch table
	k.kdb.CreateWatch(voteMsg.ID, "vote")
}

//ReactionAdd - add a reaction to the passed-in message
func ReactionAdd(s *discordgo.Session, m *discordgo.Message, reaction string) {

	switch reaction {
	case "UPVOTE":
		err := s.MessageReactionAdd(m.ChannelID, m.ID, "⬆")
		if err != nil {
			s.ChannelMessageSend(m.ChannelID, "Unable to use upvote emote")
		}
		break

	case "DOWNVOTE":
		err := s.MessageReactionAdd(m.ChannelID, m.ID, "⬇")
		if err != nil {
			s.ChannelMessageSend(m.ChannelID, "Unable to use downvote emote")
		}
		break
	default:
		s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Unable to post emote: %s", reaction))
		break
	}
}

// HandleVote - check if a vote is valid and process accordingly
func (vote *Vote) HandleVote(s *discordgo.Session, r *discordgo.MessageReactionAdd) {
	msg, err := s.ChannelMessage(r.ChannelID, r.MessageID)
	if err != nil {
		panic(err)
	}

	logInfo := fmt.Sprintf("[Reaction] \"%s\" by [%s] found on message %s", r.Emoji.Name, r.UserID, r.MessageID)
	k.Log("VOTE", logInfo)

	for i, react := range msg.Reactions {
		if react.Me && react.Count > k.botConfig.MinVotes {
			vote.Result = i + 1
			vote.UpdateVote()
		}

		if react.Me && r.UserID == vote.SubmitterID && vote.Quote && react.Emoji.Name == "⬇" {
			vote.EndVote()
		}
	}

	if vote.Quote && vote.Result != -1 {
		if vote.Result == 1 {
			// Print finalized quote with identifier
			s.ChannelMessageSend(r.ChannelID, "Vote succeeded, yay!")
			k.kdb.DeleteWatch(r.MessageID)
			quoteAdded := k.kdb.CreateQuote(s, vote.GuildID, vote.VoteText)
			quoteAdded.RequestIdentifier(s, r)

		} else {
			s.ChannelMessageSend(r.ChannelID, "Vote failed, yikes!")
			k.kdb.DeleteWatch(r.MessageID)
		}
		return
	}

	k.Log("TEST", vote.VoteText)

	if strings.Contains(vote.VoteText, "👢") {
		guild, err := s.Guild(r.GuildID)
		if err != nil {
			panic(err)
		}

		// Pull userID out of the  voteText
		magic := regexp.MustCompile(`\((.*?)\)`)
		result := magic.FindStringSubmatch(vote.VoteText)
		userID := result[1]

		err = s.GuildMemberMove(r.GuildID, userID, &guild.AfkChannelID)
		if err != nil {
			s.ChannelMessageSend(r.ChannelID, "ERR: Unable to 'kick' member: "+err.Error())
		} else {
			s.ChannelMessageSend(r.ChannelID, "See ya!")
		}
		vote.EndVote()
		k.kdb.DeleteWatch(r.MessageID)
		return
	}

	if strings.Contains(vote.VoteText, "✨") {
		guild := k.kdb.ReadGuild(s, r.GuildID)

		err := s.GuildMemberRoleAdd(r.GuildID, vote.SubmitterID, guild.MemberRole)
		if err != nil {
			s.ChannelMessageSend(r.ChannelID, "ERR: Unable to 'add' member: "+err.Error())
		} else {
			s.ChannelMessageSend(r.ChannelID, "Added member successfully!")
		}
		vote.EndVote()
		k.kdb.DeleteWatch(r.MessageID)
		return
	}

	if vote.Result >= 0 {
		optionArray := strings.SplitAfter(vote.VoteText, "|")
		option := strings.TrimSpace(strings.TrimRight(optionArray[vote.Result-1], "|"))

		s.ChannelMessageSend(r.ChannelID, fmt.Sprintf("```\nOption %d, \"%s\", wins the vote!\n```", vote.Result, option))
		vote.EndVote()
		k.kdb.DeleteWatch(r.MessageID)
	}
}

// VotePrint - print out a vote and add reactions
func VotePrint(s *discordgo.Session, m *discordgo.MessageCreate, voteText string, quote bool) (message *discordgo.Message) {
	var err error
	array := strings.SplitAfter(voteText, "|")

	if quote {
		voteMsg := "```\n" + voteText + "\n```"
		message, err = s.ChannelMessageSend(m.ChannelID, voteMsg)
		if err != nil {
			panic(err)
		}
	} else {

		voteMsg := "```\n"
		for i, option := range array {
			cleanOption := strings.TrimSpace(strings.TrimRight(option, "|"))
			voteIndex := ""
			if len(array) > 1 {
				voteIndex = fmt.Sprintf("%d. ", i+1)
			}
			voteMsg += fmt.Sprintf("%s%s\n", voteIndex, cleanOption)
		}
		voteMsg += "```"

		message, err = s.ChannelMessageSend(m.ChannelID, voteMsg)
		if err != nil {
			panic(err)
		}
	}

	// Add reactions (up/down for single or numbers per option)
	if len(array) == 1 {
		err := s.MessageReactionAdd(m.ChannelID, message.ID, "⬆")
		if err != nil {
			panic(err)
		}
		err = s.MessageReactionAdd(m.ChannelID, message.ID, "⬇")
		if err != nil {
			panic(err)
		}
		return message
	}

	for i := range array {
		err := s.MessageReactionAdd(m.ChannelID, message.ID, numBlocks[i])
		if err != nil {
			panic(err)
		}
	}

	return message
}
<commit_msg>Made the vote being yes or no actually matter<commit_after>/* 	vote.go
_________________________________
Parses votes and executes them for Kylixor Discord Bot
Andrew Langhill
kylixor.com
*/

package main

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/bwmarrin/discordgo"
)

// "0⃣"
var numBlocks = []string{"1⃣", "2⃣", "3⃣", "4⃣", "5⃣", "6⃣", "7⃣", "8⃣", "9⃣"}

// StartVote - begin a vote with variable vote options
func StartVote(s *discordgo.Session, m *discordgo.MessageCreate, data string, quote bool) {
	// Parse the incoming command into an array of options
	array := strings.SplitAfter(data, "|")
	options := len(array)

	if options > 9 {
		s.ChannelMessageSend(m.ChannelID, "9 options max, since Discord only has 9 number emotes")
		return
	}

	s.ChannelMessageSend(m.ChannelID, "```\nStarting vote...react now!\n```")

	// Send vote text
	voteMsg := VotePrint(s, m, data, quote)

	// Insert the vote into the vote table
	k.kdb.CreateVote(voteMsg.ID, m.GuildID, m.Author.ID, options, quote, data)

	// Upsert the vote in the watch table
	k.kdb.CreateWatch(voteMsg.ID, "vote")
}

//ReactionAdd - add a reaction to the passed-in message
func ReactionAdd(s *discordgo.Session, m *discordgo.Message, reaction string) {

	switch reaction {
	case "UPVOTE":
		err := s.MessageReactionAdd(m.ChannelID, m.ID, "⬆")
		if err != nil {
			s.ChannelMessageSend(m.ChannelID, "Unable to use upvote emote")
		}
		break

	case "DOWNVOTE":
		err := s.MessageReactionAdd(m.ChannelID, m.ID, "⬇")
		if err != nil {
			s.ChannelMessageSend(m.ChannelID, "Unable to use downvote emote")
		}
		break
	default:
		s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Unable to post emote: %s", reaction))
		break
	}
}

// HandleVote - check if a vote is valid and process accordingly
func (vote *Vote) HandleVote(s *discordgo.Session, r *discordgo.MessageReactionAdd) {
	msg, err := s.ChannelMessage(r.ChannelID, r.MessageID)
	if err != nil {
		panic(err)
	}

	logInfo := fmt.Sprintf("[Reaction] \"%s\" by [%s] found on message %s", r.Emoji.Name, r.UserID, r.MessageID)
	k.Log("VOTE", logInfo)

	for i, react := range msg.Reactions {
		if react.Me && react.Count > k.botConfig.MinVotes {
			vote.Result = i + 1
			vote.UpdateVote()
		}

		if react.Me && r.UserID == vote.SubmitterID && vote.Quote && react.Emoji.Name == "⬇" {
			vote.EndVote()
		}
	}

	if vote.Quote && vote.Result != -1 {
		if vote.Result == 1 {
			// Print finalized quote with identifier
			s.ChannelMessageSend(r.ChannelID, "Vote succeeded, yay!")
			k.kdb.DeleteWatch(r.MessageID)
			quoteAdded := k.kdb.CreateQuote(s, vote.GuildID, vote.VoteText)
			quoteAdded.RequestIdentifier(s, r)

		} else {
			s.ChannelMessageSend(r.ChannelID, "Vote failed, yikes!")
			k.kdb.DeleteWatch(r.MessageID)
		}
		return
	}

	k.Log("TEST", vote.VoteText)

	if strings.Contains(vote.VoteText, "👢") && vote.Result == 1 {
		guild, err := s.Guild(r.GuildID)
		if err != nil {
			panic(err)
		}

		// Pull userID out of the  voteText
		magic := regexp.MustCompile(`\((.*?)\)`)
		result := magic.FindStringSubmatch(vote.VoteText)
		userID := result[1]

		err = s.GuildMemberMove(r.GuildID, userID, &guild.AfkChannelID)
		if err != nil {
			s.ChannelMessageSend(r.ChannelID, "ERR: Unable to 'kick' member: "+err.Error())
		} else {
			s.ChannelMessageSend(r.ChannelID, "See ya!")
		}
		vote.EndVote()
		k.kdb.DeleteWatch(r.MessageID)
		return
	}

	if strings.Contains(vote.VoteText, "✨") {
		if vote.Result == 1 {
			guild := k.kdb.ReadGuild(s, r.GuildID)

			err := s.GuildMemberRoleAdd(r.GuildID, vote.SubmitterID, guild.MemberRole)
			if err != nil {
				s.ChannelMessageSend(r.ChannelID, "ERR: Unable to 'add' member: "+err.Error())
			} else {
				s.ChannelMessageSend(r.ChannelID, "Added member successfully!")
			}
			vote.EndVote()
			k.kdb.DeleteWatch(r.MessageID)
		}
		return
	}

	if vote.Result >= 0 {
		optionArray := strings.SplitAfter(vote.VoteText, "|")
		option := strings.TrimSpace(strings.TrimRight(optionArray[vote.Result-1], "|"))

		s.ChannelMessageSend(r.ChannelID, fmt.Sprintf("```\nOption %d, \"%s\", wins the vote!\n```", vote.Result, option))
		vote.EndVote()
		k.kdb.DeleteWatch(r.MessageID)
	}
}

// VotePrint - print out a vote and add reactions
func VotePrint(s *discordgo.Session, m *discordgo.MessageCreate, voteText string, quote bool) (message *discordgo.Message) {
	var err error
	array := strings.SplitAfter(voteText, "|")

	if quote {
		voteMsg := "```\n" + voteText + "\n```"
		message, err = s.ChannelMessageSend(m.ChannelID, voteMsg)
		if err != nil {
			panic(err)
		}
	} else {

		voteMsg := "```\n"
		for i, option := range array {
			cleanOption := strings.TrimSpace(strings.TrimRight(option, "|"))
			voteIndex := ""
			if len(array) > 1 {
				voteIndex = fmt.Sprintf("%d. ", i+1)
			}
			voteMsg += fmt.Sprintf("%s%s\n", voteIndex, cleanOption)
		}
		voteMsg += "```"

		message, err = s.ChannelMessageSend(m.ChannelID, voteMsg)
		if err != nil {
			panic(err)
		}
	}

	// Add reactions (up/down for single or numbers per option)
	if len(array) == 1 {
		err := s.MessageReactionAdd(m.ChannelID, message.ID, "⬆")
		if err != nil {
			panic(err)
		}
		err = s.MessageReactionAdd(m.ChannelID, message.ID, "⬇")
		if err != nil {
			panic(err)
		}
		return message
	}

	for i := range array {
		err := s.MessageReactionAdd(m.ChannelID, message.ID, numBlocks[i])
		if err != nil {
			panic(err)
		}
	}

	return message
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/OneOfOne/xxhash/native"
	"github.com/albrow/scribble/compilers"
	"github.com/albrow/scribble/config"
	"github.com/albrow/scribble/log"
	"github.com/albrow/scribble/util"
	"github.com/howeyc/fsnotify"
	"io"
	"os"
	"path/filepath"
	"sync"
)

var watchedPaths = []string{}
var watcher *fsnotify.Watcher
var watchMutex = sync.Mutex{}

// a map of known file hashes. This is required to
// determine whether a file actually changed. It is a workaround
// to fix the bug that occurs when a text editor uses atomic saves,
// which triggers multiple watch events even though the file was
// only saved once.
var fileHashes = map[string][]byte{}

// watchAll begins watching all the files in config.SourceDir and reacts
// to any changes.
func watchAll() {
	log.Default.Println("Watching for changes...")
	if watcher == nil {
		var err error
		watcher, err = createWatcher()
		if err != nil {
			panic(err)
		}
	}

	// walk through source dir and watch all subdirectories
	// we have to do this because fsnotify is currently not recursive
	if err := watcher.Watch(config.SourceDir); err != nil {
		panic(err)
	}
	if err := filepath.Walk(config.SourceDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.Name()[0] == '.' {
			// ignore hidden system files
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		if info.IsDir() {
			watchMutex.Lock()
			watchedPaths = append(watchedPaths, path)
			if err := watcher.Watch(path); err != nil {
				panic(err)
			}
			watchMutex.Unlock()
		}
		return nil
	}); err != nil {
		panic(err)
	}
}

// createWatcher creates and returns an fsnotify.Watcher.
func createWatcher() (*fsnotify.Watcher, error) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	// Process events
	go func() {
		defer func() {
			// Recover from panics and then restart the watchLoop if needed
			util.Recovery(*compileTrace || *serveTrace)
			watchLoop(watcher)
		}()
		watchLoop(watcher)
	}()
	return watcher, nil
}

func watchLoop(watcher *fsnotify.Watcher) {
	for {
		select {
		case ev := <-watcher.Event:
			if changed, err := fileDidChange(ev.Name); err != nil {
				util.ChimeError(err)
			} else if changed {
				if err := compilers.FileChanged(ev.Name, ev); err != nil {
					util.ChimeError(err)
				}
			}
		case err := <-watcher.Error:
			util.ChimeError(err)
		}
	}
}

// fileDidChange uses the last known hash to determine whether or
// not the file actually changed. It solves the problem of false positives
// coming from fsnotify when used with a text editor that uses atomic saves.
func fileDidChange(path string) (bool, error) {
	if hash, found := fileHashes[path]; !found {
		// we have not hashed the file before.
		// hash it now and store the value
		if newHash, exists, err := calculateHashForPath(path); err != nil {
			return false, err
		} else if exists {
			fileHashes[path] = newHash
		}
		return true, nil
	} else {
		if newHash, exists, err := calculateHashForPath(path); err != nil {
			return false, err
		} else if !exists {
			// if the file no longer exists, it has been deleted
			// we should consider that a change and recompile
			delete(fileHashes, path)
			return true, nil
		} else if string(newHash) != string(hash) {
			// if the file does exist and has a different hash, there
			// was an actual change and we should recompile
			fileHashes[path] = newHash
			return true, nil
		}
		return false, nil
	}
}

// calculateHashForPath calculates a hash for the file at the given path.
// If the file does not exist, the second return value will be false.
func calculateHashForPath(path string) ([]byte, bool, error) {
	h := xxhash.New64()
	// Check if the file is a directory
	info, err := os.Lstat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		} else {
			return nil, false, err
		}
	}
	if info.IsDir() {
		// For dirs, return nil hash
		return nil, true, nil
	}
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		} else {
			return nil, false, err
		}
	}
	if _, err := io.Copy(h, f); err != nil {
		return nil, false, err
	}
	if h.Size() == 0 {
		// The file existed, but it was empty
		return nil, true, nil
	}
	result := h.Sum(nil)
	return result, true, nil
}
<commit_msg>Fix bug where empty files were causing watch/serve to crash<commit_after>package main

import (
	"github.com/OneOfOne/xxhash/native"
	"github.com/albrow/scribble/compilers"
	"github.com/albrow/scribble/config"
	"github.com/albrow/scribble/log"
	"github.com/albrow/scribble/util"
	"github.com/howeyc/fsnotify"
	"io"
	"os"
	"path/filepath"
	"sync"
)

var watchedPaths = []string{}
var watcher *fsnotify.Watcher
var watchMutex = sync.Mutex{}

// a map of known file hashes. This is required to
// determine whether a file actually changed. It is a workaround
// to fix the bug that occurs when a text editor uses atomic saves,
// which triggers multiple watch events even though the file was
// only saved once.
var fileHashes = map[string][]byte{}

// watchAll begins watching all the files in config.SourceDir and reacts
// to any changes.
func watchAll() {
	log.Default.Println("Watching for changes...")
	if watcher == nil {
		var err error
		watcher, err = createWatcher()
		if err != nil {
			panic(err)
		}
	}

	// walk through source dir and watch all subdirectories
	// we have to do this because fsnotify is currently not recursive
	if err := watcher.Watch(config.SourceDir); err != nil {
		panic(err)
	}
	if err := filepath.Walk(config.SourceDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.Name()[0] == '.' {
			// ignore hidden system files
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		if info.IsDir() {
			watchMutex.Lock()
			watchedPaths = append(watchedPaths, path)
			if err := watcher.Watch(path); err != nil {
				panic(err)
			}
			watchMutex.Unlock()
		}
		return nil
	}); err != nil {
		panic(err)
	}
}

// createWatcher creates and returns an fsnotify.Watcher.
func createWatcher() (*fsnotify.Watcher, error) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	// Process events
	go func() {
		defer func() {
			// Recover from panics and then restart the watchLoop if needed
			util.Recovery(*compileTrace || *serveTrace)
			watchLoop(watcher)
		}()
		watchLoop(watcher)
	}()
	return watcher, nil
}

func watchLoop(watcher *fsnotify.Watcher) {
	for {
		select {
		case ev := <-watcher.Event:
			if changed, err := fileDidChange(ev.Name); err != nil {
				util.ChimeError(err)
			} else if changed {
				if err := compilers.FileChanged(ev.Name, ev); err != nil {
					util.ChimeError(err)
				}
			}
		case err := <-watcher.Error:
			util.ChimeError(err)
		}
	}
}

// fileDidChange uses the last known hash to determine whether or
// not the file actually changed. It solves the problem of false positives
// coming from fsnotify when used with a text editor that uses atomic saves.
func fileDidChange(path string) (bool, error) {
	if hash, found := fileHashes[path]; !found {
		// we have not hashed the file before.
		// hash it now and store the value
		if newHash, exists, err := calculateHashForPath(path); err != nil {
			return false, err
		} else if exists {
			fileHashes[path] = newHash
		}
		return true, nil
	} else {
		if newHash, exists, err := calculateHashForPath(path); err != nil {
			return false, err
		} else if !exists {
			// if the file no longer exists, it has been deleted
			// we should consider that a change and recompile
			delete(fileHashes, path)
			return true, nil
		} else if string(newHash) != string(hash) {
			// if the file does exist and has a different hash, there
			// was an actual change and we should recompile
			fileHashes[path] = newHash
			return true, nil
		}
		return false, nil
	}
}

// calculateHashForPath calculates a hash for the file at the given path.
// If the file does not exist, the second return value will be false.
func calculateHashForPath(path string) ([]byte, bool, error) {
	h := xxhash.New64()
	// Check if the file is a directory
	info, err := os.Lstat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		} else {
			return nil, false, err
		}
	}
	if info.IsDir() {
		// For dirs, return nil hash
		return nil, true, nil
	}
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		} else {
			return nil, false, err
		}
	}
	if _, err := io.Copy(h, f); err != nil {
		return nil, false, err
	}

	fstat, err := f.Stat()
	if err != nil {
		return nil, false, err
	}
	if fstat.Size() == 0 {
		// The file existed, but it was empty
		return nil, true, nil
	}
	result := h.Sum(nil)
	return result, true, nil
}
<|endoftext|>


<commit_before>package watch

import (
	"sync/atomic"
	"time"

	"github.com/omeid/gonzo/context"
	"github.com/omeid/kargar"
	"github.com/go-gonzo/fs/glob"
	"gopkg.in/fsnotify.v1"
)

func throttle(limit time.Duration) func(func()) bool {
	var last int64
	lims := limit.Seconds()

	return func(cb func()) bool {
		now := time.Now().Unix()
		l := atomic.LoadInt64(&last)

		if l+int64(lims) < now {
			cb()
			atomic.StoreInt64(&last, now)
			return true
		}
		return false
	}
}

func Watcher(ctx context.Context, cb func(string), globs ...string) error {

	files, err := glob.Glob(globs...)

	if err != nil {
		return err
	}

	w, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	for matchpair := range files {
		w.Add(matchpair.Name)
	}

	throttled := throttle(50 * time.Millisecond)
	go func() {
		for {
			select {
			case event := <-w.Events:
				//if event.Op&fsnotify.Write == fsnotify.Write {
				//event.Op&fsnotify.Create == fsnotify.Create ||
				throttled(func() {
					cb(event.Name)
				})
				//}
			case err := <-w.Errors:
				if err != nil {
					ctx.Error(err)
				}
			case <-ctx.Done():
				w.Close()
				return
			}
		}
	}()

	return nil
}

func WatchSet(cb func(context.Context, ...string) error, watches map[string][]string) kargar.Action {
	return func(ctx context.Context) error {

		//function wrapper to copy set and files.
		for set, files := range watches {
			var s = set
			err := Watcher(
				ctx,
				func(string) { cb(ctx, s) },
				files...,
			)
			if err != nil {
				return err
			}
		}
		<-ctx.Done()
		return nil

	}
}
<commit_msg>Watch the dir of go files so that it works with vim et al.<commit_after>package watch

import (
	"path/filepath"
	"sync/atomic"
	"time"

	"github.com/go-gonzo/fs/glob"
	"github.com/omeid/gonzo/context"
	"github.com/omeid/kargar"
	"gopkg.in/fsnotify.v1"
)

func throttle(limit time.Duration) func(func()) bool {
	var last int64
	lims := limit.Seconds()

	return func(cb func()) bool {
		now := time.Now().Unix()
		l := atomic.LoadInt64(&last)

		if l+int64(lims) < now {
			cb()
			atomic.StoreInt64(&last, now)
			return true
		}
		return false
	}
}

// Watcher ...
func Watcher(ctx context.Context, cb func(string), globs ...string) error {

	files, err := glob.Glob(globs...)

	if err != nil {
		return err
	}

	w, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	for matchpair := range files {
		w.Add(filepath.Dir(matchpair.Name))
		w.Add(matchpair.Name)
	}

	throttled := throttle(50 * time.Millisecond)
	go func() {
		for {
			select {
			case event := <-w.Events:
				ctx.Debugf("gonzo:watch:Event %v on %v", event, event.Name)
				//if event.Op&fsnotify.Write == fsnotify.Write {
				//event.Op&fsnotify.Create == fsnotify.Create ||
				throttled(func() {
					cb(event.Name)
				})
				//}
			case err := <-w.Errors:
				if err != nil {
					ctx.Error(err)
				}
			case <-ctx.Done():
				w.Close()
				return
			}
		}
	}()

	return nil
}

// WatcherSet is like watcher...
func WatcherSet(cb func(context.Context, ...string) error, watches map[string][]string) kargar.Action {
	return func(ctx context.Context) error {

		//function wrapper to copy set and files.
		for set, files := range watches {
			var s = set
			err := Watcher(
				ctx,
				func(string) { cb(ctx, s) },
				files...,
			)
			if err != nil {
				return err
			}
		}
		<-ctx.Done()
		return nil

	}
}
<|endoftext|>


<commit_before>package news_getter

import(
	"fmt"
	"encoding/json"
	"time"

	"web_apps/news_aggregator/modules/database"
)

var (
	loop_counter_delay = 10
	hacker_news_provider = "https://news.ycombinator.com"
	hacker_news_name	= "hackernews"
)

type HackerNewsTopStoriesId []int

func StartHackerNews() {
	top_stories_ids := topStoriesId()
	content_out := make(chan jsonNewsBody)
	time_profiler := make(chan string)

	go func() {
		for t := range time.Tick(time.Duration(loop_counter_delay) * time.Second) {
			fmt.Println("running the loop")
			_ = t
			for _, id := range top_stories_ids {
				go func(id int, content_out chan jsonNewsBody, time_profiler chan string) {
					start := time.Now()
					news_content := hackerNewsReader(id)
					content_out <- news_content
					time_profiler <- fmt.Sprintf("%v", time.Since(start))
				}(id, content_out, time_profiler)
			}
		}
	}()

	go func() {
		for {
			content_out_msg := <- content_out
			time_profiler_out := <- time_profiler
			//fmt.Println(content_out_msg.Title)
			//fmt.Println(content_out_msg.Url)
			time_f := content_out_msg.Time
			content_out_msg.CreatedAt = fmt.Sprintf("%v", time.Now().Local())
			content_out_msg.ProviderUrl = hacker_news_provider
			content_out_msg.ProviderName = hacker_news_name
			//fmt.Println(time.Unix(int64(time_f), 0))
			_ = time_f

			// check if can save
			// then save
			can_save := database.HackerNewsFindIfExist(content_out_msg.Title)
			if can_save {
				database.HackerNewsInsert(content_out_msg)
			} else {
				//fmt.Println("did not save!")
			}
			_ = time_profiler_out
			//fmt.Println(time_profiler_out)
			//fmt.Println("----------------------------")
		}
	}()

}

func topStoriesId() []int {
	var top_stories_id_url string = "https://hacker-news.firebaseio.com/v0/topstories.json"
	var id_containers HackerNewsTopStoriesId
	response, _ := httpGet(top_stories_id_url)
	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &id_containers); err != nil {
		return id_containers
	}

	// make error handler
	return id_containers
}

func hackerNewsReader(id int) jsonNewsBody{
	news_url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
	var news_content jsonNewsBody
	response, err := httpGet(news_url)
	if err != nil {
		//fmt.Println(err)
		return news_content
	}
	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &news_content); err != nil {
		return news_content
	}
	return news_content
}
<commit_msg>update timer and top stories id<commit_after>package news_getter

import(
	"fmt"
	"encoding/json"
	"time"

	"web_apps/news_aggregator/modules/database"
)

var (
	loop_counter_delay = 300
	hacker_news_provider = "https://news.ycombinator.com"
	hacker_news_name	= "hackernews"
)

type HackerNewsTopStoriesId []int

func StartHackerNews() {
	content_out := make(chan jsonNewsBody)
	time_profiler := make(chan string)

	go func() {
		for t := range time.Tick(time.Duration(loop_counter_delay) * time.Second) {
			top_stories_ids := topStoriesId()
			fmt.Println("running the loop")
			_ = t
			for _, id := range top_stories_ids {
				go func(id int, content_out chan jsonNewsBody, time_profiler chan string) {
					start := time.Now()
					news_content := hackerNewsReader(id)
					content_out <- news_content
					time_profiler <- fmt.Sprintf("%v", time.Since(start))
				}(id, content_out, time_profiler)
			}
		}
	}()

	go func() {
		for {
			content_out_msg := <- content_out
			time_profiler_out := <- time_profiler
			//fmt.Println(content_out_msg.Title)
			//fmt.Println(content_out_msg.Url)
			time_f := content_out_msg.Time
			content_out_msg.CreatedAt = fmt.Sprintf("%v", time.Now().Local())
			content_out_msg.ProviderUrl = hacker_news_provider
			content_out_msg.ProviderName = hacker_news_name
			//fmt.Println(time.Unix(int64(time_f), 0))
			_ = time_f

			// check if can save
			// then save
			can_save := database.HackerNewsFindIfExist(content_out_msg.Title)
			if can_save {
				database.HackerNewsInsert(content_out_msg)
			} else {
				//fmt.Println("did not save!")
			}
			_ = time_profiler_out
			//fmt.Println(time_profiler_out)
			//fmt.Println("----------------------------")
		}
	}()

}

func topStoriesId() []int {
	var top_stories_id_url string = "https://hacker-news.firebaseio.com/v0/topstories.json"
	var id_containers HackerNewsTopStoriesId
	response, _ := httpGet(top_stories_id_url)
	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &id_containers); err != nil {
		return id_containers
	}

	// make error handler
	return id_containers
}

func hackerNewsReader(id int) jsonNewsBody{
	news_url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
	var news_content jsonNewsBody
	response, err := httpGet(news_url)
	if err != nil {
		//fmt.Println(err)
		return news_content
	}
	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &news_content); err != nil {
		return news_content
	}
	return news_content
}
<|endoftext|>


<commit_before>//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ../../../fakes/fake_registry_store.go ../../../../vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go Query
package grpc

import (
	"context"
	"net"
	"net/url"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/sirupsen/logrus"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc"
	"google.golang.org/grpc/connectivity"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/fakes"
	"github.com/operator-framework/operator-registry/pkg/api"
	opregistry "github.com/operator-framework/operator-registry/pkg/registry"
	opserver "github.com/operator-framework/operator-registry/pkg/server"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry"
)

func server(store opregistry.Query) (func(), string, func()) {
	lis, err := net.Listen("tcp", "localhost:")
	if err != nil {
		logrus.Fatalf("failed to listen: %v", err)
	}
	s := grpc.NewServer()

	api.RegisterRegistryServer(s, opserver.NewRegistryServer(store))

	serve := func() {
		if err := s.Serve(lis); err != nil {
			logrus.Fatalf("failed to serve: %v", err)
		}
	}

	stop := func() {
		s.Stop()
	}

	return serve, lis.Addr().String(), stop
}

type FakeSourceSyncer struct {
	// using a map[int] to preserve order
	History map[registry.CatalogKey][]connectivity.State

	sync.Mutex
	expectedEvents int
	done           chan struct{}
}

func (f *FakeSourceSyncer) sync(state SourceState) {
	f.Lock()
	if f.History[state.Key] == nil {
		f.History[state.Key] = []connectivity.State{}
	}
	f.History[state.Key] = append(f.History[state.Key], state.State)
	f.expectedEvents--
	if f.expectedEvents == 0 {
		f.done <- struct{}{}
	}
	f.Unlock()
}

func NewFakeSourceSyncer(expectedEvents int) *FakeSourceSyncer {
	return &FakeSourceSyncer{
		History:        map[registry.CatalogKey][]connectivity.State{},
		expectedEvents: expectedEvents,
		done:           make(chan struct{}),
	}
}

func TestConnectionEvents(t *testing.T) {
	type testcase struct {
		name            string
		expectedHistory map[registry.CatalogKey][]connectivity.State
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			// start server for each catalog
			totalEvents := 0
			addresses := map[registry.CatalogKey]string{}

			for catalog, events := range tt.expectedHistory {
				totalEvents += len(events)
				serve, address, stop := server(&fakes.FakeQuery{})
				addresses[catalog] = address
				go serve()
				defer stop()
			}

			// start source manager
			syncer := NewFakeSourceSyncer(totalEvents)
			sources := NewSourceStore(logrus.New(), 1*time.Second, 5*time.Second, syncer.sync)
			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
			defer cancel()
			sources.Start(ctx)

			// add source for each catalog
			for catalog, address := range addresses {
				_, err := sources.Add(catalog, address)
				require.NoError(t, err)
			}

			// wait for syncing to finish
			<-syncer.done

			// verify sync events
			for catalog, events := range tt.expectedHistory {
				recordedEvents := syncer.History[catalog]
				for i := 0; i < len(recordedEvents); i++ {
					require.Equal(t, (events[i]).String(), (recordedEvents[i]).String())
				}
			}
		}
	}

	cases := []testcase{
		{
			name: "Basic",
			expectedHistory: map[registry.CatalogKey][]connectivity.State{
				{Name: "test", Namespace: "test"}: {
					connectivity.Connecting,
					connectivity.Ready,
				},
			},
		},
		{
			name: "Multiple",
			expectedHistory: map[registry.CatalogKey][]connectivity.State{
				{Name: "test", Namespace: "test"}: {
					connectivity.Connecting,
					connectivity.Ready,
				},
				{Name: "test2", Namespace: "test2"}: {
					connectivity.Connecting,
					connectivity.Ready,
				},
			},
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}

func TestGetEnvAny(t *testing.T) {
	type envVar struct {
		key   string
		value string
	}

	type testcase struct {
		name          string
		envVars       []envVar
		expectedValue string
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			for _, envVar := range tt.envVars {
				os.Setenv(envVar.key, envVar.value)
			}

			defer func() {
				for _, envVar := range tt.envVars {
					os.Setenv(envVar.key, "")
				}
			}()

			require.Equal(t, getEnvAny("NO_PROXY", "no_proxy"), tt.expectedValue)
		}
	}

	cases := []testcase{
		{
			name:          "NotFound",
			expectedValue: "",
		},
		{
			name: "LowerCaseFound",
			envVars: []envVar{
				{
					key:   "no_proxy",
					value: "foo",
				},
			},
			expectedValue: "foo",
		},
		{
			name: "UpperCaseFound",
			envVars: []envVar{
				{
					key:   "NO_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
		{
			name: "OrderPreference",
			envVars: []envVar{
				{
					key:   "no_proxy",
					value: "foo",
				},
				{
					key:   "NO_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}

func TestGetGRPCProxyEnv(t *testing.T) {
	type envVar struct {
		key   string
		value string
	}

	type testcase struct {
		name          string
		envVars       []envVar
		expectedValue string
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			for _, envVar := range tt.envVars {
				os.Setenv(envVar.key, envVar.value)
			}

			defer func() {
				for _, envVar := range tt.envVars {
					os.Setenv(envVar.key, "")
				}
			}()

			require.Equal(t, getGRPCProxyEnv(), tt.expectedValue)
		}
	}

	cases := []testcase{
		{
			name:          "NotFound",
			expectedValue: "",
		},
		{
			name: "LowerCaseFound",
			envVars: []envVar{
				{
					key:   "grpc_proxy",
					value: "foo",
				},
			},
			expectedValue: "foo",
		},
		{
			name: "UpperCaseFound",
			envVars: []envVar{
				{
					key:   "GRPC_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
		{
			name: "UpperCasePreference",
			envVars: []envVar{
				{
					key:   "grpc_proxy",
					value: "foo",
				},
				{
					key:   "GRPC_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}

func TestGRPCProxyURL(t *testing.T) {
	type envVar struct {
		key   string
		value string
	}

	type testcase struct {
		name          string
		address       string
		envVars       []envVar
		expectedProxy string
		expectedError error
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			for _, envVar := range tt.envVars {
				os.Setenv(envVar.key, envVar.value)
			}

			defer func() {
				for _, envVar := range tt.envVars {
					os.Setenv(envVar.key, "")
				}
			}()

			var expectedProxyURL *url.URL
			var err error
			if tt.expectedProxy != "" {
				expectedProxyURL, err = url.Parse(tt.expectedProxy)
				require.NoError(t, err)
			}

			proxyURL, err := grpcProxyURL(tt.address)
			require.Equal(t, expectedProxyURL, proxyURL)
			require.Equal(t, tt.expectedError, err)
		}
	}

	cases := []testcase{
		{
			name:          "NoGRPCProxySet",
			address:       "foo.com:8080",
			expectedProxy: "",
			expectedError: nil,
		},
		{
			name:    "GRPCProxyFoundForAddress",
			address: "foo.com:8080",
			envVars: []envVar{
				{
					key:   "GRPC_PROXY",
					value: "http://my-proxy:8080",
				},
			},
			expectedProxy: "http://my-proxy:8080",
			expectedError: nil,
		},
		{
			name:    "GRPCNoProxyIncludesAddress",
			address: "foo.com:8080",
			envVars: []envVar{
				{
					key:   "GRPC_PROXY",
					value: "http://my-proxy:8080",
				},
				{
					key:   "NO_PROXY",
					value: "foo.com:8080",
				},
			},
			expectedProxy: "",
			expectedError: nil,
		},
		{
			name:          "MissingPort",
			address:       "foo.com",
			expectedProxy: "",
			expectedError: error(&net.AddrError{Err: "missing port in address", Addr: "foo.com"}),
		},
		{
			name:          "TooManyColons",
			address:       "http://bar.com:8080",
			expectedProxy: "",
			expectedError: error(&net.AddrError{Err: "too many colons in address", Addr: "http://bar.com:8080"}),
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}
<commit_msg>fix registry grpc unit test intermittent failure (#2521)<commit_after>//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ../../../fakes/fake_registry_store.go ../../../../vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go Query
package grpc

import (
	"context"
	"net"
	"net/url"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/sirupsen/logrus"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc"
	"google.golang.org/grpc/connectivity"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/fakes"
	"github.com/operator-framework/operator-registry/pkg/api"
	opregistry "github.com/operator-framework/operator-registry/pkg/registry"
	opserver "github.com/operator-framework/operator-registry/pkg/server"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry"
)

func server(store opregistry.Query) (func(), string, func()) {
	lis, err := net.Listen("tcp", "localhost:")
	if err != nil {
		logrus.Fatalf("failed to listen: %v", err)
	}
	s := grpc.NewServer()

	api.RegisterRegistryServer(s, opserver.NewRegistryServer(store))

	serve := func() {
		if err := s.Serve(lis); err != nil {
			logrus.Fatalf("failed to serve: %v", err)
		}
	}

	stop := func() {
		s.Stop()
	}

	return serve, lis.Addr().String(), stop
}

type FakeSourceSyncer struct {
	// using a map[int] to preserve order
	History map[registry.CatalogKey][]connectivity.State

	sync.Mutex
	expectedReadies int
	done            chan struct{}
}

func (f *FakeSourceSyncer) sync(state SourceState) {
	f.Lock()
	if f.History[state.Key] == nil {
		f.History[state.Key] = []connectivity.State{}
	}
	f.History[state.Key] = append(f.History[state.Key], state.State)
	if state.State == connectivity.Ready {
		f.expectedReadies--
	}
	if f.expectedReadies == 0 {
		f.done <- struct{}{}
	}
	f.Unlock()
}

func NewFakeSourceSyncer(expectedReadies int) *FakeSourceSyncer {
	return &FakeSourceSyncer{
		History:         map[registry.CatalogKey][]connectivity.State{},
		expectedReadies: expectedReadies,
		done:            make(chan struct{}),
	}
}

func TestConnectionEvents(t *testing.T) {
	type testcase struct {
		name            string
		expectedHistory map[registry.CatalogKey][]connectivity.State
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			// start server for each catalog
			addresses := map[registry.CatalogKey]string{}

			for catalog := range tt.expectedHistory {
				serve, address, stop := server(&fakes.FakeQuery{})
				addresses[catalog] = address
				go serve()
				defer stop()
			}

			// start source manager
			syncer := NewFakeSourceSyncer(len(tt.expectedHistory))
			sources := NewSourceStore(logrus.New(), 1*time.Second, 5*time.Second, syncer.sync)
			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
			defer cancel()
			sources.Start(ctx)

			// add source for each catalog
			for catalog, address := range addresses {
				_, err := sources.Add(catalog, address)
				require.NoError(t, err)
			}

			// wait for syncing to finish
			<-syncer.done

			// verify sync events
			for catalog, events := range tt.expectedHistory {
				recordedEvents := syncer.History[catalog]
				for i := 0; i < len(recordedEvents); i++ {
					found := false
					for _, event := range events {
						if event.String() == recordedEvents[i].String() {
							found = true
						}
					}
					require.True(t, found)
				}
			}
		}
	}

	cases := []testcase{
		{
			name: "Basic",
			expectedHistory: map[registry.CatalogKey][]connectivity.State{
				{Name: "test", Namespace: "test"}: {
					connectivity.Connecting,
					connectivity.Ready,
				},
			},
		},
		{
			name: "Multiple",
			expectedHistory: map[registry.CatalogKey][]connectivity.State{
				{Name: "test", Namespace: "test"}: {
					connectivity.Connecting,
					connectivity.Ready,
				},
				{Name: "test2", Namespace: "test2"}: {
					connectivity.Connecting,
					connectivity.Ready,
				},
			},
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}

func TestGetEnvAny(t *testing.T) {
	type envVar struct {
		key   string
		value string
	}

	type testcase struct {
		name          string
		envVars       []envVar
		expectedValue string
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			for _, envVar := range tt.envVars {
				os.Setenv(envVar.key, envVar.value)
			}

			defer func() {
				for _, envVar := range tt.envVars {
					os.Setenv(envVar.key, "")
				}
			}()

			require.Equal(t, getEnvAny("NO_PROXY", "no_proxy"), tt.expectedValue)
		}
	}

	cases := []testcase{
		{
			name:          "NotFound",
			expectedValue: "",
		},
		{
			name: "LowerCaseFound",
			envVars: []envVar{
				{
					key:   "no_proxy",
					value: "foo",
				},
			},
			expectedValue: "foo",
		},
		{
			name: "UpperCaseFound",
			envVars: []envVar{
				{
					key:   "NO_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
		{
			name: "OrderPreference",
			envVars: []envVar{
				{
					key:   "no_proxy",
					value: "foo",
				},
				{
					key:   "NO_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}

func TestGetGRPCProxyEnv(t *testing.T) {
	type envVar struct {
		key   string
		value string
	}

	type testcase struct {
		name          string
		envVars       []envVar
		expectedValue string
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			for _, envVar := range tt.envVars {
				os.Setenv(envVar.key, envVar.value)
			}

			defer func() {
				for _, envVar := range tt.envVars {
					os.Setenv(envVar.key, "")
				}
			}()

			require.Equal(t, getGRPCProxyEnv(), tt.expectedValue)
		}
	}

	cases := []testcase{
		{
			name:          "NotFound",
			expectedValue: "",
		},
		{
			name: "LowerCaseFound",
			envVars: []envVar{
				{
					key:   "grpc_proxy",
					value: "foo",
				},
			},
			expectedValue: "foo",
		},
		{
			name: "UpperCaseFound",
			envVars: []envVar{
				{
					key:   "GRPC_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
		{
			name: "UpperCasePreference",
			envVars: []envVar{
				{
					key:   "grpc_proxy",
					value: "foo",
				},
				{
					key:   "GRPC_PROXY",
					value: "bar",
				},
			},
			expectedValue: "bar",
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}

func TestGRPCProxyURL(t *testing.T) {
	type envVar struct {
		key   string
		value string
	}

	type testcase struct {
		name          string
		address       string
		envVars       []envVar
		expectedProxy string
		expectedError error
	}

	test := func(tt testcase) func(t *testing.T) {
		return func(t *testing.T) {
			for _, envVar := range tt.envVars {
				os.Setenv(envVar.key, envVar.value)
			}

			defer func() {
				for _, envVar := range tt.envVars {
					os.Setenv(envVar.key, "")
				}
			}()

			var expectedProxyURL *url.URL
			var err error
			if tt.expectedProxy != "" {
				expectedProxyURL, err = url.Parse(tt.expectedProxy)
				require.NoError(t, err)
			}

			proxyURL, err := grpcProxyURL(tt.address)
			require.Equal(t, expectedProxyURL, proxyURL)
			require.Equal(t, tt.expectedError, err)
		}
	}

	cases := []testcase{
		{
			name:          "NoGRPCProxySet",
			address:       "foo.com:8080",
			expectedProxy: "",
			expectedError: nil,
		},
		{
			name:    "GRPCProxyFoundForAddress",
			address: "foo.com:8080",
			envVars: []envVar{
				{
					key:   "GRPC_PROXY",
					value: "http://my-proxy:8080",
				},
			},
			expectedProxy: "http://my-proxy:8080",
			expectedError: nil,
		},
		{
			name:    "GRPCNoProxyIncludesAddress",
			address: "foo.com:8080",
			envVars: []envVar{
				{
					key:   "GRPC_PROXY",
					value: "http://my-proxy:8080",
				},
				{
					key:   "NO_PROXY",
					value: "foo.com:8080",
				},
			},
			expectedProxy: "",
			expectedError: nil,
		},
		{
			name:          "MissingPort",
			address:       "foo.com",
			expectedProxy: "",
			expectedError: error(&net.AddrError{Err: "missing port in address", Addr: "foo.com"}),
		},
		{
			name:          "TooManyColons",
			address:       "http://bar.com:8080",
			expectedProxy: "",
			expectedError: error(&net.AddrError{Err: "too many colons in address", Addr: "http://bar.com:8080"}),
		},
	}

	for _, tt := range cases {
		t.Run(tt.name, test(tt))
	}
}
<|endoftext|>