<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logging

import (
	"testing"

	"github.com/google/go-cmp/cmp"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	. "knative.dev/pkg/configmap/testing"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/system"
	_ "knative.dev/pkg/system/testing"
)

func TestNewConfigNoEntry(t *testing.T) {
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: "knative-something",
			Name:      logging.ConfigMapName(),
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}

	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}

	if diff := cmp.Diff(cm, c); diff != "" {
		t.Fatalf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
	if got := c.LoggingConfig; got == "" {
		t.Error("LoggingConfig = empty, want not empty")
	}
	if got, want := len(c.LoggingLevel), 0; got != want {
		t.Errorf("len(LoggingLevel) = %d, want %d", got, want)
	}
	for _, component := range []string{"controller", "queueproxy", "webhook", "activator", "autoscaler"} {
		if got, want := c.LoggingLevel[component], zap.InfoLevel; got != want {
			t.Errorf("LoggingLevel[%s] = %q, want %q", component, got, want)
		}
	}
}

func TestNewConfig(t *testing.T) {
	wantCfg := "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}"
	wantLevel := zapcore.InfoLevel
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   wantCfg,
			"loglevel.queueproxy": wantLevel.String(),
		},
	}

	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}
	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}
	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
	if got := c.LoggingConfig; got != wantCfg {
		t.Errorf("LoggingConfig = %v, want %v", got, wantCfg)
	}
	if got := c.LoggingLevel["queueproxy"]; got != wantLevel {
		t.Errorf("LoggingLevel[queueproxy] = %v, want %v", got, wantLevel)
	}
}

func TestOurConfig(t *testing.T) {
	cm, example := ConfigMapsFromTestFile(t, logging.ConfigMapName())

	if cfg, err := logging.NewConfigFromConfigMap(cm); err != nil {
		t.Error("Expected no errors. got:", err)
	} else if cfg == nil {
		t.Errorf("NewConfigFromConfigMap(actual) = %v, want non-nil", cfg)
	}

	if cfg, err := logging.NewConfigFromConfigMap(example); err != nil {
		t.Error("Expected no errors. got:", err)
	} else if cfg == nil {
		t.Errorf("NewConfigFromConfigMap(example) = %v, want non-nil", cfg)
	}
}

func TestNewLoggerFromConfig(t *testing.T) {
	c, _, _ := getTestConfig(t)
	_, atomicLevel := logging.NewLoggerFromConfig(c, "queueproxy")
	if atomicLevel.Level() != zapcore.DebugLevel {
		t.Error("logger level wanted: DebugLevel, got:", atomicLevel)
	}
}

func TestEmptyLevel(t *testing.T) {
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}",
			"loglevel.queueproxy": "",
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("Expected no errors, got:", err)
	}

	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("Expected no errors, got:", err)
	}

	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}

	if got, want := c.LoggingLevel["queueproxy"], zapcore.InfoLevel; got != want {
		t.Errorf("LoggingLevel[queueproxy] = %v, want: %v", got, want)
	}
}

func TestInvalidLevel(t *testing.T) {
	wantCfg := "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}"
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   wantCfg,
			"loglevel.queueproxy": "invalid",
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err == nil {
		t.Error("Expected errors. got nothing")
	}

	c, err := logging.NewConfigFromMap(configMap.Data)
	if err == nil {
		t.Error("Expected errors. got nothing")
	}

	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
}

func getTestConfig(t *testing.T) (*logging.Config, string, string) {
	wantCfg := "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}"
	wantLevel := "debug"
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   wantCfg,
			"loglevel.queueproxy": wantLevel,
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("err:", err)
	}
	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("err:", err)
	}
	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
	return c, wantCfg, wantLevel
}

func TestUpdateLevelFromConfigMap(t *testing.T) {
	logger, atomicLevel := logging.NewLogger("", "debug")
	want := zapcore.DebugLevel
	if atomicLevel.Level() != zapcore.DebugLevel {
		t.Fatalf("Expected initial logger level to %v, got: %v", want, atomicLevel.Level())
	}

	cm := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   "",
			"loglevel.controller": "panic",
		},
	}

	tests := []struct {
		setLevel  string
		wantLevel zapcore.Level
	}{
		{"info", zapcore.InfoLevel},
		{"error", zapcore.ErrorLevel},
		{"invalid", zapcore.ErrorLevel},
		{"debug", zapcore.DebugLevel},
		{"debug", zapcore.DebugLevel},
	}

	u := logging.UpdateLevelFromConfigMap(logger, atomicLevel, "controller")
	for _, tt := range tests {
		cm.Data["loglevel.controller"] = tt.setLevel
		u(cm)
		if atomicLevel.Level() != tt.wantLevel {
			t.Errorf("Invalid logging level. want: %v, got: %v", tt.wantLevel, atomicLevel.Level())
		}
	}
}
<commit_msg>Relax empty logging config tests (#9989)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logging

import (
	"testing"

	"github.com/google/go-cmp/cmp"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	. "knative.dev/pkg/configmap/testing"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/system"
	_ "knative.dev/pkg/system/testing"
)

func TestNewConfigNoEntry(t *testing.T) {
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: "knative-something",
			Name:      logging.ConfigMapName(),
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}

	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}

	if diff := cmp.Diff(cm, c); diff != "" {
		t.Fatalf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
	// TODO: assert that c.LoggingConfig is empty.
	if got, want := len(c.LoggingLevel), 0; got != want {
		t.Errorf("len(LoggingLevel) = %d, want %d", got, want)
	}
	for _, component := range []string{"controller", "queueproxy", "webhook", "activator", "autoscaler"} {
		if got, want := c.LoggingLevel[component], zap.InfoLevel; got != want {
			t.Errorf("LoggingLevel[%s] = %q, want %q", component, got, want)
		}
	}
}

func TestNewConfig(t *testing.T) {
	wantCfg := "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}"
	wantLevel := zapcore.InfoLevel
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   wantCfg,
			"loglevel.queueproxy": wantLevel.String(),
		},
	}

	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}
	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("Expected no errors. got:", err)
	}
	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
	if got := c.LoggingConfig; got != wantCfg {
		t.Errorf("LoggingConfig = %v, want %v", got, wantCfg)
	}
	if got := c.LoggingLevel["queueproxy"]; got != wantLevel {
		t.Errorf("LoggingLevel[queueproxy] = %v, want %v", got, wantLevel)
	}
}

func TestOurConfig(t *testing.T) {
	cm, example := ConfigMapsFromTestFile(t, logging.ConfigMapName())

	if cfg, err := logging.NewConfigFromConfigMap(cm); err != nil {
		t.Error("Expected no errors. got:", err)
	} else if cfg == nil {
		t.Errorf("NewConfigFromConfigMap(actual) = %v, want non-nil", cfg)
	}

	if cfg, err := logging.NewConfigFromConfigMap(example); err != nil {
		t.Error("Expected no errors. got:", err)
	} else if cfg == nil {
		t.Errorf("NewConfigFromConfigMap(example) = %v, want non-nil", cfg)
	}
}

func TestNewLoggerFromConfig(t *testing.T) {
	c, _, _ := getTestConfig(t)
	_, atomicLevel := logging.NewLoggerFromConfig(c, "queueproxy")
	if atomicLevel.Level() != zapcore.DebugLevel {
		t.Error("logger level wanted: DebugLevel, got:", atomicLevel)
	}
}

func TestEmptyLevel(t *testing.T) {
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}",
			"loglevel.queueproxy": "",
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("Expected no errors, got:", err)
	}

	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("Expected no errors, got:", err)
	}

	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}

	if got, want := c.LoggingLevel["queueproxy"], zapcore.InfoLevel; got != want {
		t.Errorf("LoggingLevel[queueproxy] = %v, want: %v", got, want)
	}
}

func TestInvalidLevel(t *testing.T) {
	wantCfg := "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}"
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   wantCfg,
			"loglevel.queueproxy": "invalid",
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err == nil {
		t.Error("Expected errors. got nothing")
	}

	c, err := logging.NewConfigFromMap(configMap.Data)
	if err == nil {
		t.Error("Expected errors. got nothing")
	}

	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
}

func getTestConfig(t *testing.T) (*logging.Config, string, string) {
	wantCfg := "{\"level\": \"error\",\n\"outputPaths\": [\"stdout\"],\n\"errorOutputPaths\": [\"stderr\"],\n\"encoding\": \"json\"}"
	wantLevel := "debug"
	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   wantCfg,
			"loglevel.queueproxy": wantLevel,
		},
	}
	cm, err := logging.NewConfigFromConfigMap(configMap)
	if err != nil {
		t.Error("err:", err)
	}
	c, err := logging.NewConfigFromMap(configMap.Data)
	if err != nil {
		t.Error("err:", err)
	}
	if diff := cmp.Diff(cm, c); diff != "" {
		t.Errorf("Config mismatch: diff(-want,+got):\n%s", diff)
	}
	return c, wantCfg, wantLevel
}

func TestUpdateLevelFromConfigMap(t *testing.T) {
	logger, atomicLevel := logging.NewLogger("", "debug")
	want := zapcore.DebugLevel
	if atomicLevel.Level() != zapcore.DebugLevel {
		t.Fatalf("Expected initial logger level to %v, got: %v", want, atomicLevel.Level())
	}

	cm := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: system.Namespace(),
			Name:      logging.ConfigMapName(),
		},
		Data: map[string]string{
			"zap-logger-config":   "",
			"loglevel.controller": "panic",
		},
	}

	tests := []struct {
		setLevel  string
		wantLevel zapcore.Level
	}{
		{"info", zapcore.InfoLevel},
		{"error", zapcore.ErrorLevel},
		{"invalid", zapcore.ErrorLevel},
		{"debug", zapcore.DebugLevel},
		{"debug", zapcore.DebugLevel},
	}

	u := logging.UpdateLevelFromConfigMap(logger, atomicLevel, "controller")
	for _, tt := range tests {
		cm.Data["loglevel.controller"] = tt.setLevel
		u(cm)
		if atomicLevel.Level() != tt.wantLevel {
			t.Errorf("Invalid logging level. want: %v, got: %v", tt.wantLevel, atomicLevel.Level())
		}
	}
}
<|endoftext|>


<commit_before>package service

import (
	"github.com/fagongzi/gateway/pkg/pb/metapb"
	"github.com/fagongzi/grpcx"
	"github.com/fagongzi/log"
	"github.com/labstack/echo"
)

func initPluginRouter(server *echo.Group) {
	server.GET("/plugins/:id",
		grpcx.NewGetHTTPHandle(idParamFactory, getPluginHandler))
	server.DELETE("/plugins/:id",
		grpcx.NewGetHTTPHandle(idParamFactory, deletePluginHandler))
	server.PUT("/plugins",
		grpcx.NewJSONBodyHTTPHandle(putPluginFactory, postPluginHandler))
	server.GET("/plugins",
		grpcx.NewGetHTTPHandle(limitQueryFactory, listPluginHandler))
}

func postPluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	id, err := Store.PutPlugin(value.(*metapb.Plugin))
	if err != nil {
		log.Errorf("api-plugin-put: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{Data: id}, nil
}

func deletePluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	err := Store.RemovePlugin(value.(uint64))
	if err != nil {
		log.Errorf("api-plugin-delete: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{}, nil
}

func getPluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	value, err := Store.GetPlugin(value.(uint64))
	if err != nil {
		log.Errorf("api-plugin-get: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{Data: value}, nil
}

func listPluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	query := value.(*limitQuery)
	var values []*metapb.Plugin

	err := Store.GetPlugins(limit, func(data interface{}) error {
		v := data.(*metapb.Plugin)
		if int64(len(values)) < query.limit && v.ID > query.afterID {
			values = append(values, v)
		}
		return nil
	})
	if err != nil {
		log.Errorf("api-plugin-list-get: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{Data: values}, nil
}

func putPluginFactory() interface{} {
	return &metapb.Plugin{}
}
<commit_msg>dev: add applied plugin api<commit_after>package service

import (
	"github.com/fagongzi/gateway/pkg/pb/metapb"
	"github.com/fagongzi/grpcx"
	"github.com/fagongzi/log"
	"github.com/labstack/echo"
)

func initPluginRouter(server *echo.Group) {
	server.GET("/plugins/:id",
		grpcx.NewGetHTTPHandle(idParamFactory, getPluginHandler))
	server.DELETE("/plugins/:id",
		grpcx.NewGetHTTPHandle(idParamFactory, deletePluginHandler))
	server.PUT("/plugins",
		grpcx.NewJSONBodyHTTPHandle(putPluginFactory, postPluginHandler))
	server.GET("/plugins",
		grpcx.NewGetHTTPHandle(limitQueryFactory, listPluginHandler))
	server.PUT("/plugins/apply",
		grpcx.NewJSONBodyHTTPHandle(putPluginAppliedFactory, putPluginAppliedHandler))
	server.GET("/plugins/apply",
		grpcx.NewGetHTTPHandle(emptyParamFactory, getPluginAppliedHandler))
}

func getPluginAppliedHandler(value interface{}) (*grpcx.JSONResult, error) {
	value, err := Store.GetAppliedPlugins()
	if err != nil {
		log.Errorf("api-plugin-get-applied: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{Data: value}, nil
}

func putPluginAppliedHandler(value interface{}) (*grpcx.JSONResult, error) {
	err := Store.ApplyPlugins(value.(*metapb.AppliedPlugins))
	if err != nil {
		log.Errorf("api-plugin-put-applied: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{}, nil
}

func postPluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	id, err := Store.PutPlugin(value.(*metapb.Plugin))
	if err != nil {
		log.Errorf("api-plugin-put: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{Data: id}, nil
}

func deletePluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	err := Store.RemovePlugin(value.(uint64))
	if err != nil {
		log.Errorf("api-plugin-delete: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{}, nil
}

func getPluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	value, err := Store.GetPlugin(value.(uint64))
	if err != nil {
		log.Errorf("api-plugin-get: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{Data: value}, nil
}

func listPluginHandler(value interface{}) (*grpcx.JSONResult, error) {
	query := value.(*limitQuery)
	var values []*metapb.Plugin

	err := Store.GetPlugins(limit, func(data interface{}) error {
		v := data.(*metapb.Plugin)
		if int64(len(values)) < query.limit && v.ID > query.afterID {
			values = append(values, v)
		}
		return nil
	})
	if err != nil {
		log.Errorf("api-plugin-list-get: req %+v, errors:%+v", value, err)
		return &grpcx.JSONResult{Code: -1, Data: err.Error()}, nil
	}

	return &grpcx.JSONResult{Data: values}, nil
}

func putPluginFactory() interface{} {
	return &metapb.Plugin{}
}

func putPluginAppliedFactory() interface{} {
	return &metapb.AppliedPlugins{}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package net

import (
	"fmt"
	"strconv"
	"strings"
)

// PortRange represents a range of TCP/UDP ports.  To represent a single port,
// set Size to 1.
type PortRange struct {
	Base int
	Size int
}

// Contains tests whether a given port falls within the PortRange.
func (pr *PortRange) Contains(p int) bool {
	return (p >= pr.Base) && ((p - pr.Base) < pr.Size)
}

// String converts the PortRange to a string representation, which can be
// parsed by PortRange.Set or ParsePortRange.
func (pr PortRange) String() string {
	if pr.Size == 0 {
		return ""
	}
	return fmt.Sprintf("%d-%d", pr.Base, pr.Base+pr.Size-1)
}

const (
	SinglePortNotation = iota
	HyphenNotation     = 1 << iota
	PlusNotation
)

// Set parses a string of the form "min-max", inclusive at both ends, and
// sets the PortRange from it.  This is part of the flag.Value and pflag.Value
// interfaces.
func (pr *PortRange) Set(value string) error {

	value = strings.TrimSpace(value)
	hyphenIndex := strings.Index(value, "-")
	plusIndex := strings.Index(value, "+")

	if value == "" {
		pr.Base = 0
		pr.Size = 0
		return nil
	}

	var err error
	var notation int
	var low, high int

	if hyphenIndex >= 0 {
		notation |= HyphenNotation
	}
	if plusIndex != -1 {
		notation |= PlusNotation
	}

	switch notation {
	case SinglePortNotation:
		var port int
		port, err = strconv.Atoi(value)
		if err != nil {
			return err
		}
		low = port
		high = port
	case HyphenNotation:
		low, err = strconv.Atoi(value[:hyphenIndex])
		if err != nil {
			return err
		}
		high, err = strconv.Atoi(value[hyphenIndex+1:])
		if err != nil {
			return err
		}
	case PlusNotation:
		var offset int
		low, err = strconv.Atoi(value[:plusIndex])
		if err != nil {
			return err
		}
		offset, err = strconv.Atoi(value[plusIndex+1:])
		if err != nil {
			return err
		}
		high = low + offset
	default:
		return fmt.Errorf("unable to parse port range: %s", value)
	}

	if low > 65535 || high > 65535 {
		return fmt.Errorf("the port range cannot be greater than 65535: %s", value)
	}

	if high < low {
		return fmt.Errorf("end port cannot be less than start port: %s", value)
	}

	pr.Base = low
	pr.Size = 1 + high - low
	return nil
}

// Type returns a descriptive string about this type.  This is part of the
// pflag.Value interface.
func (*PortRange) Type() string {
	return "portRange"
}

// ParsePortRange parses a string of the form "min-max", inclusive at both
// ends, and initializs a new PortRange from it.
func ParsePortRange(value string) (*PortRange, error) {
	pr := &PortRange{}
	err := pr.Set(value)
	if err != nil {
		return nil, err
	}
	return pr, nil
}

func ParsePortRangeOrDie(value string) *PortRange {
	pr, err := ParsePortRange(value)
	if err != nil {
		panic(fmt.Sprintf("couldn't parse port range %q: %v", value, err))
	}
	return pr
}
<commit_msg>move enum into function local<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package net

import (
	"fmt"
	"strconv"
	"strings"
)

// PortRange represents a range of TCP/UDP ports.  To represent a single port,
// set Size to 1.
type PortRange struct {
	Base int
	Size int
}

// Contains tests whether a given port falls within the PortRange.
func (pr *PortRange) Contains(p int) bool {
	return (p >= pr.Base) && ((p - pr.Base) < pr.Size)
}

// String converts the PortRange to a string representation, which can be
// parsed by PortRange.Set or ParsePortRange.
func (pr PortRange) String() string {
	if pr.Size == 0 {
		return ""
	}
	return fmt.Sprintf("%d-%d", pr.Base, pr.Base+pr.Size-1)
}

// Set parses a string of the form "value", "min-max", or "min+offset", inclusive at both ends, and
// sets the PortRange from it.  This is part of the flag.Value and pflag.Value
// interfaces.
func (pr *PortRange) Set(value string) error {
	const (
		SinglePortNotation = 1 << iota
		HyphenNotation
		PlusNotation
	)

	value = strings.TrimSpace(value)
	hyphenIndex := strings.Index(value, "-")
	plusIndex := strings.Index(value, "+")

	if value == "" {
		pr.Base = 0
		pr.Size = 0
		return nil
	}

	var err error
	var low, high int
	var notation int

	if plusIndex == -1 && hyphenIndex == -1 {
		notation |= SinglePortNotation
	}
	if hyphenIndex != -1 {
		notation |= HyphenNotation
	}
	if plusIndex != -1 {
		notation |= PlusNotation
	}

	switch notation {
	case SinglePortNotation:
		var port int
		port, err = strconv.Atoi(value)
		if err != nil {
			return err
		}
		low = port
		high = port
	case HyphenNotation:
		low, err = strconv.Atoi(value[:hyphenIndex])
		if err != nil {
			return err
		}
		high, err = strconv.Atoi(value[hyphenIndex+1:])
		if err != nil {
			return err
		}
	case PlusNotation:
		var offset int
		low, err = strconv.Atoi(value[:plusIndex])
		if err != nil {
			return err
		}
		offset, err = strconv.Atoi(value[plusIndex+1:])
		if err != nil {
			return err
		}
		high = low + offset
	default:
		return fmt.Errorf("unable to parse port range: %s", value)
	}

	if low > 65535 || high > 65535 {
		return fmt.Errorf("the port range cannot be greater than 65535: %s", value)
	}

	if high < low {
		return fmt.Errorf("end port cannot be less than start port: %s", value)
	}

	pr.Base = low
	pr.Size = 1 + high - low
	return nil
}

// Type returns a descriptive string about this type.  This is part of the
// pflag.Value interface.
func (*PortRange) Type() string {
	return "portRange"
}

// ParsePortRange parses a string of the form "min-max", inclusive at both
// ends, and initializs a new PortRange from it.
func ParsePortRange(value string) (*PortRange, error) {
	pr := &PortRange{}
	err := pr.Set(value)
	if err != nil {
		return nil, err
	}
	return pr, nil
}

func ParsePortRangeOrDie(value string) *PortRange {
	pr, err := ParsePortRange(value)
	if err != nil {
		panic(fmt.Sprintf("couldn't parse port range %q: %v", value, err))
	}
	return pr
}
<|endoftext|>


<commit_before>// Copyright 2015
// Author: huangjunwei@youmi.net

package blog4go

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strconv"
	"sync"
	"time"
)

// 各种日志结构接口
type LogWriter interface {
	// 关闭log writer的处理方法
	// 善后
	Close()

	// 用于内部写log的方法
	write(level Level, format string)
	writef(level Level, format string, args ...interface{})

	// 供用户强制刷日志到输出
	Flush()
}

var DefaultFileLogWriter *FileLogWriter = new(FileLogWriter)

const (
	// 好像buffer size 调大点benchmark效果更好
	DefaultBufferSize = 4096
)

// 装逼的logger
type FileLogWriter struct {
	level Level

	// log文件
	filename string
	file     *os.File
	writer   *bufio.Writer

	// TODO logrotate
	rotate bool

	lock   *sync.RWMutex
	closed bool
}

// 包初始化函数
func init() {
	timeCache.now = time.Now()
	timeCache.format = timeCache.now.Format("[2006/01/02:15:04:05]")
}

// 创建file writer
func NewFileLogWriter(filename string) (fileWriter *FileLogWriter, err error) {
	fileWriter = new(FileLogWriter)

	// 打开文件描述符
	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0644))
	if nil != err {
		return nil, err
	}
	fileWriter.filename = filename
	fileWriter.file = file
	fileWriter.writer = bufio.NewWriterSize(file, DefaultBufferSize)
	fileWriter.lock = new(sync.RWMutex)
	fileWriter.closed = false

	go fileWriter.daemon()

	return fileWriter, nil
}

func (self *FileLogWriter) Close() {
	self.lock.Lock()
	self.writer.Flush()
	self.file.Close()
	self.writer = nil
	self.closed = false
	self.lock.Unlock()
}

func (self *FileLogWriter) Flush() {
	self.writer.Flush()
}

func (self *FileLogWriter) SetLevel(level Level) *FileLogWriter {
	self.level = level
	return self
}

func (self *FileLogWriter) GetLevel() Level {
	return self.level
}

// 时间格式化的cache
type timeFormatCacheType struct {
	now    time.Time
	format string
}

var timeCache = timeFormatCacheType{}

// 常驻goroutine, 更新格式化的时间及logrotate
func (self *FileLogWriter) daemon() {
	t := time.Tick(1 * time.Second)
	for {
		select {
		case <-t:
			now := time.Now()
			timeCache.now = now
			timeCache.format = now.Format("[2006/01/02:15:04:05]")

		}
	}

}

func (self *FileLogWriter) write(level Level, format string, args ...interface{}) {
	if level < self.level {
		return
	}

	self.lock.RLock()
	defer self.lock.RUnlock()

	if self.closed {
		return
	}

	self.writer.WriteString(timeCache.format)
	self.writer.WriteString(" [")
	self.writer.WriteString(level.String())
	self.writer.WriteString("] ")
	self.writer.WriteString(format)
	self.writer.WriteString("\n")

	//self.writer.Flush()
}

func (self *FileLogWriter) writef(level Level, format string, args ...interface{}) (err error) {
	if level < self.level {
		return
	}
	// 格式化构造message
	// 使用 % 作占位符

	self.lock.RLock()
	defer self.lock.RUnlock()

	if self.closed {
		return
	}

	self.writer.WriteString(timeCache.format)
	self.writer.WriteString(" [")
	self.writer.WriteString(level.String())
	self.writer.WriteString("] ")

	// 识别占位符标记
	var tag bool = false
	// 转义字符标记
	var escape bool = false
	// 在处理的args 下标
	var n int = 0
	// 未输出的，第一个普通字符位置
	var last int = 0 //

	for i := 0; i < len(format); i++ {
		if tag {
			switch format[i] {
			// 类型检查/ 特殊字符处理
			// 占位符，有意义部分
			// 字符串
			case 's':
				if escape {
					escape = false
				}

				if str, ok := args[n].(string); ok {
					self.writer.WriteString(str)
					n++
					last = i + 1
				} else {
					return errors.New("Wrong format type.")
				}
				tag = false
			// 整型
			case 'd':
				if escape {
					escape = false
				}

				// 判断数据类型
				if number, ok := args[n].(int); ok {
					self.writer.WriteString(strconv.Itoa(number))
					n++
					last = i + 1
				} else {
					return errors.New("Wrong format type.")
				}
				tag = false
			// 浮点型
			// 暂时不支持
			case 'f':
				if escape {

					escape = false
				}

				if f, ok := args[n].(float64); ok {
					// 获取精确度
					prec, _ := strconv.ParseInt(format[i-1:i], 10, 0)
					// %.xf
					self.writer.WriteString(strconv.FormatFloat(f, 'f', int(prec), 64))
					n++
					last = i + 1
				} else {
					return errors.New("Wrong format type.")
				}
				tag = false

			//转义符
			case '\\':
				if escape {
					self.writer.WriteString("\\")
				}
				escape = !escape

			//默认
			default:

			}

		} else {
			// 占位符，百分号
			if '%' == format[i] && !escape {
				tag = true
				self.writer.WriteString(format[last:i])
			}
		}
	}
	self.writer.WriteString(format[last:])
	self.writer.WriteString("\n")
	//self.writer.Flush()

	return
}

func (self *FileLogWriter) Debug(format string) {
	self.write(DEBUG, format)
}

func (self *FileLogWriter) Debugf(format string, args ...interface{}) {
	self.writef(DEBUG, format, args...)
}
<commit_msg>bug fix<commit_after>// Copyright 2015
// Author: huangjunwei@youmi.net

package blog4go

import (
	"bufio"
	"errors"
	"os"
	"strconv"
	"sync"
	"time"
)

// 各种日志结构接口
type LogWriter interface {
	// 关闭log writer的处理方法
	// 善后
	Close()

	// 用于内部写log的方法
	write(level Level, format string)
	writef(level Level, format string, args ...interface{})

	// 供用户强制刷日志到输出
	Flush()
}

var DefaultFileLogWriter *FileLogWriter = new(FileLogWriter)

const (
	// 好像buffer size 调大点benchmark效果更好
	DefaultBufferSize = 4096
)

// 装逼的logger
type FileLogWriter struct {
	level Level

	// log文件
	filename string
	file     *os.File
	writer   *bufio.Writer

	// TODO logrotate
	rotate bool

	lock   *sync.RWMutex
	closed bool
}

// 时间格式化的cache
type timeFormatCacheType struct {
	now    time.Time
	format string
}

// 包初始化函数
func init() {
	timeCache.now = time.Now()
	timeCache.format = timeCache.now.Format("[2006/01/02:15:04:05]")
}

// 创建file writer
func NewFileLogWriter(filename string) (fileWriter *FileLogWriter, err error) {
	fileWriter = new(FileLogWriter)

	// 打开文件描述符
	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0644))
	if nil != err {
		return nil, err
	}
	fileWriter.filename = filename
	fileWriter.file = file
	fileWriter.writer = bufio.NewWriterSize(file, DefaultBufferSize)
	fileWriter.lock = new(sync.RWMutex)
	fileWriter.closed = false

	go fileWriter.daemon()

	return fileWriter, nil
}

func (self *FileLogWriter) Close() {
	self.lock.Lock()
	self.writer.Flush()
	self.file.Close()
	self.writer = nil
	self.closed = false
	self.lock.Unlock()
}

func (self *FileLogWriter) Flush() {
	self.writer.Flush()
}

func (self *FileLogWriter) SetLevel(level Level) *FileLogWriter {
	self.level = level
	return self
}

func (self *FileLogWriter) GetLevel() Level {
	return self.level
}

var timeCache = timeFormatCacheType{}

// 常驻goroutine, 更新格式化的时间及logrotate
func (self *FileLogWriter) daemon() {
	t := time.Tick(1 * time.Second)
	for {
		select {
		case <-t:
			now := time.Now()
			timeCache.now = now
			timeCache.format = now.Format("[2006/01/02:15:04:05]")

		}
	}

}

func (self *FileLogWriter) write(level Level, format string, args ...interface{}) {
	if level < self.level {
		return
	}

	self.lock.RLock()
	defer self.lock.RUnlock()

	if self.closed {
		return
	}

	self.writer.WriteString(timeCache.format)
	self.writer.WriteString(" [")
	self.writer.WriteString(level.String())
	self.writer.WriteString("] ")
	self.writer.WriteString(format)
	self.writer.WriteString("\n")

	//self.writer.Flush()
}

func (self *FileLogWriter) writef(level Level, format string, args ...interface{}) (err error) {
	if level < self.level {
		return
	}
	// 格式化构造message
	// 使用 % 作占位符

	self.lock.RLock()
	defer self.lock.RUnlock()

	if self.closed {
		return
	}

	self.writer.WriteString(timeCache.format)
	self.writer.WriteString(" [")
	self.writer.WriteString(level.String())
	self.writer.WriteString("] ")

	// 识别占位符标记
	var tag bool = false
	// 转义字符标记
	var escape bool = false
	// 在处理的args 下标
	var n int = 0
	// 未输出的，第一个普通字符位置
	var last int = 0 //

	for i := 0; i < len(format); i++ {
		if tag {
			switch format[i] {
			// 类型检查/ 特殊字符处理
			// 占位符，有意义部分
			// 字符串
			case 's':
				if escape {
					escape = false
				}

				if str, ok := args[n].(string); ok {
					self.writer.WriteString(str)
					n++
					last = i + 1
				} else {
					return errors.New("Wrong format type.")
				}
				tag = false
			// 整型
			case 'd':
				if escape {
					escape = false
				}

				// 判断数据类型
				if number, ok := args[n].(int); ok {
					self.writer.WriteString(strconv.Itoa(number))
					n++
					last = i + 1
				} else {
					return errors.New("Wrong format type.")
				}
				tag = false
			// 浮点型
			// 暂时不支持
			case 'f':
				if escape {

					escape = false
				}

				if f, ok := args[n].(float64); ok {
					// 获取精确度
					prec, _ := strconv.ParseInt(format[i-1:i], 10, 0)
					// %.xf
					self.writer.WriteString(strconv.FormatFloat(f, 'f', int(prec), 64))
					n++
					last = i + 1
				} else {
					return errors.New("Wrong format type.")
				}
				tag = false

			//转义符
			case '\\':
				if escape {
					self.writer.WriteString("\\")
				}
				escape = !escape

			//默认
			default:

			}

		} else {
			// 占位符，百分号
			if '%' == format[i] && !escape {
				tag = true
				self.writer.WriteString(format[last:i])
			}
		}
	}
	self.writer.WriteString(format[last:])
	self.writer.WriteString("\n")
	//self.writer.Flush()

	return
}

func (self *FileLogWriter) Debug(format string) {
	self.write(DEBUG, format)
}

func (self *FileLogWriter) Debugf(format string, args ...interface{}) {
	self.writef(DEBUG, format, args...)
}
<|endoftext|>


<commit_before>package redisc

import (
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/PuerkitoBio/juggler/internal/redistest"
	"github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
)

func TestClusterRefreshNormalServer(t *testing.T) {
	cmd, port := redistest.StartServer(t, nil, "")
	defer cmd.Process.Kill()

	c := &Cluster{
		StartupNodes: []string{":" + port},
	}
	err := c.Refresh()
	if assert.Error(t, err, "Refresh") {
		assert.Contains(t, err.Error(), "redisc: all nodes failed", "expected error message")
	}
}

func TestClusterRefresh(t *testing.T) {
	fn, ports := redistest.StartCluster(t, nil)
	defer fn()

	for i, p := range ports {
		ports[i] = ":" + p
	}
	c := &Cluster{
		StartupNodes: ports,
	}

	err := c.Refresh()
	if assert.NoError(t, err, "Refresh") {
		var prev string
		pix := -1
		for ix, master := range c.mapping {
			if master != prev || ix == len(c.mapping)-1 {
				prev = master
				t.Logf("%5d: %s\n", ix, master)
				pix++
			}
			if assert.NotEmpty(t, master) {
				split := strings.Index(master, ":")
				assert.Contains(t, ports, master[split:], "expected master")
			}
		}
	}
}

func TestClusterClose(t *testing.T) {
	c := &Cluster{
		StartupNodes: []string{":6379"},
		DialOptions:  []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)},
		CreatePool:   createPool,
	}
	assert.NoError(t, c.Close(), "Close")
	if err := c.Close(); assert.Error(t, err, "Close after Close") {
		assert.Contains(t, err.Error(), "redisc: closed", "expected message")
	}
	if conn := c.Get(); assert.Error(t, conn.Err(), "Get after Close") {
		assert.Contains(t, conn.Err().Error(), "redisc: closed", "expected message")
	}
	if _, err := c.Dial(); assert.Error(t, err, "Dial after Close") {
		assert.Contains(t, err.Error(), "redisc: closed", "expected message")
	}
	if err := c.Refresh(); assert.Error(t, err, "Refresh after Close") {
		assert.Contains(t, err.Error(), "redisc: closed", "expected message")
	}
}

func createPool(addr string, opts ...redis.DialOption) (*redis.Pool, error) {
	return &redis.Pool{
		MaxIdle:     5,
		MaxActive:   10,
		IdleTimeout: time.Minute,
		Dial: func() (redis.Conn, error) {
			return redis.Dial("tcp", addr, opts...)
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}, nil
}

type redisCmd struct {
	name   string
	args   redis.Args
	resp   interface{} // if resp is of type lenResult, asserts that there is a result at least this long
	errMsg string
}

type lenResult int

func TestCommands(t *testing.T) {
	fn, ports := redistest.StartCluster(t, nil)
	defer fn()

	cmdsPerGroup := map[string][]redisCmd{
		"cluster": {
			{"CLUSTER", redis.Args{"INFO"}, lenResult(10), ""},
			{"READONLY", nil, "OK", ""},
			{"READWRITE", nil, "OK", ""},
			{"CLUSTER", redis.Args{"COUNTKEYSINSLOT", 12345}, int64(0), ""},
			{"CLUSTER", redis.Args{"KEYSLOT", "a"}, int64(15495), ""},
			{"CLUSTER", redis.Args{"NODES"}, lenResult(100), ""},
		},
		"connection": {
			{"AUTH", redis.Args{"pwd"}, nil, "ERR Client sent AUTH, but no password is set"},
			{"ECHO", redis.Args{"a"}, []byte("a"), ""},
			{"PING", nil, "PONG", ""},
			{"SELECT", redis.Args{1}, nil, "ERR SELECT is not allowed in cluster mode"},
			{"QUIT", nil, "OK", ""},
		},
		"hashes": {
			{"HSET", redis.Args{"ha", "f1", "1"}, int64(1), ""},
			{"HLEN", redis.Args{"ha"}, int64(1), ""},
		},
	}

	for i, p := range ports {
		ports[i] = ":" + p
	}
	c := &Cluster{
		StartupNodes: ports,
		DialOptions:  []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)},
		CreatePool:   createPool,
	}

	var wg sync.WaitGroup
	wg.Add(len(cmdsPerGroup))
	for _, cmds := range cmdsPerGroup {
		go func(cmds []redisCmd) {
			defer wg.Done()
			runCommands(t, c, cmds)
		}(cmds)
	}
	wg.Wait()
}

func runCommands(t *testing.T, c *Cluster, cmds []redisCmd) {
	for _, cmd := range cmds {
		conn := c.Get()
		res, err := conn.Do(cmd.name, cmd.args...)
		if cmd.errMsg != "" {
			if assert.Error(t, err, cmd.name) {
				assert.Contains(t, err.Error(), cmd.errMsg, cmd.name)
			}
		} else {
			assert.NoError(t, err, cmd.name)
			if lr, ok := cmd.resp.(lenResult); ok {
				if assert.IsType(t, []byte(nil), res, "bytes result") {
					assert.True(t, len(res.([]byte)) >= int(lr), "result has at least %d bytes", lr)
				}
			} else {
				if !assert.Equal(t, cmd.resp, res, cmd.name) {
					t.Logf("%T vs %T", cmd.resp, res)
				}
			}
		}
	}
}
<commit_msg>juggler/internal/redistest: cleanup nodes config files on exit<commit_after>package redisc

import (
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/PuerkitoBio/juggler/internal/redistest"
	"github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestClusterRefreshNormalServer(t *testing.T) {
	cmd, port := redistest.StartServer(t, nil, "")
	defer cmd.Process.Kill()

	c := &Cluster{
		StartupNodes: []string{":" + port},
	}
	err := c.Refresh()
	if assert.Error(t, err, "Refresh") {
		assert.Contains(t, err.Error(), "redisc: all nodes failed", "expected error message")
	}
}

func TestClusterRefresh(t *testing.T) {
	fn, ports := redistest.StartCluster(t, nil)
	defer fn()

	for i, p := range ports {
		ports[i] = ":" + p
	}
	c := &Cluster{
		StartupNodes: ports,
	}

	err := c.Refresh()
	if assert.NoError(t, err, "Refresh") {
		var prev string
		pix := -1
		for ix, master := range c.mapping {
			if master != prev || ix == len(c.mapping)-1 {
				prev = master
				t.Logf("%5d: %s\n", ix, master)
				pix++
			}
			if assert.NotEmpty(t, master) {
				split := strings.Index(master, ":")
				assert.Contains(t, ports, master[split:], "expected master")
			}
		}
	}
}

func TestClusterClose(t *testing.T) {
	c := &Cluster{
		StartupNodes: []string{":6379"},
		DialOptions:  []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)},
		CreatePool:   createPool,
	}
	assert.NoError(t, c.Close(), "Close")
	if err := c.Close(); assert.Error(t, err, "Close after Close") {
		assert.Contains(t, err.Error(), "redisc: closed", "expected message")
	}
	if conn := c.Get(); assert.Error(t, conn.Err(), "Get after Close") {
		assert.Contains(t, conn.Err().Error(), "redisc: closed", "expected message")
	}
	if _, err := c.Dial(); assert.Error(t, err, "Dial after Close") {
		assert.Contains(t, err.Error(), "redisc: closed", "expected message")
	}
	if err := c.Refresh(); assert.Error(t, err, "Refresh after Close") {
		assert.Contains(t, err.Error(), "redisc: closed", "expected message")
	}
}

func createPool(addr string, opts ...redis.DialOption) (*redis.Pool, error) {
	return &redis.Pool{
		MaxIdle:     5,
		MaxActive:   10,
		IdleTimeout: time.Minute,
		Dial: func() (redis.Conn, error) {
			return redis.Dial("tcp", addr, opts...)
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}, nil
}

type redisCmd struct {
	name   string
	args   redis.Args
	resp   interface{} // if resp is of type lenResult, asserts that there is a result at least this long
	errMsg string
}

type lenResult int

func TestCommands(t *testing.T) {
	fn, ports := redistest.StartCluster(t, nil)
	defer fn()

	cmdsPerGroup := map[string][]redisCmd{
		"cluster": {
			{"CLUSTER", redis.Args{"INFO"}, lenResult(10), ""},
			{"READONLY", nil, "OK", ""},
			{"READWRITE", nil, "OK", ""},
			{"CLUSTER", redis.Args{"COUNTKEYSINSLOT", 12345}, int64(0), ""},
			{"CLUSTER", redis.Args{"KEYSLOT", "a"}, int64(15495), ""},
			{"CLUSTER", redis.Args{"NODES"}, lenResult(100), ""},
		},
		"connection": {
			{"AUTH", redis.Args{"pwd"}, nil, "ERR Client sent AUTH, but no password is set"},
			{"ECHO", redis.Args{"a"}, []byte("a"), ""},
			{"PING", nil, "PONG", ""},
			{"SELECT", redis.Args{1}, nil, "ERR SELECT is not allowed in cluster mode"},
			{"QUIT", nil, "OK", ""},
		},
		"hashes": {
			{"HSET", redis.Args{"ha", "f1", "1"}, int64(1), ""},
			{"HLEN", redis.Args{"ha"}, int64(1), ""},
		},
	}

	for i, p := range ports {
		ports[i] = ":" + p
	}
	c := &Cluster{
		StartupNodes: ports,
		DialOptions:  []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)},
		CreatePool:   createPool,
	}
	require.NoError(t, c.Refresh(), "Refresh")

	var wg sync.WaitGroup
	wg.Add(len(cmdsPerGroup))
	for _, cmds := range cmdsPerGroup {
		go func(cmds []redisCmd) {
			defer wg.Done()
			runCommands(t, c, cmds)
		}(cmds)
	}
	wg.Wait()
}

func runCommands(t *testing.T, c *Cluster, cmds []redisCmd) {
	for _, cmd := range cmds {
		conn := c.Get()
		res, err := conn.Do(cmd.name, cmd.args...)
		if cmd.errMsg != "" {
			if assert.Error(t, err, cmd.name) {
				assert.Contains(t, err.Error(), cmd.errMsg, cmd.name)
			}
		} else {
			assert.NoError(t, err, cmd.name)
			if lr, ok := cmd.resp.(lenResult); ok {
				if assert.IsType(t, []byte(nil), res, "bytes result") {
					assert.True(t, len(res.([]byte)) >= int(lr), "result has at least %d bytes", lr)
				}
			} else {
				if !assert.Equal(t, cmd.resp, res, cmd.name) {
					t.Logf("%T vs %T", cmd.resp, res)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package orchprovider

import (
	"os"
	"strings"
)

// Detect the Container Orchestrator based on ENV variables
func DetectOrchProviderFromEnv() string {

	_, ok := os.LookupEnv("KUBERNETES_SERVICE_HOST")
	if ok {
		return "KUBERNETES"
	}
	for _, e := range os.Environ() {
		ok := strings.Contains(e, "NOMAD_ADDR")
		if ok {
			return "NOMAD"
		}
	}
	return "Unknown"
}
<commit_msg>golint issue on detect.go<commit_after>package orchprovider

import (
	"os"
	"strings"
)

// DetectOrchProviderFromEnv detects the Container Orchestrator from ENV variables
func DetectOrchProviderFromEnv() string {

	_, ok := os.LookupEnv("KUBERNETES_SERVICE_HOST")
	if ok {
		return "KUBERNETES"
	}
	for _, e := range os.Environ() {
		ok := strings.Contains(e, "NOMAD_ADDR")
		if ok {
			return "NOMAD"
		}
	}
	return "Unknown"
}
<|endoftext|>


<commit_before>// Copyright 2020 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buf

import (
	"context"
	"time"

	"github.com/bufbuild/buf/internal/buf/bufcli"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/breaking"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/lint"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/lsbreakingcheckers"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/lslintcheckers"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/generate"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/image/build"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/image/convert"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/lsfiles"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/protoc"
	"github.com/bufbuild/buf/internal/pkg/app/appcmd"
	"github.com/bufbuild/buf/internal/pkg/app/appflag"
)

// Version is the version of buf.
const Version = "0.26.0-dev"

// Main is the main.
func Main(name string, options ...MainOption) {
	mainOptions := &mainOptions{
		moduleResolverReaderProvider: bufcli.NopModuleResolverReaderProvider{},
	}
	for _, option := range options {
		option(mainOptions)
	}
	appcmd.Main(
		context.Background(),
		newRootCommand(
			name,
			mainOptions.rootCommandModifier,
			mainOptions.moduleResolverReaderProvider,
		),
	)
}

// MainOption is an option for command construction.
type MainOption func(*mainOptions)

// WithModuleResolverAndReaderProvider returns a new MainOption that uses the given ModuleResolverReaderProvider.
func WithModuleResolverAndReaderProvider(moduleResolverReaderProvider bufcli.ModuleResolverReaderProvider) MainOption {
	return func(options *mainOptions) {
		options.moduleResolverReaderProvider = moduleResolverReaderProvider
	}
}

// WithRootCommandModifier returns a new MainOption that modifies the root Command.
func WithRootCommandModifier(rootCommandModifier func(*appcmd.Command, appflag.Builder, bufcli.ModuleResolverReaderProvider)) MainOption {
	return func(mainOptions *mainOptions) {
		mainOptions.rootCommandModifier = rootCommandModifier
	}
}

type mainOptions struct {
	rootCommandModifier          func(*appcmd.Command, appflag.Builder, bufcli.ModuleResolverReaderProvider)
	moduleResolverReaderProvider bufcli.ModuleResolverReaderProvider
}

func newRootCommand(
	name string,
	rootCommandModifier func(*appcmd.Command, appflag.Builder, bufcli.ModuleResolverReaderProvider),
	moduleResolverReaderProvider bufcli.ModuleResolverReaderProvider,
) *appcmd.Command {
	builder := appflag.NewBuilder(
		name,
		appflag.BuilderWithTimeout(120*time.Second),
		appflag.BuilderWithTracing(),
	)
	rootCommand := &appcmd.Command{
		Use: name,
		SubCommands: []*appcmd.Command{
			{
				Use:   "image",
				Short: "Work with Images and FileDescriptorSets.",
				SubCommands: []*appcmd.Command{
					build.NewCommand("build", builder, moduleResolverReaderProvider),
				},
			},
			{
				Use:   "check",
				Short: "Run lint or breaking change checks.",
				SubCommands: []*appcmd.Command{
					lint.NewCommand("lint", builder, moduleResolverReaderProvider),
					breaking.NewCommand("breaking", builder, moduleResolverReaderProvider),
					lslintcheckers.NewCommand("ls-lint-checkers", builder),
					lsbreakingcheckers.NewCommand("ls-breaking-checkers", builder),
				},
			},
			protoc.NewCommand("protoc", builder, moduleResolverReaderProvider),
			lsfiles.NewCommand("ls-files", builder, moduleResolverReaderProvider),
			{
				Use:   "beta",
				Short: "Beta commands. Unstable and will likely change.",
				SubCommands: []*appcmd.Command{
					generate.NewCommand("generate", builder, moduleResolverReaderProvider),
					{
						Use:   "image",
						Short: "Work with Images and FileDescriptorSets.",
						SubCommands: []*appcmd.Command{
							convert.NewCommand("convert", builder, ""),
						},
					},
				},
			},
			{
				Use:        "experimental",
				Short:      "Experimental commands. Unstable and will likely change.",
				Deprecated: `use "beta" instead.`,
				Hidden:     true,
				SubCommands: []*appcmd.Command{
					{
						Use:        "image",
						Short:      "Work with Images and FileDescriptorSets.",
						Deprecated: `use "beta image" instead.`,
						SubCommands: []*appcmd.Command{
							convert.NewCommand("convert", builder, `use "beta image convert" instead.`),
						},
					},
				},
			},
		},
		BindPersistentFlags: builder.BindRoot,
		Version:             Version,
	}
	if rootCommandModifier != nil {
		rootCommandModifier(rootCommand, builder, moduleResolverReaderProvider)
	}
	return rootCommand
}
<commit_msg>Update to v0.26.0<commit_after>// Copyright 2020 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buf

import (
	"context"
	"time"

	"github.com/bufbuild/buf/internal/buf/bufcli"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/breaking"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/lint"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/lsbreakingcheckers"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/check/lslintcheckers"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/generate"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/image/build"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/image/convert"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/lsfiles"
	"github.com/bufbuild/buf/internal/buf/cmd/buf/command/protoc"
	"github.com/bufbuild/buf/internal/pkg/app/appcmd"
	"github.com/bufbuild/buf/internal/pkg/app/appflag"
)

// Version is the version of buf.
const Version = "0.26.0"

// Main is the main.
func Main(name string, options ...MainOption) {
	mainOptions := &mainOptions{
		moduleResolverReaderProvider: bufcli.NopModuleResolverReaderProvider{},
	}
	for _, option := range options {
		option(mainOptions)
	}
	appcmd.Main(
		context.Background(),
		newRootCommand(
			name,
			mainOptions.rootCommandModifier,
			mainOptions.moduleResolverReaderProvider,
		),
	)
}

// MainOption is an option for command construction.
type MainOption func(*mainOptions)

// WithModuleResolverAndReaderProvider returns a new MainOption that uses the given ModuleResolverReaderProvider.
func WithModuleResolverAndReaderProvider(moduleResolverReaderProvider bufcli.ModuleResolverReaderProvider) MainOption {
	return func(options *mainOptions) {
		options.moduleResolverReaderProvider = moduleResolverReaderProvider
	}
}

// WithRootCommandModifier returns a new MainOption that modifies the root Command.
func WithRootCommandModifier(rootCommandModifier func(*appcmd.Command, appflag.Builder, bufcli.ModuleResolverReaderProvider)) MainOption {
	return func(mainOptions *mainOptions) {
		mainOptions.rootCommandModifier = rootCommandModifier
	}
}

type mainOptions struct {
	rootCommandModifier          func(*appcmd.Command, appflag.Builder, bufcli.ModuleResolverReaderProvider)
	moduleResolverReaderProvider bufcli.ModuleResolverReaderProvider
}

func newRootCommand(
	name string,
	rootCommandModifier func(*appcmd.Command, appflag.Builder, bufcli.ModuleResolverReaderProvider),
	moduleResolverReaderProvider bufcli.ModuleResolverReaderProvider,
) *appcmd.Command {
	builder := appflag.NewBuilder(
		name,
		appflag.BuilderWithTimeout(120*time.Second),
		appflag.BuilderWithTracing(),
	)
	rootCommand := &appcmd.Command{
		Use: name,
		SubCommands: []*appcmd.Command{
			{
				Use:   "image",
				Short: "Work with Images and FileDescriptorSets.",
				SubCommands: []*appcmd.Command{
					build.NewCommand("build", builder, moduleResolverReaderProvider),
				},
			},
			{
				Use:   "check",
				Short: "Run lint or breaking change checks.",
				SubCommands: []*appcmd.Command{
					lint.NewCommand("lint", builder, moduleResolverReaderProvider),
					breaking.NewCommand("breaking", builder, moduleResolverReaderProvider),
					lslintcheckers.NewCommand("ls-lint-checkers", builder),
					lsbreakingcheckers.NewCommand("ls-breaking-checkers", builder),
				},
			},
			protoc.NewCommand("protoc", builder, moduleResolverReaderProvider),
			lsfiles.NewCommand("ls-files", builder, moduleResolverReaderProvider),
			{
				Use:   "beta",
				Short: "Beta commands. Unstable and will likely change.",
				SubCommands: []*appcmd.Command{
					generate.NewCommand("generate", builder, moduleResolverReaderProvider),
					{
						Use:   "image",
						Short: "Work with Images and FileDescriptorSets.",
						SubCommands: []*appcmd.Command{
							convert.NewCommand("convert", builder, ""),
						},
					},
				},
			},
			{
				Use:        "experimental",
				Short:      "Experimental commands. Unstable and will likely change.",
				Deprecated: `use "beta" instead.`,
				Hidden:     true,
				SubCommands: []*appcmd.Command{
					{
						Use:        "image",
						Short:      "Work with Images and FileDescriptorSets.",
						Deprecated: `use "beta image" instead.`,
						SubCommands: []*appcmd.Command{
							convert.NewCommand("convert", builder, `use "beta image convert" instead.`),
						},
					},
				},
			},
		},
		BindPersistentFlags: builder.BindRoot,
		Version:             Version,
	}
	if rootCommandModifier != nil {
		rootCommandModifier(rootCommand, builder, moduleResolverReaderProvider)
	}
	return rootCommand
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gopathwalk is like filepath.Walk but specialized for finding Go
// packages, particularly in $GOPATH and $GOROOT.
package gopathwalk

import (
	"bufio"
	"bytes"
	"fmt"
	"go/build"
	"golang.org/x/tools/internal/fastwalk"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// Options controls the behavior of a Walk call.
type Options struct {
	Debug          bool // Enable debug logging
	ModulesEnabled bool // Search module caches. Also disables legacy goimports ignore rules.
}

// RootType indicates the type of a Root.
type RootType int

const (
	RootUnknown RootType = iota
	RootGOROOT
	RootGOPATH
	RootCurrentModule
	RootModuleCache
)

// A Root is a starting point for a Walk.
type Root struct {
	Path string
	Type RootType
}

// SrcDirsRoots returns the roots from build.Default.SrcDirs(). Not modules-compatible.
func SrcDirsRoots() []Root {
	var roots []Root
	roots = append(roots, Root{filepath.Join(build.Default.GOROOT, "src"), RootGOROOT})
	for _, p := range filepath.SplitList(build.Default.GOPATH) {
		roots = append(roots, Root{filepath.Join(p, "src"), RootGOPATH})
	}
	return roots
}

// Walk walks Go source directories ($GOROOT, $GOPATH, etc) to find packages.
// For each package found, add will be called (concurrently) with the absolute
// paths of the containing source directory and the package directory.
// add will be called concurrently.
func Walk(roots []Root, add func(root Root, dir string), opts Options) {
	for _, root := range roots {
		walkDir(root, add, opts)
	}
}

func walkDir(root Root, add func(Root, string), opts Options) {
	if _, err := os.Stat(root.Path); os.IsNotExist(err) {
		if opts.Debug {
			log.Printf("skipping nonexistant directory: %v", root.Path)
		}
		return
	}
	if opts.Debug {
		log.Printf("scanning %s", root.Path)
	}
	w := &walker{
		root: root,
		add:  add,
		opts: opts,
	}
	w.init()
	if err := fastwalk.Walk(root.Path, w.walk); err != nil {
		log.Printf("gopathwalk: scanning directory %v: %v", root.Path, err)
	}

	if opts.Debug {
		log.Printf("scanned %s", root.Path)
	}
}

// walker is the callback for fastwalk.Walk.
type walker struct {
	root Root               // The source directory to scan.
	add  func(Root, string) // The callback that will be invoked for every possible Go package dir.
	opts Options            // Options passed to Walk by the user.

	ignoredDirs []os.FileInfo // The ignored directories, loaded from .goimportsignore files.
}

// init initializes the walker based on its Options.
func (w *walker) init() {
	var ignoredPaths []string
	if w.root.Type == RootModuleCache {
		ignoredPaths = []string{"cache"}
	}
	if !w.opts.ModulesEnabled && w.root.Type == RootGOPATH {
		ignoredPaths = w.getIgnoredDirs(w.root.Path)
		ignoredPaths = append(ignoredPaths, "v", "mod")
	}

	for _, p := range ignoredPaths {
		full := filepath.Join(w.root.Path, p)
		if fi, err := os.Stat(full); err == nil {
			w.ignoredDirs = append(w.ignoredDirs, fi)
			if w.opts.Debug {
				log.Printf("Directory added to ignore list: %s", full)
			}
		} else if w.opts.Debug {
			log.Printf("Error statting ignored directory: %v", err)
		}
	}
}

// getIgnoredDirs reads an optional config file at <path>/.goimportsignore
// of relative directories to ignore when scanning for go files.
// The provided path is one of the $GOPATH entries with "src" appended.
func (w *walker) getIgnoredDirs(path string) []string {
	file := filepath.Join(path, ".goimportsignore")
	slurp, err := ioutil.ReadFile(file)
	if w.opts.Debug {
		if err != nil {
			log.Print(err)
		} else {
			log.Printf("Read %s", file)
		}
	}
	if err != nil {
		return nil
	}

	var ignoredDirs []string
	bs := bufio.NewScanner(bytes.NewReader(slurp))
	for bs.Scan() {
		line := strings.TrimSpace(bs.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		ignoredDirs = append(ignoredDirs, line)
	}
	return ignoredDirs
}

func (w *walker) shouldSkipDir(fi os.FileInfo) bool {
	for _, ignoredDir := range w.ignoredDirs {
		if os.SameFile(fi, ignoredDir) {
			return true
		}
	}
	return false
}

func (w *walker) walk(path string, typ os.FileMode) error {
	dir := filepath.Dir(path)
	if typ.IsRegular() {
		if dir == w.root.Path {
			// Doesn't make sense to have regular files
			// directly in your $GOPATH/src or $GOROOT/src.
			return fastwalk.SkipFiles
		}
		if !strings.HasSuffix(path, ".go") {
			return nil
		}

		w.add(w.root, dir)
		return fastwalk.SkipFiles
	}
	if typ == os.ModeDir {
		base := filepath.Base(path)
		if base == "" || base[0] == '.' || base[0] == '_' ||
			base == "testdata" || (!w.opts.ModulesEnabled && base == "node_modules") {
			return filepath.SkipDir
		}
		fi, err := os.Lstat(path)
		if err == nil && w.shouldSkipDir(fi) {
			return filepath.SkipDir
		}
		return nil
	}
	if typ == os.ModeSymlink {
		base := filepath.Base(path)
		if strings.HasPrefix(base, ".#") {
			// Emacs noise.
			return nil
		}
		fi, err := os.Lstat(path)
		if err != nil {
			// Just ignore it.
			return nil
		}
		if w.shouldTraverse(dir, fi) {
			return fastwalk.TraverseLink
		}
	}
	return nil
}

// shouldTraverse reports whether the symlink fi, found in dir,
// should be followed.  It makes sure symlinks were never visited
// before to avoid symlink loops.
func (w *walker) shouldTraverse(dir string, fi os.FileInfo) bool {
	path := filepath.Join(dir, fi.Name())
	target, err := filepath.EvalSymlinks(path)
	if err != nil {
		return false
	}
	ts, err := os.Stat(target)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return false
	}
	if !ts.IsDir() {
		return false
	}
	if w.shouldSkipDir(ts) {
		return false
	}
	// Check for symlink loops by statting each directory component
	// and seeing if any are the same file as ts.
	for {
		parent := filepath.Dir(path)
		if parent == path {
			// Made it to the root without seeing a cycle.
			// Use this symlink.
			return true
		}
		parentInfo, err := os.Stat(parent)
		if err != nil {
			return false
		}
		if os.SameFile(ts, parentInfo) {
			// Cycle. Don't traverse.
			return false
		}
		path = parent
	}

}
<commit_msg>internal/gopathwalk: ignore $GOROOT/.../vendor/ in module mode<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gopathwalk is like filepath.Walk but specialized for finding Go
// packages, particularly in $GOPATH and $GOROOT.
package gopathwalk

import (
	"bufio"
	"bytes"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"golang.org/x/tools/internal/fastwalk"
)

// Options controls the behavior of a Walk call.
type Options struct {
	Debug          bool // Enable debug logging
	ModulesEnabled bool // Search module caches. Also disables legacy goimports ignore rules.
}

// RootType indicates the type of a Root.
type RootType int

const (
	RootUnknown RootType = iota
	RootGOROOT
	RootGOPATH
	RootCurrentModule
	RootModuleCache
)

// A Root is a starting point for a Walk.
type Root struct {
	Path string
	Type RootType
}

// SrcDirsRoots returns the roots from build.Default.SrcDirs(). Not modules-compatible.
func SrcDirsRoots() []Root {
	var roots []Root
	roots = append(roots, Root{filepath.Join(build.Default.GOROOT, "src"), RootGOROOT})
	for _, p := range filepath.SplitList(build.Default.GOPATH) {
		roots = append(roots, Root{filepath.Join(p, "src"), RootGOPATH})
	}
	return roots
}

// Walk walks Go source directories ($GOROOT, $GOPATH, etc) to find packages.
// For each package found, add will be called (concurrently) with the absolute
// paths of the containing source directory and the package directory.
// add will be called concurrently.
func Walk(roots []Root, add func(root Root, dir string), opts Options) {
	for _, root := range roots {
		walkDir(root, add, opts)
	}
}

func walkDir(root Root, add func(Root, string), opts Options) {
	if _, err := os.Stat(root.Path); os.IsNotExist(err) {
		if opts.Debug {
			log.Printf("skipping nonexistant directory: %v", root.Path)
		}
		return
	}
	if opts.Debug {
		log.Printf("scanning %s", root.Path)
	}
	w := &walker{
		root: root,
		add:  add,
		opts: opts,
	}
	w.init()
	if err := fastwalk.Walk(root.Path, w.walk); err != nil {
		log.Printf("gopathwalk: scanning directory %v: %v", root.Path, err)
	}

	if opts.Debug {
		log.Printf("scanned %s", root.Path)
	}
}

// walker is the callback for fastwalk.Walk.
type walker struct {
	root Root               // The source directory to scan.
	add  func(Root, string) // The callback that will be invoked for every possible Go package dir.
	opts Options            // Options passed to Walk by the user.

	ignoredDirs []os.FileInfo // The ignored directories, loaded from .goimportsignore files.
}

// init initializes the walker based on its Options.
func (w *walker) init() {
	var ignoredPaths []string
	if w.root.Type == RootModuleCache {
		ignoredPaths = []string{"cache"}
	}
	if !w.opts.ModulesEnabled && w.root.Type == RootGOPATH {
		ignoredPaths = w.getIgnoredDirs(w.root.Path)
		ignoredPaths = append(ignoredPaths, "v", "mod")
	}

	for _, p := range ignoredPaths {
		full := filepath.Join(w.root.Path, p)
		if fi, err := os.Stat(full); err == nil {
			w.ignoredDirs = append(w.ignoredDirs, fi)
			if w.opts.Debug {
				log.Printf("Directory added to ignore list: %s", full)
			}
		} else if w.opts.Debug {
			log.Printf("Error statting ignored directory: %v", err)
		}
	}
}

// getIgnoredDirs reads an optional config file at <path>/.goimportsignore
// of relative directories to ignore when scanning for go files.
// The provided path is one of the $GOPATH entries with "src" appended.
func (w *walker) getIgnoredDirs(path string) []string {
	file := filepath.Join(path, ".goimportsignore")
	slurp, err := ioutil.ReadFile(file)
	if w.opts.Debug {
		if err != nil {
			log.Print(err)
		} else {
			log.Printf("Read %s", file)
		}
	}
	if err != nil {
		return nil
	}

	var ignoredDirs []string
	bs := bufio.NewScanner(bytes.NewReader(slurp))
	for bs.Scan() {
		line := strings.TrimSpace(bs.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		ignoredDirs = append(ignoredDirs, line)
	}
	return ignoredDirs
}

func (w *walker) shouldSkipDir(fi os.FileInfo) bool {
	for _, ignoredDir := range w.ignoredDirs {
		if os.SameFile(fi, ignoredDir) {
			return true
		}
	}
	return false
}

func (w *walker) walk(path string, typ os.FileMode) error {
	dir := filepath.Dir(path)
	if typ.IsRegular() {
		if dir == w.root.Path {
			// Doesn't make sense to have regular files
			// directly in your $GOPATH/src or $GOROOT/src.
			return fastwalk.SkipFiles
		}
		if !strings.HasSuffix(path, ".go") {
			return nil
		}

		w.add(w.root, dir)
		return fastwalk.SkipFiles
	}
	if typ == os.ModeDir {
		base := filepath.Base(path)
		if base == "" || base[0] == '.' || base[0] == '_' ||
			base == "testdata" ||
			(w.root.Type == RootGOROOT && w.opts.ModulesEnabled && base == "vendor") ||
			(!w.opts.ModulesEnabled && base == "node_modules") {
			return filepath.SkipDir
		}
		fi, err := os.Lstat(path)
		if err == nil && w.shouldSkipDir(fi) {
			return filepath.SkipDir
		}
		return nil
	}
	if typ == os.ModeSymlink {
		base := filepath.Base(path)
		if strings.HasPrefix(base, ".#") {
			// Emacs noise.
			return nil
		}
		fi, err := os.Lstat(path)
		if err != nil {
			// Just ignore it.
			return nil
		}
		if w.shouldTraverse(dir, fi) {
			return fastwalk.TraverseLink
		}
	}
	return nil
}

// shouldTraverse reports whether the symlink fi, found in dir,
// should be followed.  It makes sure symlinks were never visited
// before to avoid symlink loops.
func (w *walker) shouldTraverse(dir string, fi os.FileInfo) bool {
	path := filepath.Join(dir, fi.Name())
	target, err := filepath.EvalSymlinks(path)
	if err != nil {
		return false
	}
	ts, err := os.Stat(target)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return false
	}
	if !ts.IsDir() {
		return false
	}
	if w.shouldSkipDir(ts) {
		return false
	}
	// Check for symlink loops by statting each directory component
	// and seeing if any are the same file as ts.
	for {
		parent := filepath.Dir(path)
		if parent == path {
			// Made it to the root without seeing a cycle.
			// Use this symlink.
			return true
		}
		parentInfo, err := os.Stat(parent)
		if err != nil {
			return false
		}
		if os.SameFile(ts, parentInfo) {
			// Cycle. Don't traverse.
			return false
		}
		path = parent
	}

}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	_ "net/http/pprof"

	"github.com/dtylman/gowd"
	"github.com/dtylman/pictures/cmd/app/view"
	"github.com/dtylman/pictures/conf"
	"github.com/dtylman/pictures/indexer/db"

	"log"

	"runtime"

	lumberjack "gopkg.in/natefinch/lumberjack.v2"
)

func init() {
	go func() {
		http.ListenAndServe("localhost:6060", nil)
	}()
	log.SetOutput(&lumberjack.Logger{
		Filename:   "bome.log",
		MaxSize:    1, // megabytes
		MaxBackups: 1,
		MaxAge:     7, //days
	})
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func run() error {
	err := conf.Load()
	if err != nil {
		return err
	}
	err = db.Open()
	if err != nil {
		return err
	}
	defer db.Close()
	view.InitializeComponents()
	err = gowd.Run(view.Root.Element)
	if err != nil {
		return err
	}
	return err
}

func main() {
	err := run()
	if err != nil {
		gowd.Error(err)
	}
}
<commit_msg>close db when panic or signal.<commit_after>package main

import (
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"

	"github.com/dtylman/gowd"
	"github.com/dtylman/pictures/cmd/app/view"
	"github.com/dtylman/pictures/conf"
	"github.com/dtylman/pictures/indexer/db"

	"log"

	"runtime"

	lumberjack "gopkg.in/natefinch/lumberjack.v2"
)

func init() {
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt, os.Kill)
	go func() {
		for _ = range signalChan {
			db.Close()
		}
	}()

	go func() {
		http.ListenAndServe("localhost:6060", nil)
	}()

	log.SetOutput(&lumberjack.Logger{
		Filename:   "bome.log",
		MaxSize:    1, // megabytes
		MaxBackups: 1,
		MaxAge:     7, //days
	})
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func run() error {
	err := conf.Load()
	if err != nil {
		return err
	}
	err = db.Open()
	if err != nil {
		return err
	}
	defer func() {
		r := recover()
		if r != nil {
			log.Printf("Recovered from %v", r)
		}
		db.Close()
	}()
	view.InitializeComponents()
	err = gowd.Run(view.Root.Element)
	if err != nil {
		return err
	}
	return err
}

func main() {
	err := run()
	if err != nil {
		gowd.Error(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphics

import (
	"strings"

	"github.com/hajimehoshi/ebiten/internal/web"
)

type shaderId int

const (
	shaderVertexModelview shaderId = iota
	shaderFragmentTexture
)

func shader(id shaderId) string {
	s := shaders[id]
	defs := []string{}
	if web.IsMobileBrowser() {
		defs = append(defs, "#define IS_MOBILE_BROWSER")
	}
	s = strings.Replace(s, "{{Definitions}}", strings.Join(defs, "\n"), -1)
	return s
}

var shaders = map[shaderId]string{
	shaderVertexModelview: `
{{Definitions}}

uniform mat4 projection_matrix;
attribute vec2 vertex;
attribute vec4 tex_coord;
attribute vec4 geo_matrix_body;
attribute vec2 geo_matrix_translation;
varying vec2 varying_tex_coord;
varying vec2 varying_tex_coord_min;
varying vec2 varying_tex_coord_max;

void main(void) {
  varying_tex_coord = vec2(tex_coord[0], tex_coord[1]);
  varying_tex_coord_min =
    vec2(min(tex_coord[0], tex_coord[2]), min(tex_coord[1], tex_coord[3]));
  varying_tex_coord_max =
    vec2(max(tex_coord[0], tex_coord[2]), max(tex_coord[1], tex_coord[3]));
  mat4 geo_matrix = mat4(
    vec4(geo_matrix_body[0], geo_matrix_body[2], 0, 0),
    vec4(geo_matrix_body[1], geo_matrix_body[3], 0, 0),
    vec4(0, 0, 1, 0),
    vec4(geo_matrix_translation, 0, 1)
  );
  gl_Position = projection_matrix * geo_matrix * vec4(vertex, 0, 1);
}
`,
	shaderFragmentTexture: `
#if defined(GL_ES)
precision mediump float;
#else
#define lowp
#define mediump
#define highp
#endif

{{Definitions}}

uniform sampler2D texture;
uniform mat4 color_matrix;
uniform vec4 color_matrix_translation;
uniform vec2 source_size;
uniform int filter_type;

varying vec2 varying_tex_coord;
varying vec2 varying_tex_coord_min;
varying vec2 varying_tex_coord_max;

vec2 roundTexel(vec2 p) {
#if defined(IS_MOBILE_BROWSER)
  // This is a temporary hack: on mobile browsers like Android Chrome,
  // the code to round a float might cause unexpected behavior.
  // This is confirmed with rendering a relatively large (e.g. 4096x512) image
  // with specifying a source rectangle near its edges.
  // TODO: examples/moire doesn't work anyway.
  // TODO: Does this function work on an Android native app?
  return p;
#else
  vec2 factor = 1.0 / (source_size * 256.0);
  if (factor.x > 0.0) {
    p.x -= mod(p.x + factor.x * 0.5, factor.x) - factor.x * 0.5;
  }
  if (factor.y > 0.0) {
    p.y -= mod(p.y + factor.y * 0.5, factor.y) - factor.y * 0.5;
  }
  return p;
#endif
}

vec4 getColorAt(vec2 pos) {
  if (pos.x < varying_tex_coord_min.x ||
      pos.y < varying_tex_coord_min.y ||
      varying_tex_coord_max.x <= pos.x ||
      varying_tex_coord_max.y <= pos.y) {
    return vec4(0, 0, 0, 0);
  }
  return texture2D(texture, pos);
}

void main(void) {
  vec4 color = vec4(0, 0, 0, 0);

  vec2 pos = roundTexel(varying_tex_coord);
  if (filter_type == 1) {
    // Nearest neighbor
    color = getColorAt(pos);
  } else if (filter_type == 2) {
    // Bi-linear
    vec2 texel_size = 1.0 / source_size;
    pos -= texel_size * 0.5;
    vec4 c0 = getColorAt(pos);
    vec4 c1 = getColorAt(pos + vec2(texel_size.x, 0));
    vec4 c2 = getColorAt(pos + vec2(0, texel_size.y));
    vec4 c3 = getColorAt(pos + texel_size);
    float rateX = fract(pos.x * source_size.x);
    float rateY = fract(pos.y * source_size.y);
    color = mix(mix(c0, c1, rateX), mix(c2, c3, rateX), rateY);
  } else {
    color = vec4(1, 0, 0, 1);
  }

  // Un-premultiply alpha
  if (0.0 < color.a) {
    color.rgb /= color.a;
  }
  // Apply the color matrix
  color = (color_matrix * color) + color_matrix_translation;
  color = clamp(color, 0.0, 1.0);
  // Premultiply alpha
  color.rgb *= color.a;

  gl_FragColor = color;
}
`,
}
<commit_msg>graphics: texel should be represented as highp for precision<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphics

type shaderID int

const (
	shaderVertexModelview shaderID = iota
	shaderFragmentTexture
)

func shader(id shaderID) string {
	return shaders[id]
}

var shaders = map[shaderID]string{
	shaderVertexModelview: `
uniform mat4 projection_matrix;
attribute vec2 vertex;
attribute vec4 tex_coord;
attribute vec4 geo_matrix_body;
attribute vec2 geo_matrix_translation;
varying vec2 varying_tex_coord;
varying vec2 varying_tex_coord_min;
varying vec2 varying_tex_coord_max;

void main(void) {
  varying_tex_coord = vec2(tex_coord[0], tex_coord[1]);
  varying_tex_coord_min =
    vec2(min(tex_coord[0], tex_coord[2]), min(tex_coord[1], tex_coord[3]));
  varying_tex_coord_max =
    vec2(max(tex_coord[0], tex_coord[2]), max(tex_coord[1], tex_coord[3]));
  mat4 geo_matrix = mat4(
    vec4(geo_matrix_body[0], geo_matrix_body[2], 0, 0),
    vec4(geo_matrix_body[1], geo_matrix_body[3], 0, 0),
    vec4(0, 0, 1, 0),
    vec4(geo_matrix_translation, 0, 1)
  );
  gl_Position = projection_matrix * geo_matrix * vec4(vertex, 0, 1);
}
`,
	shaderFragmentTexture: `
#if defined(GL_ES)
precision mediump float;
#else
#define lowp
#define mediump
#define highp
#endif

uniform sampler2D texture;
uniform mat4 color_matrix;
uniform vec4 color_matrix_translation;
uniform vec2 source_size;
uniform int filter_type;

varying highp vec2 varying_tex_coord;
varying highp vec2 varying_tex_coord_min;
varying highp vec2 varying_tex_coord_max;

highp vec2 roundTexel(highp vec2 p) {
  // highp (relative) precision is 2^(-16) in the spec.
  // As the maximum source size is 4096, the minimum value for a denominator is
  // 65536 (= 4096 * 16).
  highp vec2 factor = 1.0 / (source_size * 16.0);
  if (factor.x * 0.5 > 0.0 && factor.y * 0.5 > 0.0) {
    p.x -= mod(p.x + factor.x * 0.5, factor.x) - factor.x * 0.5;
    p.y -= mod(p.y + factor.y * 0.5, factor.y) - factor.y * 0.5;
  }
  return p;
}

vec4 getColorAt(highp vec2 pos) {
  if (pos.x < varying_tex_coord_min.x ||
      pos.y < varying_tex_coord_min.y ||
      varying_tex_coord_max.x <= pos.x ||
      varying_tex_coord_max.y <= pos.y) {
    return vec4(0, 0, 0, 0);
  }
  return texture2D(texture, pos);
}

void main(void) {
  vec4 color = vec4(0, 0, 0, 0);

  vec2 pos = roundTexel(varying_tex_coord);
  if (filter_type == 1) {
    // Nearest neighbor
    color = getColorAt(pos);
  } else if (filter_type == 2) {
    // Bi-linear
    highp vec2 texel_size = 1.0 / source_size;
    pos -= texel_size * 0.5;
    vec4 c0 = getColorAt(pos);
    vec4 c1 = getColorAt(pos + vec2(texel_size.x, 0));
    vec4 c2 = getColorAt(pos + vec2(0, texel_size.y));
    vec4 c3 = getColorAt(pos + texel_size);
    float rateX = fract(pos.x * source_size.x);
    float rateY = fract(pos.y * source_size.y);
    color = mix(mix(c0, c1, rateX), mix(c2, c3, rateX), rateY);
  } else {
    color = vec4(1, 0, 0, 1);
  }

  // Un-premultiply alpha
  if (0.0 < color.a) {
    color.rgb /= color.a;
  }
  // Apply the color matrix
  color = (color_matrix * color) + color_matrix_translation;
  color = clamp(color, 0.0, 1.0);
  // Premultiply alpha
  color.rgb *= color.a;

  gl_FragColor = color;
}
`,
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The fetch command runs a server that fetches modules from a proxy and writes
// them to the discovery database.
package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	cloudtasks "cloud.google.com/go/cloudtasks/apiv2"
	"cloud.google.com/go/errorreporting"
	"cloud.google.com/go/profiler"
	"github.com/go-redis/redis/v7"
	"golang.org/x/discovery/internal/config"
	"golang.org/x/discovery/internal/database"
	"golang.org/x/discovery/internal/dcensus"
	"golang.org/x/discovery/internal/etl"
	"golang.org/x/discovery/internal/index"
	"golang.org/x/discovery/internal/queue"

	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/middleware"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/proxy"

	"contrib.go.opencensus.io/integrations/ocsql"
)

var (
	timeout    = config.GetEnv("GO_DISCOVERY_ETL_TIMEOUT_MINUTES", "10")
	queueName  = config.GetEnv("GO_DISCOVERY_ETL_TASK_QUEUE", "dev-fetch-tasks")
	workers    = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
	staticPath = flag.String("static", "content/static", "path to folder containing static files served")
)

func main() {
	flag.Parse()

	ctx := context.Background()

	cfg, err := config.Init(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	cfg.Dump(os.Stderr)

	if cfg.UseProfiler {
		if err := profiler.Start(profiler.Config{}); err != nil {
			log.Fatalf(ctx, "profiler.Start: %v", err)
		}
	}

	readProxyRemoved(ctx)

	// Wrap the postgres driver with OpenCensus instrumentation.
	driverName, err := ocsql.Register("postgres", ocsql.WithAllTraceOptions())
	if err != nil {
		log.Fatalf(ctx, "unable to register the ocsql driver: %v\n", err)
	}
	ddb, err := database.Open(driverName, cfg.DBConnInfo())
	if err != nil {
		log.Fatalf(ctx, "database.Open: %v", err)
	}
	db := postgres.New(ddb)
	defer db.Close()

	populateExcluded(ctx, db)

	indexClient, err := index.New(cfg.IndexURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	proxyClient, err := proxy.New(cfg.ProxyURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	fetchQueue := newQueue(ctx, cfg, proxyClient, db)
	reportingClient := reportingClient(ctx, cfg)
	redisClient := getRedis(ctx, cfg)
	server, err := etl.NewServer(cfg, db, indexClient, proxyClient, redisClient, fetchQueue, reportingClient, *staticPath)
	if err != nil {
		log.Fatal(ctx, err)
	}
	router := dcensus.NewRouter(nil)
	server.Install(router.Handle)

	views := append(dcensus.ClientViews, dcensus.ServerViews...)
	if err := dcensus.Init(cfg, views...); err != nil {
		log.Fatal(ctx, err)
	}
	// We are not currently forwarding any ports on AppEngine, so serving debug
	// information is broken.
	if !cfg.OnAppEngine() {
		dcensusServer, err := dcensus.NewServer()
		if err != nil {
			log.Fatal(ctx, err)
		}
		go http.ListenAndServe(cfg.DebugAddr("localhost:8001"), dcensusServer)
	}

	handlerTimeout, err := strconv.Atoi(timeout)
	if err != nil {
		log.Fatalf(ctx, "strconv.Atoi(%q): %v", timeout, err)
	}
	requestLogger := logger(ctx, cfg)
	mw := middleware.Chain(
		middleware.RequestLog(requestLogger),
		middleware.Timeout(time.Duration(handlerTimeout)*time.Minute),
	)
	http.Handle("/", mw(router))

	addr := cfg.HostAddr("localhost:8000")
	log.Infof(ctx, "Listening on addr %s", addr)
	log.Fatal(ctx, http.ListenAndServe(addr, nil))
}

func newQueue(ctx context.Context, cfg *config.Config, proxyClient *proxy.Client, db *postgres.DB) queue.Queue {
	if !cfg.OnAppEngine() {
		return queue.NewInMemory(ctx, proxyClient, db, *workers, etl.FetchAndUpdateState)
	}
	client, err := cloudtasks.NewClient(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	return queue.NewGCP(cfg, client, queueName)
}

func getRedis(ctx context.Context, cfg *config.Config) *redis.Client {
	var dialTimeout time.Duration
	if dl, ok := ctx.Deadline(); ok {
		dialTimeout = time.Until(dl)
	}
	if cfg.RedisHAHost != "" {
		return redis.NewClient(&redis.Options{
			Addr:        cfg.RedisHAHost + ":" + cfg.RedisHAPort,
			DialTimeout: dialTimeout,
			// We update completions with one big pipeline, so we need long write
			// timeouts. ReadTimeout is increased only to be consistent with
			// WriteTimeout.
			WriteTimeout: 5 * time.Minute,
			ReadTimeout:  5 * time.Minute,
		})
	}
	return nil
}

func reportingClient(ctx context.Context, cfg *config.Config) *errorreporting.Client {
	if !cfg.OnAppEngine() {
		return nil
	}
	reporter, err := errorreporting.NewClient(ctx, cfg.ProjectID, errorreporting.Config{
		ServiceName: cfg.ServiceID,
		OnError: func(err error) {
			log.Errorf(ctx, "Error reporting failed: %v", err)
		},
	})
	if err != nil {
		log.Fatal(ctx, err)
	}
	return reporter
}

func logger(ctx context.Context, cfg *config.Config) middleware.Logger {
	if cfg.OnAppEngine() {
		logger, err := log.UseStackdriver(ctx, cfg, "etl-log")
		if err != nil {
			log.Fatal(ctx, err)
		}
		return logger
	}
	return middleware.LocalLogger{}
}

// Read a file of module versions that we should ignore because
// the are in the index but not stored in the proxy.
// Format of the file: each line is
//     module@version
func readProxyRemoved(ctx context.Context) {
	filename := config.GetEnv("GO_DISCOVERY_PROXY_REMOVED", "")
	if filename == "" {
		return
	}
	lines, err := readFileLines(filename)
	if err != nil {
		log.Fatal(ctx, err)
	}
	for _, line := range lines {
		etl.ProxyRemoved[line] = true
	}
	log.Infof(ctx, "read %d excluded module versions from %s", len(etl.ProxyRemoved), filename)
}

// populateExcluded adds each element of excludedPrefixes to the excluded_prefixes
// table if it isn't already present.
func populateExcluded(ctx context.Context, db *postgres.DB) {
	const excludedFilename = "excluded.txt"
	lines, err := readFileLines(excludedFilename)
	if err != nil {
		log.Fatal(ctx, err)
	}
	user := os.Getenv("USER")
	if user == "" {
		user = "etl"
	}
	for _, line := range lines {
		var prefix, reason string
		i := strings.IndexAny(line, " \t")
		if i >= 0 {
			prefix = line[:i]
			reason = strings.TrimSpace(line[i+1:])
		}
		if reason == "" {
			log.Fatalf(ctx, "missing reason in %s, line %q", excludedFilename, line)
		}
		fmt.Printf("#### %q ->\n", line)
		fmt.Printf(" prefix=%q, reason=%q\n", prefix, reason)
		present, err := db.IsExcluded(ctx, prefix)
		if err != nil {
			log.Fatalf(ctx, "db.IsExcluded(%q): %v", prefix, err)
		}
		if !present {
			if err := db.InsertExcludedPrefix(ctx, prefix, user, reason); err != nil {
				log.Fatalf(ctx, "db.InsertExcludedPrefix(%q, %q, %q): %v", prefix, user, reason, err)
			}
		}
	}
}

// readFileLines reads filename and returns its lines, trimmed of whitespace.
// Blank lines and lines whose first non-blank character is '#' are omitted.
func readFileLines(filename string) ([]string, error) {
	var lines []string
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		line := strings.TrimSpace(scan.Text())
		if line == "" || line[0] == '#' {
			continue
		}
		lines = append(lines, line)
	}
	if err := scan.Err(); err != nil {
		return nil, err
	}
	return lines, nil
}
<commit_msg>cmd/etl: fix path to excluded.txt<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The fetch command runs a server that fetches modules from a proxy and writes
// them to the discovery database.
package main

import (
	"bufio"
	"context"
	"flag"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	cloudtasks "cloud.google.com/go/cloudtasks/apiv2"
	"cloud.google.com/go/errorreporting"
	"cloud.google.com/go/profiler"
	"github.com/go-redis/redis/v7"
	"golang.org/x/discovery/internal/config"
	"golang.org/x/discovery/internal/database"
	"golang.org/x/discovery/internal/dcensus"
	"golang.org/x/discovery/internal/etl"
	"golang.org/x/discovery/internal/index"
	"golang.org/x/discovery/internal/queue"

	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/middleware"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/proxy"

	"contrib.go.opencensus.io/integrations/ocsql"
)

var (
	timeout    = config.GetEnv("GO_DISCOVERY_ETL_TIMEOUT_MINUTES", "10")
	queueName  = config.GetEnv("GO_DISCOVERY_ETL_TASK_QUEUE", "dev-fetch-tasks")
	workers    = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
	staticPath = flag.String("static", "content/static", "path to folder containing static files served")
)

func main() {
	flag.Parse()

	ctx := context.Background()

	cfg, err := config.Init(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	cfg.Dump(os.Stderr)

	if cfg.UseProfiler {
		if err := profiler.Start(profiler.Config{}); err != nil {
			log.Fatalf(ctx, "profiler.Start: %v", err)
		}
	}

	readProxyRemoved(ctx)

	// Wrap the postgres driver with OpenCensus instrumentation.
	driverName, err := ocsql.Register("postgres", ocsql.WithAllTraceOptions())
	if err != nil {
		log.Fatalf(ctx, "unable to register the ocsql driver: %v\n", err)
	}
	ddb, err := database.Open(driverName, cfg.DBConnInfo())
	if err != nil {
		log.Fatalf(ctx, "database.Open: %v", err)
	}
	db := postgres.New(ddb)
	defer db.Close()

	populateExcluded(ctx, db)

	indexClient, err := index.New(cfg.IndexURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	proxyClient, err := proxy.New(cfg.ProxyURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	fetchQueue := newQueue(ctx, cfg, proxyClient, db)
	reportingClient := reportingClient(ctx, cfg)
	redisClient := getRedis(ctx, cfg)
	server, err := etl.NewServer(cfg, db, indexClient, proxyClient, redisClient, fetchQueue, reportingClient, *staticPath)
	if err != nil {
		log.Fatal(ctx, err)
	}
	router := dcensus.NewRouter(nil)
	server.Install(router.Handle)

	views := append(dcensus.ClientViews, dcensus.ServerViews...)
	if err := dcensus.Init(cfg, views...); err != nil {
		log.Fatal(ctx, err)
	}
	// We are not currently forwarding any ports on AppEngine, so serving debug
	// information is broken.
	if !cfg.OnAppEngine() {
		dcensusServer, err := dcensus.NewServer()
		if err != nil {
			log.Fatal(ctx, err)
		}
		go http.ListenAndServe(cfg.DebugAddr("localhost:8001"), dcensusServer)
	}

	handlerTimeout, err := strconv.Atoi(timeout)
	if err != nil {
		log.Fatalf(ctx, "strconv.Atoi(%q): %v", timeout, err)
	}
	requestLogger := logger(ctx, cfg)
	mw := middleware.Chain(
		middleware.RequestLog(requestLogger),
		middleware.Timeout(time.Duration(handlerTimeout)*time.Minute),
	)
	http.Handle("/", mw(router))

	addr := cfg.HostAddr("localhost:8000")
	log.Infof(ctx, "Listening on addr %s", addr)
	log.Fatal(ctx, http.ListenAndServe(addr, nil))
}

func newQueue(ctx context.Context, cfg *config.Config, proxyClient *proxy.Client, db *postgres.DB) queue.Queue {
	if !cfg.OnAppEngine() {
		return queue.NewInMemory(ctx, proxyClient, db, *workers, etl.FetchAndUpdateState)
	}
	client, err := cloudtasks.NewClient(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	return queue.NewGCP(cfg, client, queueName)
}

func getRedis(ctx context.Context, cfg *config.Config) *redis.Client {
	var dialTimeout time.Duration
	if dl, ok := ctx.Deadline(); ok {
		dialTimeout = time.Until(dl)
	}
	if cfg.RedisHAHost != "" {
		return redis.NewClient(&redis.Options{
			Addr:        cfg.RedisHAHost + ":" + cfg.RedisHAPort,
			DialTimeout: dialTimeout,
			// We update completions with one big pipeline, so we need long write
			// timeouts. ReadTimeout is increased only to be consistent with
			// WriteTimeout.
			WriteTimeout: 5 * time.Minute,
			ReadTimeout:  5 * time.Minute,
		})
	}
	return nil
}

func reportingClient(ctx context.Context, cfg *config.Config) *errorreporting.Client {
	if !cfg.OnAppEngine() {
		return nil
	}
	reporter, err := errorreporting.NewClient(ctx, cfg.ProjectID, errorreporting.Config{
		ServiceName: cfg.ServiceID,
		OnError: func(err error) {
			log.Errorf(ctx, "Error reporting failed: %v", err)
		},
	})
	if err != nil {
		log.Fatal(ctx, err)
	}
	return reporter
}

func logger(ctx context.Context, cfg *config.Config) middleware.Logger {
	if cfg.OnAppEngine() {
		logger, err := log.UseStackdriver(ctx, cfg, "etl-log")
		if err != nil {
			log.Fatal(ctx, err)
		}
		return logger
	}
	return middleware.LocalLogger{}
}

// Read a file of module versions that we should ignore because
// the are in the index but not stored in the proxy.
// Format of the file: each line is
//     module@version
func readProxyRemoved(ctx context.Context) {
	filename := config.GetEnv("GO_DISCOVERY_PROXY_REMOVED", "")
	if filename == "" {
		return
	}
	lines, err := readFileLines(filename)
	if err != nil {
		log.Fatal(ctx, err)
	}
	for _, line := range lines {
		etl.ProxyRemoved[line] = true
	}
	log.Infof(ctx, "read %d excluded module versions from %s", len(etl.ProxyRemoved), filename)
}

// populateExcluded adds each element of excludedPrefixes to the excluded_prefixes
// table if it isn't already present.
func populateExcluded(ctx context.Context, db *postgres.DB) {
	const excludedFilename = "cmd/etl/excluded.txt"
	lines, err := readFileLines(excludedFilename)
	if err != nil {
		log.Fatal(ctx, err)
	}
	user := os.Getenv("USER")
	if user == "" {
		user = "etl"
	}
	for _, line := range lines {
		var prefix, reason string
		i := strings.IndexAny(line, " \t")
		if i >= 0 {
			prefix = line[:i]
			reason = strings.TrimSpace(line[i+1:])
		}
		if reason == "" {
			log.Fatalf(ctx, "missing reason in %s, line %q", excludedFilename, line)
		}
		present, err := db.IsExcluded(ctx, prefix)
		if err != nil {
			log.Fatalf(ctx, "db.IsExcluded(%q): %v", prefix, err)
		}
		if !present {
			if err := db.InsertExcludedPrefix(ctx, prefix, user, reason); err != nil {
				log.Fatalf(ctx, "db.InsertExcludedPrefix(%q, %q, %q): %v", prefix, user, reason, err)
			}
		}
	}
}

// readFileLines reads filename and returns its lines, trimmed of whitespace.
// Blank lines and lines whose first non-blank character is '#' are omitted.
func readFileLines(filename string) ([]string, error) {
	var lines []string
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		line := strings.TrimSpace(scan.Text())
		if line == "" || line[0] == '#' {
			continue
		}
		lines = append(lines, line)
	}
	if err := scan.Err(); err != nil {
		return nil, err
	}
	return lines, nil
}
<|endoftext|>


<commit_before>// Copyright 2014, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

// A high performance pseudo-random number generator.
package main

import "os"
import "io"
import "fmt"
import "syscall"
import "runtime"
import "unsafe"
import "github.com/ogier/pflag"
import "bitbucket.org/rawr/gorandom/rand"

// Check if the given file descriptor writes to a terminal.
func isatty(fd int) bool {
	var termios syscall.Termios
	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCGETS, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
	return err == 0
}

func main() {
	// Basic user configuration variables
	count := pflag.Int64P("count", "n", -1, "Number of random bytes to generate")
	force := pflag.BoolP("force", "f", false, "Force output to terminal")
	procs := pflag.IntP("procs", "p", runtime.NumCPU(), "Maximum number of concurrent workers")
	pflag.Parse()

	if !(*force) && isatty(syscall.Stdout) {
		fmt.Fprintf(os.Stderr, "Random data not written to terminal.\n\n")
		pflag.Usage()
		os.Exit(1)
	}
	if (*procs) < 1 {
		fmt.Fprintf(os.Stderr, "Number of workers must be positive.\n\n")
		pflag.Usage()
		os.Exit(1)
	}

	runtime.GOMAXPROCS(*procs)
	rand.SetNumRoutines(*procs)

	// Copy random data to stdout
	var err error
	if (*count) < 0 {
		_, err = io.Copy(os.Stdout, rand.Reader)
	} else {
		_, err = io.CopyN(os.Stdout, rand.Reader, *count)
	}
	if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.EPIPE {
		err = nil // Expected error is for the sink to close the pipe
	} else if err != nil {
		panic(err)
	}
}
<commit_msg>Use custom strconv to parse IEC and SI notations<commit_after>// Copyright 2014, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

// A high performance pseudo-random number generator.
package main

import "os"
import "io"
import "fmt"
import "math"
import "syscall"
import "runtime"
import "unsafe"
import "github.com/ogier/pflag"
import "bitbucket.org/rawr/gorandom/rand"
import "bitbucket.org/rawr/golib/strconv"

// Check if the given file descriptor writes to a terminal.
func isatty(fd int) bool {
	var termios syscall.Termios
	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCGETS, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
	return err == 0
}

func main() {
	// Basic user configuration variables
	count := pflag.StringP("count", "n", "+Inf", "Number of random bytes to generate.")
	force := pflag.BoolP("force", "f", false, "Force output to terminal.")
	procs := pflag.IntP("procs", "p", runtime.NumCPU(), "Maximum number of concurrent workers.")
	pflag.Parse()

	if !(*force) && isatty(syscall.Stdout) {
		fmt.Fprintf(os.Stderr, "Random data not written to terminal.\n\n")
		pflag.Usage()
		os.Exit(1)
	}
	if (*procs) < 1 {
		fmt.Fprintf(os.Stderr, "Number of workers must be positive.\n\n")
		pflag.Usage()
		os.Exit(1)
	}
	cnt, err := strconv.ParsePrefix(*count, strconv.AutoParse)
	if err != nil || math.IsNaN(cnt) {
		fmt.Fprintf(os.Stderr, "Number of bytes to generate is invalid.\n\n")
		pflag.Usage()
		os.Exit(1)
	}

	runtime.GOMAXPROCS(*procs)
	rand.SetNumRoutines(*procs)

	// Copy random data to stdout
	if int64(cnt) < 0 || math.IsInf(cnt, 0) {
		_, err = io.Copy(os.Stdout, rand.Reader)
	} else {
		_, err = io.CopyN(os.Stdout, rand.Reader, int64(cnt))
	}
	if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.EPIPE {
		err = nil // Expected error is for the sink to close the pipe
	} else if err != nil {
		panic(err)
	}
}
<|endoftext|>