<commit_before>package fundingcredit_test

import (
	"reflect"
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingcredit"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNewFundingCreditFromRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{2995368}

		got, err := fundingcredit.FromRaw(payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("valid arguments", func(t *testing.T) {
		payload := []interface{}{
			26222883,
			"fUST",
			1,
			1574013661000,
			1574079687000,
			350,
			nil,
			"ACTIVE",
			nil,
			nil,
			nil,
			0.0024,
			2,
			1574013661000,
			1574078487000,
			1,
			nil,
			nil,
			0,
			nil,
			1,
			"tBTCUST",
		}

		got, err := fundingcredit.FromRaw(payload)
		require.Nil(t, err)

		expected := &fundingcredit.Credit{
			ID:            26222883,
			Symbol:        "fUST",
			Side:          "",
			MTSCreated:    1574013661000,
			MTSUpdated:    1574079687000,
			Amount:        350,
			Status:        "ACTIVE",
			Rate:          0.0024,
			Period:        2,
			MTSOpened:     1574013661000,
			MTSLastPayout: 1574078487000,
			Notify:        true,
			Hidden:        false,
			Insure:        false,
			Renew:         false,
			RateReal:      0,
			NoClose:       true,
			PositionPair:  "tBTCUST",
		}
		assert.Equal(t, expected, got)
	})
}

func TestFundingCreditSnapshotFromRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{}
		got, err := fundingcredit.SnapshotFromRaw(payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("partially valid arguments", func(t *testing.T) {
		payload := []interface{}{
			[]interface{}{
				26222883,
				"fUST",
				1,
				1574013661000,
				1574079687000,
				350,
				nil,
				"ACTIVE",
				nil,
				nil,
				nil,
				0.0024,
				2,
				1574013661000,
				1574078487000,
				1,
				nil,
				nil,
				0,
				nil,
				1,
				"tBTCUST",
			},
			[]interface{}{26222883},
		}
		got, err := fundingcredit.SnapshotFromRaw(payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("valid arguments", func(t *testing.T) {
		payload := []interface{}{
			[]interface{}{
				26222883,
				"fUST",
				1,
				1574013661000,
				1574079687000,
				350,
				nil,
				"ACTIVE",
				nil,
				nil,
				nil,
				0.0024,
				2,
				1574013661000,
				1574078487000,
				1,
				nil,
				nil,
				0,
				nil,
				1,
				"tBTCUST",
			},
			[]interface{}{
				26222884,
				"fUST",
				1,
				1574013661000,
				1574079687000,
				350,
				nil,
				"ACTIVE",
				nil,
				nil,
				nil,
				0.0024,
				2,
				1574013661000,
				1574078487000,
				1,
				nil,
				nil,
				1,
				nil,
				1,
				"tBTCUST",
			},
		}

		got, err := fundingcredit.SnapshotFromRaw(payload)
		require.Nil(t, err)

		expected := &fundingcredit.Snapshot{
			Snapshot: []*fundingcredit.Credit{
				{
					ID:            26222883,
					Symbol:        "fUST",
					Side:          "",
					MTSCreated:    1574013661000,
					MTSUpdated:    1574079687000,
					Amount:        350,
					Status:        "ACTIVE",
					Rate:          0.0024,
					Period:        2,
					MTSOpened:     1574013661000,
					MTSLastPayout: 1574078487000,
					Notify:        true,
					Hidden:        false,
					Insure:        false,
					Renew:         false,
					RateReal:      0,
					NoClose:       true,
					PositionPair:  "tBTCUST",
				},
				{
					ID:            26222884,
					Symbol:        "fUST",
					Side:          "",
					MTSCreated:    1574013661000,
					MTSUpdated:    1574079687000,
					Amount:        350,
					Status:        "ACTIVE",
					Rate:          0.0024,
					Period:        2,
					MTSOpened:     1574013661000,
					MTSLastPayout: 1574078487000,
					Notify:        true,
					Hidden:        false,
					Insure:        false,
					Renew:         true,
					RateReal:      0,
					NoClose:       true,
					PositionPair:  "tBTCUST",
				},
			},
		}

		assert.Equal(t, expected, got)
	})
}

func TestFundingCreditCancelRequest(t *testing.T) {
	t.Run("MarshalJSON", func(t *testing.T) {
		flcr := fundingcredit.CancelRequest{ID: 123}
		got, err := flcr.MarshalJSON()

		require.Nil(t, err)

		expected := "[0, \"fcc\", null, {\"id\":123}]"
		assert.Equal(t, expected, string(got))
	})
}

func TestNewFromRaw(t *testing.T) {
	pld := []interface{}{
		26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", nil, nil,
		nil, 0.0024, 2, 1574013661000, 1574078487000, 1, nil, nil, 0, nil, 1, "tBTCUST",
	}

	expected := "fundingcredit.New"
	o, err := fundingcredit.NewFromRaw(pld)
	assert.Nil(t, err)

	got := reflect.TypeOf(o).String()
	assert.Equal(t, expected, got)
}

func TestUpdateFromRaw(t *testing.T) {
	pld := []interface{}{
		26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", nil, nil,
		nil, 0.0024, 2, 1574013661000, 1574078487000, 1, nil, nil, 0, nil, 1, "tBTCUST",
	}

	expected := "fundingcredit.Update"
	o, err := fundingcredit.UpdateFromRaw(pld)
	assert.Nil(t, err)

	got := reflect.TypeOf(o).String()
	assert.Equal(t, expected, got)
}

func TestCancelFromRaw(t *testing.T) {
	pld := []interface{}{
		26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", nil, nil,
		nil, 0.0024, 2, 1574013661000, 1574078487000, 1, nil, nil, 0, nil, 1, "tBTCUST",
	}

	expected := "fundingcredit.Cancel"
	o, err := fundingcredit.CancelFromRaw(pld)
	assert.Nil(t, err)

	got := reflect.TypeOf(o).String()
	assert.Equal(t, expected, got)
}
<commit_msg>updating funding model tests<commit_after>package fundingcredit_test

import (
	"reflect"
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingcredit"
	"github.com/stretchr/testify/assert"
)

func TestFromRaw(t *testing.T) {
	cases := map[string]struct {
		pld      []interface{}
		expected *fundingcredit.Credit
		err      func(*testing.T, error)
	}{
		"invalid pld": {
			pld:      []interface{}{"exchange"},
			expected: nil,
			err: func(t *testing.T, err error) {
				assert.NotNil(t, err)
			},
		},
		"rest funding credits item": {
			pld: []interface{}{
				26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", "FIXED", nil,
				nil, 0.0024, 2, 1574013661000, 1574078487000, 0, nil, nil, 0, nil, 0, "tBTCUST",
			},
			expected: &fundingcredit.Credit{
				ID:            26222883,
				Symbol:        "fUST",
				Side:          1,
				MTSCreated:    1574013661000,
				MTSUpdated:    1574079687000,
				Amount:        350,
				Status:        "ACTIVE",
				RateType:      "FIXED",
				Rate:          0.0024,
				Period:        2,
				MTSOpened:     1574013661000,
				MTSLastPayout: 1574078487000,
				Notify:        false,
				Hidden:        false,
				Insure:        false,
				Renew:         false,
				RateReal:      0,
				NoClose:       false,
				PositionPair:  "tBTCUST",
			},
			err: func(t *testing.T, err error) {
				assert.Nil(t, err)
			},
		},
		"rest funding credits history item": {
			pld: []interface{}{
				171988300, "fUSD", 1, 1574230085000, 1574402835000, 50.70511182, nil,
				"CLOSED (expired)", "FIXED", nil, nil, 0.00024799, 2, 1574230085000,
				1574403364000, nil, 0, nil, 0, nil, 0, "tEOSUSD",
			},
			expected: &fundingcredit.Credit{
				ID:            171988300,
				Symbol:        "fUSD",
				Side:          1,
				MTSCreated:    1574230085000,
				MTSUpdated:    1574402835000,
				Amount:        50.70511182,
				Status:        "CLOSED (expired)",
				RateType:      "FIXED",
				Rate:          0.00024799,
				Period:        2,
				MTSOpened:     1574230085000,
				MTSLastPayout: 1574403364000,
				Notify:        false,
				Hidden:        false,
				Insure:        false,
				Renew:         false,
				RateReal:      0,
				NoClose:       false,
				PositionPair:  "tEOSUSD",
			},
			err: func(t *testing.T, err error) {
				assert.Nil(t, err)
			},
		},
		"ws fcs item": {
			pld: []interface{}{
				26223578, "fUST", 1, 1575052261000, 1575296187000, 350, 0, "ACTIVE", nil, nil,
				nil, 0, 30, 1575052261000, 1575293487000, 0, 0, nil, 0, nil, 0, "tBTCUST",
			},
			expected: &fundingcredit.Credit{
				ID:            26223578,
				Symbol:        "fUST",
				Side:          1,
				MTSCreated:    1575052261000,
				MTSUpdated:    1575296187000,
				Amount:        350,
				Status:        "ACTIVE",
				Rate:          0,
				Period:        30,
				MTSOpened:     1575052261000,
				MTSLastPayout: 1575293487000,
				Notify:        false,
				Hidden:        false,
				Insure:        false,
				Renew:         false,
				RateReal:      0,
				NoClose:       false,
				PositionPair:  "tBTCUST",
			},
			err: func(t *testing.T, err error) {
				assert.Nil(t, err)
			},
		},
	}

	for k, v := range cases {
		t.Run(k, func(t *testing.T) {
			got, err := fundingcredit.FromRaw(v.pld)
			v.err(t, err)
			assert.Equal(t, v.expected, got)
		})
	}
}

func TestSnapshotFromRaw(t *testing.T) {
	cases := map[string]struct {
		pld      []interface{}
		expected *fundingcredit.Snapshot
		err      func(*testing.T, error)
	}{
		"invalid pld": {
			pld:      []interface{}{},
			expected: nil,
			err: func(t *testing.T, err error) {
				assert.NotNil(t, err)
			},
		},
		"rest funding credits": {
			pld: []interface{}{
				[]interface{}{
					26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", "FIXED", nil,
					nil, 0.0024, 2, 1574013661000, 1574078487000, 0, nil, nil, 0, nil, 0, "tBTCUST",
				},
			},
			expected: &fundingcredit.Snapshot{
				Snapshot: []*fundingcredit.Credit{
					{
						ID:            26222883,
						Symbol:        "fUST",
						Side:          1,
						MTSCreated:    1574013661000,
						MTSUpdated:    1574079687000,
						Amount:        350,
						Status:        "ACTIVE",
						RateType:      "FIXED",
						Rate:          0.0024,
						Period:        2,
						MTSOpened:     1574013661000,
						MTSLastPayout: 1574078487000,
						Notify:        false,
						Hidden:        false,
						Insure:        false,
						Renew:         false,
						RateReal:      0,
						NoClose:       false,
						PositionPair:  "tBTCUST",
					},
				},
			},
			err: func(t *testing.T, err error) {
				assert.Nil(t, err)
			},
		},
		"rest funding credits history": {
			pld: []interface{}{
				[]interface{}{
					171988300, "fUSD", 1, 1574230085000, 1574402835000, 50.70511182, nil,
					"CLOSED (expired)", "FIXED", nil, nil, 0.00024799, 2, 1574230085000,
					1574403364000, nil, 0, nil, 0, nil, 0, "tEOSUSD",
				},
			},
			expected: &fundingcredit.Snapshot{
				Snapshot: []*fundingcredit.Credit{
					{
						ID:            171988300,
						Symbol:        "fUSD",
						Side:          1,
						MTSCreated:    1574230085000,
						MTSUpdated:    1574402835000,
						Amount:        50.70511182,
						Status:        "CLOSED (expired)",
						RateType:      "FIXED",
						Rate:          0.00024799,
						Period:        2,
						MTSOpened:     1574230085000,
						MTSLastPayout: 1574403364000,
						Notify:        false,
						Hidden:        false,
						Insure:        false,
						Renew:         false,
						RateReal:      0,
						NoClose:       false,
						PositionPair:  "tEOSUSD",
					},
				},
			},
			err: func(t *testing.T, err error) {
				assert.Nil(t, err)
			},
		},
		"ws fcs": {
			pld: []interface{}{
				[]interface{}{
					26223578, "fUST", 1, 1575052261000, 1575296187000, 350, 0, "ACTIVE", nil, nil,
					nil, 0, 30, 1575052261000, 1575293487000, 0, 0, nil, 0, nil, 0, "tBTCUST",
				},
			},
			expected: &fundingcredit.Snapshot{
				Snapshot: []*fundingcredit.Credit{
					{
						ID:            26223578,
						Symbol:        "fUST",
						Side:          1,
						MTSCreated:    1575052261000,
						MTSUpdated:    1575296187000,
						Amount:        350,
						Status:        "ACTIVE",
						Rate:          0,
						Period:        30,
						MTSOpened:     1575052261000,
						MTSLastPayout: 1575293487000,
						Notify:        false,
						Hidden:        false,
						Insure:        false,
						Renew:         false,
						RateReal:      0,
						NoClose:       false,
						PositionPair:  "tBTCUST",
					},
				},
			},
			err: func(t *testing.T, err error) {
				assert.Nil(t, err)
			},
		},
	}

	for k, v := range cases {
		t.Run(k, func(t *testing.T) {
			got, err := fundingcredit.SnapshotFromRaw(v.pld)
			v.err(t, err)
			assert.Equal(t, v.expected, got)
		})
	}
}

func TestNewFromRaw(t *testing.T) {
	pld := []interface{}{
		26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", nil, nil,
		nil, 0.0024, 2, 1574013661000, 1574078487000, 1, nil, nil, 0, nil, 1, "tBTCUST",
	}

	expected := "fundingcredit.New"
	o, err := fundingcredit.NewFromRaw(pld)
	assert.Nil(t, err)

	got := reflect.TypeOf(o).String()
	assert.Equal(t, expected, got)
}

func TestUpdateFromRaw(t *testing.T) {
	pld := []interface{}{
		26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", nil, nil,
		nil, 0.0024, 2, 1574013661000, 1574078487000, 1, nil, nil, 0, nil, 1, "tBTCUST",
	}

	expected := "fundingcredit.Update"
	o, err := fundingcredit.UpdateFromRaw(pld)
	assert.Nil(t, err)

	got := reflect.TypeOf(o).String()
	assert.Equal(t, expected, got)
}

func TestCancelFromRaw(t *testing.T) {
	pld := []interface{}{
		26222883, "fUST", 1, 1574013661000, 1574079687000, 350, nil, "ACTIVE", nil, nil,
		nil, 0.0024, 2, 1574013661000, 1574078487000, 1, nil, nil, 0, nil, 1, "tBTCUST",
	}

	expected := "fundingcredit.Cancel"
	o, err := fundingcredit.CancelFromRaw(pld)
	assert.Nil(t, err)

	got := reflect.TypeOf(o).String()
	assert.Equal(t, expected, got)
}
<|endoftext|>


<commit_before>package openshift_integrated_oauth_server

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/golang/glog"
	"github.com/spf13/cobra"

	kerrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"

	configv1 "github.com/openshift/api/config/v1"
	osinv1 "github.com/openshift/api/osin/v1"
	"github.com/openshift/library-go/pkg/serviceability"
	"github.com/openshift/origin/pkg/api/legacy"
)

type OsinServer struct {
	ConfigFile string
}

func NewOsinServer(out, errout io.Writer, stopCh <-chan struct{}) *cobra.Command {
	options := &OsinServer{}

	cmd := &cobra.Command{
		Use:   "osinserver",
		Short: "Launch OpenShift osin server",
		Run: func(c *cobra.Command, args []string) {
			legacy.InstallInternalLegacyAll(legacyscheme.Scheme)

			kcmdutil.CheckErr(options.Validate())

			serviceability.StartProfiler()

			if err := options.RunOsinServer(stopCh); err != nil {
				if kerrors.IsInvalid(err) {
					if details := err.(*kerrors.StatusError).ErrStatus.Details; details != nil {
						fmt.Fprintf(errout, "Invalid %s %s\n", details.Kind, details.Name)
						for _, cause := range details.Causes {
							fmt.Fprintf(errout, "  %s: %s\n", cause.Field, cause.Message)
						}
						os.Exit(255)
					}
				}
				glog.Fatal(err)
			}
		},
	}

	flags := cmd.Flags()
	// This command only supports reading from config
	flags.StringVar(&options.ConfigFile, "config", "", "Location of the osin configuration file to run from.")
	cmd.MarkFlagFilename("config", "yaml", "yml")
	cmd.MarkFlagRequired("config")

	return cmd
}

func (o *OsinServer) Validate() error {
	if len(o.ConfigFile) == 0 {
		return errors.New("--config is required for this command")
	}

	return nil
}

func (o *OsinServer) RunOsinServer(stopCh <-chan struct{}) error {
	configContent, err := ioutil.ReadFile(o.ConfigFile)
	if err != nil {
		return err
	}

	// TODO this probably needs to be updated to a container inside openshift/api/osin/v1
	scheme := runtime.NewScheme()
	utilruntime.Must(osinv1.Install(scheme))
	codecs := serializer.NewCodecFactory(scheme)
	obj, err := runtime.Decode(codecs.UniversalDecoder(osinv1.GroupVersion, configv1.GroupVersion), configContent)
	if err != nil {
		// TODO drop this code once we remove the hypershift path
		obj = &osinv1.OsinServerConfig{}
		if jsonErr := json.Unmarshal(configContent, obj); jsonErr != nil {
			glog.Errorf("osin config parse error: %v", jsonErr)
			return err
		}
		// return err
	}

	config, ok := obj.(*osinv1.OsinServerConfig)
	if !ok {
		return fmt.Errorf("expected OsinServerConfig, got %T", config)
	}

	return RunOsinServer(config, stopCh)
}
<commit_msg>Require OsinServerConfig in OAuth server binary<commit_after>package openshift_integrated_oauth_server

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/golang/glog"
	"github.com/spf13/cobra"

	kerrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"

	configv1 "github.com/openshift/api/config/v1"
	osinv1 "github.com/openshift/api/osin/v1"
	"github.com/openshift/library-go/pkg/serviceability"
	"github.com/openshift/origin/pkg/api/legacy"
)

type OsinServer struct {
	ConfigFile string
}

func NewOsinServer(out, errout io.Writer, stopCh <-chan struct{}) *cobra.Command {
	options := &OsinServer{}

	cmd := &cobra.Command{
		Use:   "osinserver",
		Short: "Launch OpenShift osin server",
		Run: func(c *cobra.Command, args []string) {
			legacy.InstallInternalLegacyAll(legacyscheme.Scheme)

			kcmdutil.CheckErr(options.Validate())

			serviceability.StartProfiler()

			if err := options.RunOsinServer(stopCh); err != nil {
				if kerrors.IsInvalid(err) {
					if details := err.(*kerrors.StatusError).ErrStatus.Details; details != nil {
						fmt.Fprintf(errout, "Invalid %s %s\n", details.Kind, details.Name)
						for _, cause := range details.Causes {
							fmt.Fprintf(errout, "  %s: %s\n", cause.Field, cause.Message)
						}
						os.Exit(255)
					}
				}
				glog.Fatal(err)
			}
		},
	}

	flags := cmd.Flags()
	// This command only supports reading from config
	flags.StringVar(&options.ConfigFile, "config", "", "Location of the osin configuration file to run from.")
	cmd.MarkFlagFilename("config", "yaml", "yml")
	cmd.MarkFlagRequired("config")

	return cmd
}

func (o *OsinServer) Validate() error {
	if len(o.ConfigFile) == 0 {
		return errors.New("--config is required for this command")
	}

	return nil
}

func (o *OsinServer) RunOsinServer(stopCh <-chan struct{}) error {
	configContent, err := ioutil.ReadFile(o.ConfigFile)
	if err != nil {
		return err
	}

	// TODO this probably needs to be updated to a container inside openshift/api/osin/v1
	scheme := runtime.NewScheme()
	utilruntime.Must(osinv1.Install(scheme))
	codecs := serializer.NewCodecFactory(scheme)
	obj, err := runtime.Decode(codecs.UniversalDecoder(osinv1.GroupVersion, configv1.GroupVersion), configContent)
	if err != nil {
		return err
	}

	config, ok := obj.(*osinv1.OsinServerConfig)
	if !ok {
		return fmt.Errorf("expected OsinServerConfig, got %T", config)
	}

	return RunOsinServer(config, stopCh)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"gopkg.in/src-d/simple-linguist.v1"
)

func main() {
	flag.Usage = usage
	flag.Parse()
	root, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	errors := false
	o := make(map[string][]string, 0)
	err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errors = true
			log.Println(err)
			return filepath.SkipDir
		}

		if slinguist.IsVendor(f.Name()) || slinguist.IsDotFile(f.Name()) {
			if f.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if f.IsDir() {
			return nil
		}

		content, err := ioutil.ReadFile(path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		l := slinguist.GetLanguage(filepath.Base(path), content)

		r, err := filepath.Rel(root, path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		o[l] = append(o[l], r)
		return nil
	})

	if err != nil {
		log.Fatal(err)
	}

	js, _ := json.MarshalIndent(o, "", "  ")
	fmt.Printf("%s\n", js)

	if errors {
		os.Exit(2)
	}
}

func usage() {
	fmt.Fprintf(
		os.Stderr, "simple-linguist, A simple (and faster) implementation of linguist \nusage: %s <path>\n",
		os.Args[0],
	)

	flag.PrintDefaults()
}
<commit_msg>fixed cli to skip some type of directories and files<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"gopkg.in/src-d/simple-linguist.v1"
)

func main() {
	flag.Usage = usage
	flag.Parse()
	root, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	errors := false
	out := make(map[string][]string, 0)
	err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errors = true
			log.Println(err)
			return filepath.SkipDir
		}

		relativePath, err := filepath.Rel(root, path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		if relativePath == "." {
			return nil
		}

		if f.IsDir() {
			relativePath = relativePath + "/"
		}

		if slinguist.IsVendor(relativePath) || slinguist.IsDotFile(relativePath) ||
			slinguist.IsDocumentation(relativePath) || slinguist.IsConfiguration(relativePath) {
			if f.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if f.IsDir() {
			return nil
		}

		content, err := ioutil.ReadFile(path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		language := slinguist.GetLanguage(filepath.Base(path), content)
		if language == slinguist.OtherLanguage {
			return nil
		}

		out[language] = append(out[language], relativePath)
		return nil
	})

	if err != nil {
		log.Fatal(err)
	}

	data, _ := json.MarshalIndent(out, "", "  ")
	fmt.Printf("%s\n", data)

	if errors {
		os.Exit(2)
	}
}

func usage() {
	fmt.Fprintf(
		os.Stderr, "simple-linguist, A simple (and faster) implementation of linguist \nusage: %s <path>\n",
		os.Args[0],
	)

	flag.PrintDefaults()
}
<|endoftext|>


<commit_before>package cli

import (
	"flag"
	"time"

	"github.com/opentable/sous/config"
	"github.com/opentable/sous/dto"
	"github.com/opentable/sous/graph"
	sous "github.com/opentable/sous/lib"
	"github.com/opentable/sous/server"
	"github.com/opentable/sous/util/cmdr"
	"github.com/opentable/sous/util/logging"
	"github.com/opentable/sous/util/logging/messages"
	"github.com/opentable/sous/util/restful"
	"github.com/samsalisbury/semv"
)

// SousNewDeploy has the same interface as SousDeploy, but uses the new
// PUT /single-deployment endpoint to begin the deployment, and polls by
// watching the returned rectification URL.
type SousNewDeploy struct {
	DeployFilterFlags config.DeployFilterFlags `inject:"optional"`
	StateReader       graph.StateReader
	HTTPClient        *graph.ClusterSpecificHTTPClient
	TargetManifestID  graph.TargetManifestID
	LogSink           graph.LogSink
	dryrunOption      string
	waitStable        bool
	User              sous.User
}

func init() { TopLevelCommands["newdeploy"] = &SousNewDeploy{} }

const sousNewDeployHelp = `deploys a new version into a particular cluster

usage: sous newdeploy -cluster <name> -tag <semver>

EXPERIMENTAL COMMAND: This may or may not yet do what it says on the tin.
Feel free to try it out, but if it breaks, you get to keep both pieces.

sous deploy will deploy the version tag for this application in the named
cluster.
`

// Help returns the help string for this command.
func (sd *SousNewDeploy) Help() string { return sousNewDeployHelp }

// AddFlags adds the flags for sous init.
func (sd *SousNewDeploy) AddFlags(fs *flag.FlagSet) {
	MustAddFlags(fs, &sd.DeployFilterFlags, DeployFilterFlagsHelp)

	fs.BoolVar(&sd.waitStable, "wait-stable", true,
		"wait for the deploy to complete before returning (otherwise, use --wait-stable=false)")
	fs.StringVar(&sd.dryrunOption, "dry-run", "none",
		"prevent rectify from actually changing things - "+
			"values are none,scheduler,registry,both")
}

// RegisterOn adds flag options to the graph.
func (sd *SousNewDeploy) RegisterOn(psy Addable) {
	psy.Add(graph.DryrunNeither)
	psy.Add(&sd.DeployFilterFlags)
}

// Execute creates the new deployment.
func (sd *SousNewDeploy) Execute(args []string) cmdr.Result {

	cluster := sd.DeployFilterFlags.Cluster

	newVersion, err := semv.Parse(sd.DeployFilterFlags.Tag)
	if err != nil {
		return cmdr.UsageErrorf("not semver: -tag %s", sd.DeployFilterFlags.Tag)
	}

	d := server.SingleDeploymentBody{}
	q := sd.TargetManifestID.QueryMap()
	q["cluster"] = cluster
	updater, err := sd.HTTPClient.Retrieve("./single-deployment", q, &d, nil)
	if err != nil {
		return cmdr.EnsureErrorResult(err)
	}
	messages.ReportLogFieldsMessage("SousNewDeploy.Execute Retrieved Deployment",
		logging.ExtraDebug1Level, sd.LogSink, d)

	d.Deployment.Version = newVersion

	updateResponse, err := updater.Update(d, sd.User.HTTPHeaders())
	if err != nil {
		return cmdr.EnsureErrorResult(err)
	}

	if location := updateResponse.Location(); location != "" {
		//return cmdr.Successf("Deployment queued at: %s", location)
		client, _ := restful.NewClient("", sd.LogSink, nil)
		return PollDeployQueue(location, client, 600, sd.LogSink)
	}
	return cmdr.Successf("Desired version for %q in cluster %q already %q",
		sd.TargetManifestID, cluster, sd.DeployFilterFlags.Tag)

}

// PollDeployQueue is used to poll server on status of Single Deployment.
func PollDeployQueue(location string, client restful.HTTPClient, loopIteration int, log logging.LogSink) cmdr.Result {
	response := dto.R11nResponse{}
	location = "http://" + location

	for i := 0; i < loopIteration; i++ {
		_, err := client.Retrieve(location, nil, &response, nil)
		messages.ReportLogFieldsMessageToConsole("PollDeployQueue called waiting for created response...", logging.ExtraDebug1Level, log, location, response, err)
		if err != nil {
			return cmdr.InternalErrorf("Failed to deploy: %s", err)
		}
		queuePosition := response.QueuePosition
		if queuePosition < 0 && response.Resolution != nil {
			if checkResolution(*response.Resolution) {
				return cmdr.Successf("worked")
			}
		}
		time.Sleep(1 * time.Second)
	}
	return cmdr.InternalErrorf("failed to deploy %s", location)
}

func checkResolution(resolution sous.DiffResolution) bool {
	response := false
	switch resolution.Desc {
	case sous.CreateDiff:
		response = true
	}
	return response
}
<commit_msg>clean up ui responses, add duration<commit_after>package cli

import (
	"flag"
	"fmt"
	"time"

	"github.com/opentable/sous/config"
	"github.com/opentable/sous/dto"
	"github.com/opentable/sous/graph"
	sous "github.com/opentable/sous/lib"
	"github.com/opentable/sous/server"
	"github.com/opentable/sous/util/cmdr"
	"github.com/opentable/sous/util/logging"
	"github.com/opentable/sous/util/logging/messages"
	"github.com/opentable/sous/util/restful"
	"github.com/samsalisbury/semv"
)

// SousNewDeploy has the same interface as SousDeploy, but uses the new
// PUT /single-deployment endpoint to begin the deployment, and polls by
// watching the returned rectification URL.
type SousNewDeploy struct {
	DeployFilterFlags config.DeployFilterFlags `inject:"optional"`
	StateReader       graph.StateReader
	HTTPClient        *graph.ClusterSpecificHTTPClient
	TargetManifestID  graph.TargetManifestID
	LogSink           graph.LogSink
	dryrunOption      string
	waitStable        bool
	User              sous.User
}

func init() { TopLevelCommands["newdeploy"] = &SousNewDeploy{} }

const sousNewDeployHelp = `deploys a new version into a particular cluster

usage: sous newdeploy -cluster <name> -tag <semver>

EXPERIMENTAL COMMAND: This may or may not yet do what it says on the tin.
Feel free to try it out, but if it breaks, you get to keep both pieces.

sous deploy will deploy the version tag for this application in the named
cluster.
`

// Help returns the help string for this command.
func (sd *SousNewDeploy) Help() string { return sousNewDeployHelp }

// AddFlags adds the flags for sous init.
func (sd *SousNewDeploy) AddFlags(fs *flag.FlagSet) {
	MustAddFlags(fs, &sd.DeployFilterFlags, DeployFilterFlagsHelp)

	fs.BoolVar(&sd.waitStable, "wait-stable", true,
		"wait for the deploy to complete before returning (otherwise, use --wait-stable=false)")
	fs.StringVar(&sd.dryrunOption, "dry-run", "none",
		"prevent rectify from actually changing things - "+
			"values are none,scheduler,registry,both")
}

// RegisterOn adds flag options to the graph.
func (sd *SousNewDeploy) RegisterOn(psy Addable) {
	psy.Add(graph.DryrunNeither)
	psy.Add(&sd.DeployFilterFlags)
}

// Execute creates the new deployment.
func (sd *SousNewDeploy) Execute(args []string) cmdr.Result {

	cluster := sd.DeployFilterFlags.Cluster

	newVersion, err := semv.Parse(sd.DeployFilterFlags.Tag)
	if err != nil {
		return cmdr.UsageErrorf("not semver: -tag %s", sd.DeployFilterFlags.Tag)
	}

	d := server.SingleDeploymentBody{}
	q := sd.TargetManifestID.QueryMap()
	q["cluster"] = cluster
	updater, err := sd.HTTPClient.Retrieve("./single-deployment", q, &d, nil)
	if err != nil {
		return cmdr.EnsureErrorResult(err)
	}
	messages.ReportLogFieldsMessage("SousNewDeploy.Execute Retrieved Deployment",
		logging.ExtraDebug1Level, sd.LogSink, d)

	d.Deployment.Version = newVersion

	updateResponse, err := updater.Update(d, sd.User.HTTPHeaders())
	if err != nil {
		return cmdr.EnsureErrorResult(err)
	}

	if location := updateResponse.Location(); location != "" {
		//return cmdr.Successf("Deployment queued at: %s", location)
		client, _ := restful.NewClient("", sd.LogSink, nil)
		return PollDeployQueue(location, client, 600, sd.LogSink)
	}
	return cmdr.Successf("Desired version for %q in cluster %q already %q",
		sd.TargetManifestID, cluster, sd.DeployFilterFlags.Tag)

}

func timeTrack(start time.Time) string {
	elapsed := time.Since(start)
	return elapsed.String()
}

// PollDeployQueue is used to poll server on status of Single Deployment.
func PollDeployQueue(location string, client restful.HTTPClient, loopIteration int, log logging.LogSink) cmdr.Result {
	start := time.Now()
	response := dto.R11nResponse{}
	location = "http://" + location

	for i := 0; i < loopIteration; i++ {
		_, err := client.Retrieve(location, nil, &response, nil)

		if i%10 == 0 {
			msg := fmt.Sprintf("PollDeployQueue called waiting for created response... %s elapsed", timeTrack(start))
			messages.ReportLogFieldsMessageToConsole(msg, logging.ExtraDebug1Level, log, location, response, err)
		}

		if err != nil {
			return cmdr.InternalErrorf("Failed to deploy: %s duration: %s", err, timeTrack(start))
		}

		queuePosition := response.QueuePosition
		if queuePosition < 0 && response.Resolution != nil {
			if checkResolution(*response.Resolution) {
				return cmdr.Successf("Deployment Complete %s, duration: %s", response.Resolution.DeploymentID.String(), timeTrack(start))
			}
		}
		time.Sleep(1 * time.Second)
	}
	return cmdr.InternalErrorf("failed to deploy %s", location)
}

func checkResolution(resolution sous.DiffResolution) bool {
	response := false
	switch resolution.Desc {
	case sous.CreateDiff:
		response = true
	}
	return response
}
<|endoftext|>


<commit_before>package modbusone

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"sync/atomic"
	"time"
)

//FailoverRTUClient implements Client/Master side logic for RTU over a SerialContext to
//be used by a ProtocolHandler with failover function.
type FailoverRTUClient struct {
	com                  *FailoverSerialConn
	packetReader         PacketReader
	SlaveID              byte
	serverProcessingTime time.Duration
	actions              chan rtuAction
}

//FailoverRTUClient is also a Server
var _ Server = &FailoverRTUClient{}

//NewFailoverRTUClient create a new client with failover function communicating over SerialContext with the
//give slaveID as default.
//
//If isFailover is true, it is the secondary.
func NewFailoverRTUClient(com SerialContext, isFailover bool, slaveID byte) *FailoverRTUClient {
	pr, ok := com.(*FailoverSerialConn)
	if !ok {
		pr = NewFailoverConn(com, isFailover, true)
	}
	if pr.isFailover != isFailover {
		panic("A SerialContext was provided with conflicting settings.")
	}
	r := FailoverRTUClient{
		com:                  pr,
		packetReader:         pr,
		SlaveID:              slaveID,
		serverProcessingTime: time.Second,
		actions:              make(chan rtuAction),
	}
	return &r
}

//SetServerProcessingTime sets the time to wait for a server response, the total
//wait time also includes the time needed for data transmission
func (c *FailoverRTUClient) SetServerProcessingTime(t time.Duration) {
	c.serverProcessingTime = t
}

//GetTransactionTimeOut returns the total time to wait for a transaction
//(server response) to time out, given the expected length of RTU packets.
//This function is also used internally to calculate timeout.
func (c *FailoverRTUClient) GetTransactionTimeOut(reqLen, ansLen int) time.Duration {
	l := reqLen + ansLen
	return c.com.BytesDelay(l) + c.serverProcessingTime
}

//Serve serves FailoverRTUClient side handlers,
//
//A FailoverRTUClient expects a lot of "unexpected" read packets and "lost" writes so it
//is does not do the error checking that a normal client does, but instead try to guess the best
//interpretation.
func (c *FailoverRTUClient) Serve(handler ProtocolHandler) error {
	debugf("serve routine for %v", c.com.describe())
	defer c.Close()
	go func() {
		debugf("reader routine for %v", c.com.describe())
		//Reader loop that always ready to received data. This make sure that read
		//data is always new(ish), to dump data out that is received during an
		//unexpected time.
		for {
			rb := make([]byte, MaxRTUSize)
			n, err := c.packetReader.Read(rb)
			if err != nil {
				debugf("FailoverRTUClient read err:%v\n", err)
				c.actions <- rtuAction{t: clientError, err: err}
				c.Close()
				break
			}
			r := RTU(rb[:n])
			debugf("FailoverRTUClient read packet:%v\n", hex.EncodeToString(r))
			c.actions <- rtuAction{t: clientRead, data: r}
		}
	}()

	var last bytes.Buffer
	readUnexpected := func(act rtuAction, otherwise func()) {
		if act.err != nil || act.t != clientRead || len(act.data) == 0 {
			debugf("do not hand unexpected: %v", act)
			otherwise()
			return
		}
		debugf("handling unexpected: %v", act)
		pdu, err := act.data.GetPDU()
		if err != nil {
			debugf("readUnexpected GetPDU error: %v", err)
			otherwise()
			return
		}
		if !IsRequestReply(last.Bytes(), pdu) {
			if last.Len() != 0 {
				atomic.AddInt64(&c.com.Stats().OtherDrops, 1)
			}
			last.Reset()
			last.Write(pdu)
			return
		}
		defer last.Reset()

		if pdu.GetFunctionCode().IsWriteToServer() {
			//no-op for us
			return
		}

		bs, err := pdu.GetReplyValues()
		if err != nil {
			debugf("readUnexpected GetReplyValues error: %v", err)
			otherwise()
			return
		}
		err = handler.OnWrite(last.Bytes(), bs)
		if err != nil {
			debugf("readUnexpected OnWrite error: %v", err)
			otherwise()
			return
		}
	}

	for {
		act := <-c.actions
		switch act.t {
		default:
			readUnexpected(act, func() {
				atomic.AddInt64(&c.com.Stats().OtherDrops, 1)
				debugf("FailoverRTUClient drop unexpected: %v", act)
			})
			continue
		case clientError:
			return act.err
		case clientStart:
		}
		ap := act.data.fastGetPDU()
		afc := ap.GetFunctionCode()
		if afc.IsWriteToServer() {
			data, err := handler.OnRead(ap)
			if err != nil {
				act.errChan <- err
				continue
			}
			act.data = MakeRTU(act.data[0], ap.MakeWriteRequest(data))
			ap = act.data.fastGetPDU()
		}
		time.Sleep(c.com.MinDelay())
		_, err := c.com.Write(act.data)
		if err != nil {
			act.errChan <- err
			return err
		}
		c.com.lock.Lock()
		active := c.com.isActive
		c.com.lock.Unlock()
		if act.data[0] == 0 || !active {
			debugf("FailoverRTUClient skip action:%v\n", act)
			time.Sleep(c.com.BytesDelay(len(act.data)) + c.serverProcessingTime)
			act.errChan <- nil //always success
			continue           // do not wait for read on multicast or when not active
		}

		timeOutChan := time.After(c.GetTransactionTimeOut(len(act.data), MaxRTUSize))

	READ_LOOP:
		for {
		SELECT:
			select {
			case <-timeOutChan:
				act.errChan <- ErrServerTimeOut
				break READ_LOOP
			case react := <-c.actions:
				switch react.t {
				default:
					err := fmt.Errorf("unexpected action:%s", react.t)
					act.errChan <- err
					return err
				case clientError:
					return react.err
				case clientRead:
					//test for read error
					if react.err != nil {
						return react.err
					}
				}
				if react.data[0] != act.data[0] {
					atomic.AddInt64(&c.com.Stats().IDDrops, 1)
					debugf("FailoverRTUClient unexpected slaveId:%v in %v\n", act.data[0], hex.EncodeToString(react.data))
					break SELECT
				}
				rp, err := react.data.GetPDU()
				if err != nil {
					if err == ErrorCrc {
						atomic.AddInt64(&c.com.Stats().CrcErrors, 1)
					} else {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
					}
					act.errChan <- err
					break READ_LOOP
				}
				hasErr, fc := rp.GetFunctionCode().SeparateError()
				if hasErr && fc == afc {
					atomic.AddInt64(&c.com.Stats().RemoteErrors, 1)
					handler.OnError(ap, rp)
					act.errChan <- fmt.Errorf("server reply with exception:%v", hex.EncodeToString(rp))
					break READ_LOOP
				}
				if !IsRequestReply(act.data.fastGetPDU(), rp) {
					readUnexpected(act, func() {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
					})
					act.errChan <- fmt.Errorf("unexpected reply:%v", hex.EncodeToString(rp))
					break READ_LOOP
				}
				if afc.IsReadToServer() {
					//read from server, write here
					bs, err := rp.GetReplyValues()
					if err != nil {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
						act.errChan <- err
						break READ_LOOP
					}
					err = handler.OnWrite(ap, bs)
					if err != nil {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
					}
					act.errChan <- err //success if nil
					break READ_LOOP
				}
				act.errChan <- nil //success
				break READ_LOOP
			}
		}
	}
}

//Close closes the client and closes the connect
func (c *FailoverRTUClient) Close() error {
	return c.com.Close()
}

//DoTransaction starts a transaction, and returns a channel that returns an error
//or nil, with the default slaveID.
//
//DoTransaction is blocking.
//
//For read from server, the PDU is sent as is (after been warped up in RTU)
//For write to server, the data part given will be ignored, and filled in by data from handler.
func (c *FailoverRTUClient) DoTransaction(req PDU) error {
	errChan := make(chan error)
	c.StartTransactionToServer(c.SlaveID, req, errChan)
	return <-errChan
}

//StartTransactionToServer starts a transaction, with a custom slaveID.
//errChan is required and usable, an error is set is the transaction failed, or
//nil for success.
//
//StartTransactionToServer is not blocking.
//
//For read from server, the PDU is sent as is (after been warped up in RTU)
//For write to server, the data part given will be ignored, and filled in by data from handler.
func (c *FailoverRTUClient) StartTransactionToServer(slaveID byte, req PDU, errChan chan error) {
	c.actions <- rtuAction{t: clientStart, data: MakeRTU(slaveID, req), errChan: errChan}
}
<commit_msg>add debug log<commit_after>package modbusone

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"sync/atomic"
	"time"
)

//FailoverRTUClient implements Client/Master side logic for RTU over a SerialContext to
//be used by a ProtocolHandler with failover function.
type FailoverRTUClient struct {
	com                  *FailoverSerialConn
	packetReader         PacketReader
	SlaveID              byte
	serverProcessingTime time.Duration
	actions              chan rtuAction
}

//FailoverRTUClient is also a Server
var _ Server = &FailoverRTUClient{}

//NewFailoverRTUClient create a new client with failover function communicating over SerialContext with the
//give slaveID as default.
//
//If isFailover is true, it is the secondary.
func NewFailoverRTUClient(com SerialContext, isFailover bool, slaveID byte) *FailoverRTUClient {
	pr, ok := com.(*FailoverSerialConn)
	if !ok {
		pr = NewFailoverConn(com, isFailover, true)
	}
	if pr.isFailover != isFailover {
		panic("A SerialContext was provided with conflicting settings.")
	}
	r := FailoverRTUClient{
		com:                  pr,
		packetReader:         pr,
		SlaveID:              slaveID,
		serverProcessingTime: time.Second,
		actions:              make(chan rtuAction),
	}
	return &r
}

//SetServerProcessingTime sets the time to wait for a server response, the total
//wait time also includes the time needed for data transmission
func (c *FailoverRTUClient) SetServerProcessingTime(t time.Duration) {
	c.serverProcessingTime = t
}

//GetTransactionTimeOut returns the total time to wait for a transaction
//(server response) to time out, given the expected length of RTU packets.
//This function is also used internally to calculate timeout.
func (c *FailoverRTUClient) GetTransactionTimeOut(reqLen, ansLen int) time.Duration {
	l := reqLen + ansLen
	return c.com.BytesDelay(l) + c.serverProcessingTime
}

//Serve serves FailoverRTUClient side handlers,
//
//A FailoverRTUClient expects a lot of "unexpected" read packets and "lost" writes so it
//is does not do the error checking that a normal client does, but instead try to guess the best
//interpretation.
func (c *FailoverRTUClient) Serve(handler ProtocolHandler) error {
	debugf("serve routine for %v", c.com.describe())
	defer c.Close()
	go func() {
		debugf("reader routine for %v", c.com.describe())
		//Reader loop that always ready to received data. This make sure that read
		//data is always new(ish), to dump data out that is received during an
		//unexpected time.
		for {
			rb := make([]byte, MaxRTUSize)
			n, err := c.packetReader.Read(rb)
			if err != nil {
				debugf("FailoverRTUClient read err:%v\n", err)
				c.actions <- rtuAction{t: clientError, err: err}
				c.Close()
				break
			}
			r := RTU(rb[:n])
			debugf("FailoverRTUClient read packet:%v\n", hex.EncodeToString(r))
			c.actions <- rtuAction{t: clientRead, data: r}
		}
	}()

	var last bytes.Buffer
	readUnexpected := func(act rtuAction, otherwise func()) {
		if act.err != nil || act.t != clientRead || len(act.data) == 0 {
			debugf("do not hand unexpected: %v", act)
			otherwise()
			return
		}
		debugf("handling unexpected: %v", act)
		pdu, err := act.data.GetPDU()
		if err != nil {
			debugf("readUnexpected GetPDU error: %v", err)
			otherwise()
			return
		}
		if !IsRequestReply(last.Bytes(), pdu) {
			if last.Len() != 0 {
				atomic.AddInt64(&c.com.Stats().OtherDrops, 1)
			}
			debugf("failover client serve set last: %x", pdu)
			last.Reset()
			last.Write(pdu)
			return
		}
		defer last.Reset()

		if pdu.GetFunctionCode().IsWriteToServer() {
			//no-op for us
			return
		}

		bs, err := pdu.GetReplyValues()
		if err != nil {
			debugf("readUnexpected GetReplyValues error: %v", err)
			otherwise()
			return
		}
		err = handler.OnWrite(last.Bytes(), bs)
		if err != nil {
			debugf("readUnexpected OnWrite error: %v", err)
			otherwise()
			return
		}
	}

	for {
		act := <-c.actions
		switch act.t {
		default:
			readUnexpected(act, func() {
				atomic.AddInt64(&c.com.Stats().OtherDrops, 1)
				debugf("FailoverRTUClient drop unexpected: %v", act)
			})
			continue
		case clientError:
			return act.err
		case clientStart:
		}
		ap := act.data.fastGetPDU()
		afc := ap.GetFunctionCode()
		if afc.IsWriteToServer() {
			data, err := handler.OnRead(ap)
			if err != nil {
				act.errChan <- err
				continue
			}
			act.data = MakeRTU(act.data[0], ap.MakeWriteRequest(data))
			ap = act.data.fastGetPDU()
		}
		time.Sleep(c.com.MinDelay())
		_, err := c.com.Write(act.data)
		if err != nil {
			act.errChan <- err
			return err
		}
		c.com.lock.Lock()
		active := c.com.isActive
		c.com.lock.Unlock()
		if act.data[0] == 0 || !active {
			debugf("FailoverRTUClient skip action:%v\n", act)
			time.Sleep(c.com.BytesDelay(len(act.data)) + c.serverProcessingTime)
			act.errChan <- nil //always success
			continue           // do not wait for read on multicast or when not active
		}

		timeOutChan := time.After(c.GetTransactionTimeOut(len(act.data), MaxRTUSize))

	READ_LOOP:
		for {
		SELECT:
			select {
			case <-timeOutChan:
				act.errChan <- ErrServerTimeOut
				break READ_LOOP
			case react := <-c.actions:
				switch react.t {
				default:
					err := fmt.Errorf("unexpected action:%s", react.t)
					act.errChan <- err
					return err
				case clientError:
					return react.err
				case clientRead:
					//test for read error
					if react.err != nil {
						return react.err
					}
				}
				if react.data[0] != act.data[0] {
					atomic.AddInt64(&c.com.Stats().IDDrops, 1)
					debugf("FailoverRTUClient unexpected slaveId:%v in %v\n", act.data[0], hex.EncodeToString(react.data))
					break SELECT
				}
				rp, err := react.data.GetPDU()
				if err != nil {
					if err == ErrorCrc {
						atomic.AddInt64(&c.com.Stats().CrcErrors, 1)
					} else {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
					}
					act.errChan <- err
					break READ_LOOP
				}
				hasErr, fc := rp.GetFunctionCode().SeparateError()
				if hasErr && fc == afc {
					atomic.AddInt64(&c.com.Stats().RemoteErrors, 1)
					handler.OnError(ap, rp)
					act.errChan <- fmt.Errorf("server reply with exception:%v", hex.EncodeToString(rp))
					break READ_LOOP
				}
				if !IsRequestReply(act.data.fastGetPDU(), rp) {
					readUnexpected(act, func() {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
					})
					act.errChan <- fmt.Errorf("unexpected reply:%v", hex.EncodeToString(rp))
					break READ_LOOP
				}
				if afc.IsReadToServer() {
					//read from server, write here
					bs, err := rp.GetReplyValues()
					if err != nil {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
						act.errChan <- err
						break READ_LOOP
					}
					err = handler.OnWrite(ap, bs)
					if err != nil {
						atomic.AddInt64(&c.com.Stats().OtherErrors, 1)
					}
					act.errChan <- err //success if nil
					break READ_LOOP
				}
				act.errChan <- nil //success
				break READ_LOOP
			}
		}
	}
}

//Close closes the client and closes the connect
func (c *FailoverRTUClient) Close() error {
	return c.com.Close()
}

//DoTransaction starts a transaction, and returns a channel that returns an error
//or nil, with the default slaveID.
//
//DoTransaction is blocking.
//
//For read from server, the PDU is sent as is (after been warped up in RTU)
//For write to server, the data part given will be ignored, and filled in by data from handler.
func (c *FailoverRTUClient) DoTransaction(req PDU) error {
	errChan := make(chan error)
	c.StartTransactionToServer(c.SlaveID, req, errChan)
	return <-errChan
}

//StartTransactionToServer starts a transaction, with a custom slaveID.
//errChan is required and usable, an error is set is the transaction failed, or
//nil for success.
//
//StartTransactionToServer is not blocking.
//
//For read from server, the PDU is sent as is (after been warped up in RTU)
//For write to server, the data part given will be ignored, and filled in by data from handler.
func (c *FailoverRTUClient) StartTransactionToServer(slaveID byte, req PDU, errChan chan error) {
	c.actions <- rtuAction{t: clientStart, data: MakeRTU(slaveID, req), errChan: errChan}
}
<|endoftext|>


<commit_before>package utils

import (
	"github.com/stretchr/testify/assert"
	"gopkg.in/yaml.v2"
	"testing"
)

func TestGetBool(t *testing.T) {
	val, err := GetAsBool("false", true)
	assert.Equal(t, val, false)
	assert.Nil(t, err)

	val, err = GetAsBool("notabool", false)
	assert.Equal(t, val, false)
	assert.NotNil(t, err)

	val, err = GetAsBool(true, false)
	assert.Equal(t, val, true)
	assert.Nil(t, err)

	val, err = GetAsBool("True", false)
	assert.Equal(t, val, true)
	assert.Nil(t, err)
}

func TestGetInt(t *testing.T) {
	val, err := GetAsInt("10", 123)
	assert.Equal(t, val, 10)
	assert.Nil(t, err)

	val, err = GetAsInt("notanint", 123)
	assert.Equal(t, val, 123)
	assert.NotNil(t, err)

	val, err = GetAsInt(12.123, 123)
	assert.Equal(t, val, 12)
	assert.Nil(t, err)

	val, err = GetAsInt(12, 123)
	assert.Equal(t, val, 12)
	assert.Nil(t, err)
}

func TestGetFloat(t *testing.T) {
	val, err := GetAsFloat("10", 123)
	assert.Equal(t, val, 10.0)
	assert.Nil(t, err)

	val, err = GetAsFloat("10.21", 123)
	assert.Equal(t, val, 10.21)
	assert.Nil(t, err)

	val, err = GetAsFloat("notafloat", 123)
	assert.Equal(t, val, 123.0)
	assert.NotNil(t, err)

	val, err = GetAsFloat(12.123, 123)
	assert.Equal(t, val, 12.123)
	assert.Nil(t, err)
}

func TestGetString(t *testing.T) {
	val := GetAsString("10")
	assert.Equal(t, val, "10")

	val = GetAsString(10)
	assert.Equal(t, val, "10")

	val = GetAsString(10.123)
	assert.Equal(t, val, "10.123")
}

func TestGetAsMap(t *testing.T) {
	// Test if string can be converted to map[string]string
	stringToParse := "{\"foo\" : \"bar\", \"alice\":\"bob\"}"
	expectedValue := map[string]string{
		"foo":   "bar",
		"alice": "bob",
	}
	actualValue, err := GetAsMap(stringToParse)
	assert.Equal(t, actualValue, expectedValue)

	// Test if map[string]interface{} can be converted to map[string]string
	interfaceMapToParse := make(map[string]interface{})
	interfaceMapToParse["foo"] = "bar"
	interfaceMapToParse["alice"] = "bob"

	actualValue, err = GetAsMap(interfaceMapToParse)
	assert.Equal(t, actualValue, expectedValue)

	actualValue, err = GetAsMap(123)
	assert.NotNil(t, err)
}

func TestGetAsSlice(t *testing.T) {
	// Test if string array can be converted to []string
	stringToParse := "[\"baz\", \"bat\"]"
	expectedValue := []string{"baz", "bat"}
	actualValue, err := GetAsSlice(stringToParse)
	assert.Equal(t, actualValue, expectedValue)

	sliceToParse := []string{"baz", "bat"}
	actualValue, err = GetAsSlice(sliceToParse)
	assert.Equal(t, actualValue, expectedValue)

	actualValue, err = GetAsSlice(123)
	assert.NotNil(t, err)
}

func TestGetAsSliceFromYAML(t *testing.T) {
	var data map[string]interface{}
	yamlString := []byte(`{"listOfStrings": ["a", "b", "c"]}`)

	err := yaml.Unmarshal(yamlString, &data)
	assert.Nil(t, err)

	if err == nil {
		temp := data.(map[string]interface{})

		res, err := GetAsSlice(temp["listOfStrings"])
		assert.Equal(t, []string{"a", "b", "c"}, res)

		res, err = GetAsSlice(123)
		assert.NotNil(t, err)
	}
}
<commit_msg>Tests should work now<commit_after>package utils

import (
	"github.com/stretchr/testify/assert"
	"gopkg.in/yaml.v2"
	"testing"
)

func TestGetBool(t *testing.T) {
	val, err := GetAsBool("false", true)
	assert.Equal(t, val, false)
	assert.Nil(t, err)

	val, err = GetAsBool("notabool", false)
	assert.Equal(t, val, false)
	assert.NotNil(t, err)

	val, err = GetAsBool(true, false)
	assert.Equal(t, val, true)
	assert.Nil(t, err)

	val, err = GetAsBool("True", false)
	assert.Equal(t, val, true)
	assert.Nil(t, err)
}

func TestGetInt(t *testing.T) {
	val, err := GetAsInt("10", 123)
	assert.Equal(t, val, 10)
	assert.Nil(t, err)

	val, err = GetAsInt("notanint", 123)
	assert.Equal(t, val, 123)
	assert.NotNil(t, err)

	val, err = GetAsInt(12.123, 123)
	assert.Equal(t, val, 12)
	assert.Nil(t, err)

	val, err = GetAsInt(12, 123)
	assert.Equal(t, val, 12)
	assert.Nil(t, err)
}

func TestGetFloat(t *testing.T) {
	val, err := GetAsFloat("10", 123)
	assert.Equal(t, val, 10.0)
	assert.Nil(t, err)

	val, err = GetAsFloat("10.21", 123)
	assert.Equal(t, val, 10.21)
	assert.Nil(t, err)

	val, err = GetAsFloat("notafloat", 123)
	assert.Equal(t, val, 123.0)
	assert.NotNil(t, err)

	val, err = GetAsFloat(12.123, 123)
	assert.Equal(t, val, 12.123)
	assert.Nil(t, err)
}

func TestGetString(t *testing.T) {
	val := GetAsString("10")
	assert.Equal(t, val, "10")

	val = GetAsString(10)
	assert.Equal(t, val, "10")

	val = GetAsString(10.123)
	assert.Equal(t, val, "10.123")
}

func TestGetAsMap(t *testing.T) {
	// Test if string can be converted to map[string]string
	stringToParse := "{\"foo\" : \"bar\", \"alice\":\"bob\"}"
	expectedValue := map[string]string{
		"foo":   "bar",
		"alice": "bob",
	}
	actualValue, err := GetAsMap(stringToParse)
	assert.Equal(t, actualValue, expectedValue)

	// Test if map[string]interface{} can be converted to map[string]string
	interfaceMapToParse := make(map[string]interface{})
	interfaceMapToParse["foo"] = "bar"
	interfaceMapToParse["alice"] = "bob"

	actualValue, err = GetAsMap(interfaceMapToParse)
	assert.Equal(t, actualValue, expectedValue)

	actualValue, err = GetAsMap(123)
	assert.NotNil(t, err)
}

func TestGetAsSlice(t *testing.T) {
	// Test if string array can be converted to []string
	stringToParse := "[\"baz\", \"bat\"]"
	expectedValue := []string{"baz", "bat"}
	actualValue, err := GetAsSlice(stringToParse)
	assert.Equal(t, actualValue, expectedValue)

	sliceToParse := []string{"baz", "bat"}
	actualValue, err = GetAsSlice(sliceToParse)
	assert.Equal(t, actualValue, expectedValue)

	actualValue, err = GetAsSlice(123)
	assert.NotNil(t, err)
}

func TestGetAsSliceFromYAML(t *testing.T) {
	var data map[string]interface{}
	yamlString := []byte(`{"listOfStrings": ["a", "b", "c"]}`)

	err := yaml.Unmarshal(yamlString, &data)
	assert.Nil(t, err)

	if err == nil {
		temp := data

		res, err := GetAsSlice(temp["listOfStrings"])
		assert.Equal(t, []string{"a", "b", "c"}, res)

		res, err = GetAsSlice(123)
		assert.NotNil(t, err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acmechallenges

import (
	"context"
	"time"

	"github.com/go-logr/logr"
	corev1 "k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	"github.com/cert-manager/cert-manager/internal/ingress"
	"github.com/cert-manager/cert-manager/pkg/acme/accounts"
	cmacme "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
	cmclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
	cmacmelisters "github.com/cert-manager/cert-manager/pkg/client/listers/acme/v1"
	cmlisters "github.com/cert-manager/cert-manager/pkg/client/listers/certmanager/v1"
	controllerpkg "github.com/cert-manager/cert-manager/pkg/controller"
	"github.com/cert-manager/cert-manager/pkg/controller/acmechallenges/scheduler"
	"github.com/cert-manager/cert-manager/pkg/issuer"
	"github.com/cert-manager/cert-manager/pkg/issuer/acme/dns"
	"github.com/cert-manager/cert-manager/pkg/issuer/acme/http"
	logf "github.com/cert-manager/cert-manager/pkg/logs"
)

type controller struct {
	// issuer helper is used to obtain references to issuers, used by Sync()
	helper issuer.Helper

	// used to fetch ACME clients used in the controller
	accountRegistry accounts.Getter

	// all the listers used by this controller
	challengeLister     cmacmelisters.ChallengeLister
	issuerLister        cmlisters.IssuerLister
	clusterIssuerLister cmlisters.ClusterIssuerLister
	secretLister        corelisters.SecretLister

	// fieldManager is the manager name used for the Apply operations.
	fieldManager string

	// ACME challenge solvers are instantiated once at the time of controller
	// construction.
	// This also allows for easy mocking of the different challenge mechanisms.
	dnsSolver  solver
	httpSolver solver
	// scheduler marks challenges as Processing=true if they can be scheduled
	// for processing. This job runs periodically every N seconds, so it cannot
	// be constructed as a traditional controller.
	scheduler *scheduler.Scheduler

	// used to record Events about resources to the API
	recorder record.EventRecorder
	// clientset used to update cert-manager API resources
	cmClient cmclient.Interface

	// maintain a reference to the workqueue for this controller
	// so the handleOwnedResource method can enqueue resources
	queue workqueue.RateLimitingInterface

	// logger to be used by this controller
	log logr.Logger

	dns01Nameservers []string

	DNS01CheckRetryPeriod time.Duration
}

func (c *controller) Register(ctx *controllerpkg.Context) (workqueue.RateLimitingInterface, []cache.InformerSynced, error) {
	// construct a new named logger to be reused throughout the controller
	c.log = logf.FromContext(ctx.RootContext, ControllerName)

	// create a queue used to queue up items to be processed
	c.queue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(time.Second*5, time.Minute*30), ControllerName)

	// obtain references to all the informers used by this controller
	challengeInformer := ctx.SharedInformerFactory.Acme().V1().Challenges()
	issuerInformer := ctx.SharedInformerFactory.Certmanager().V1().Issuers()
	secretInformer := ctx.KubeSharedInformerFactory.Core().V1().Secrets()
	// we register these informers here so the HTTP01 solver has a synced
	// cache when managing pod/service/ingress resources
	podInformer := ctx.KubeSharedInformerFactory.Core().V1().Pods()
	serviceInformer := ctx.KubeSharedInformerFactory.Core().V1().Services()

	_, ingressInformer, err := ingress.NewListerInformer(ctx)
	if err != nil {
		return nil, nil, err
	}

	// build a list of InformerSynced functions that will be returned by the Register method.
	// the controller will only begin processing items once all of these informers have synced.
	mustSync := []cache.InformerSynced{
		challengeInformer.Informer().HasSynced,
		issuerInformer.Informer().HasSynced,
		secretInformer.Informer().HasSynced,
		podInformer.Informer().HasSynced,
		serviceInformer.Informer().HasSynced,
		ingressInformer.HasSynced,
	}

	if ctx.GatewaySolverEnabled {
		gwAPIHTTPRouteInformer := ctx.GWShared.Gateway().V1alpha2().HTTPRoutes()
		mustSync = append(mustSync, gwAPIHTTPRouteInformer.Informer().HasSynced)
	}

	// set all the references to the listers for used by the Sync function
	c.challengeLister = challengeInformer.Lister()
	c.issuerLister = issuerInformer.Lister()
	c.secretLister = secretInformer.Lister()

	// if we are running in non-namespaced mode (i.e. --namespace=""), we also
	// register event handlers and obtain a lister for clusterissuers.
	if ctx.Namespace == "" {
		clusterIssuerInformer := ctx.SharedInformerFactory.Certmanager().V1().ClusterIssuers()
		mustSync = append(mustSync, clusterIssuerInformer.Informer().HasSynced)
		c.clusterIssuerLister = clusterIssuerInformer.Lister()
	}

	// register handler functions
	challengeInformer.Informer().AddEventHandler(&controllerpkg.QueuingEventHandler{Queue: c.queue})

	c.helper = issuer.NewHelper(c.issuerLister, c.clusterIssuerLister)
	c.scheduler = scheduler.New(logf.NewContext(ctx.RootContext, c.log), c.challengeLister, ctx.SchedulerOptions.MaxConcurrentChallenges)
	c.recorder = ctx.Recorder
	c.cmClient = ctx.CMClient
	c.fieldManager = ctx.FieldManager
	c.accountRegistry = ctx.ACMEOptions.AccountRegistry

	c.httpSolver, err = http.NewSolver(ctx)
	if err != nil {
		return nil, nil, err
	}
	c.dnsSolver, err = dns.NewSolver(ctx)
	if err != nil {
		return nil, nil, err
	}

	// read options from context
	c.dns01Nameservers = ctx.ACMEOptions.DNS01Nameservers
	c.DNS01CheckRetryPeriod = ctx.ACMEOptions.DNS01CheckRetryPeriod

	return c.queue, mustSync, nil
}

// MaxChallengesPerSchedule is the maximum number of challenges that can be
// scheduled with a single call to the scheduler.
// This provides a very crude rate limit on how many challenges we will schedule
// per second. It may be better to remove this altogether in favour of some
// other method of rate limiting creations.
// TODO: make this configurable
const MaxChallengesPerSchedule = 20

// runScheduler will execute the scheduler's ScheduleN function to determine
// which, if any, challenges should be rescheduled.
// TODO: it should also only re-run the scheduler if a change to challenges has
// been observed, to save needless work
func (c *controller) runScheduler(ctx context.Context) {
	log := logf.FromContext(ctx, "scheduler")

	toSchedule, err := c.scheduler.ScheduleN(MaxChallengesPerSchedule)
	if err != nil {
		log.Error(err, "error determining set of challenges that should be scheduled for processing")
		return
	}

	for _, ch := range toSchedule {
		log := logf.WithResource(log, ch)
		ch = ch.DeepCopy()
		// Apply a finalizer, to ensure that the challenge is not
		// garbage collected before cert-manager has a chance to clean
		// up resources created for the challenge.
		hasFinalizer := false
		for _, finalizer := range ch.Finalizers {
			if finalizer == cmacme.ACMEFinalizer {
				hasFinalizer = true
				break
			}
		}
		if !hasFinalizer {
			ch.Finalizers = append(ch.Finalizers, cmacme.ACMEFinalizer)
			_, updateErr := c.updateOrApply(ctx, ch)
			if updateErr != nil {
				log.Error(err, "error applying finalizer to the challenge")
				return
			}
		}

		ch.Status.Processing = true
		_, err := c.updateStatusOrApply(ctx, ch)
		if err != nil {
			log.Error(err, "error scheduling challenge for processing")
			return
		}

		c.recorder.Event(ch, corev1.EventTypeNormal, "Started", "Challenge scheduled for processing")
	}

	if len(toSchedule) > 0 {
		log.V(logf.DebugLevel).Info("scheduled challenges for processing", "number_scheduled", len(toSchedule))
	}
}

func (c *controller) ProcessItem(ctx context.Context, key string) error {
	log := logf.FromContext(ctx)
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		log.Error(err, "invalid resource key")
		return nil
	}

	ch, err := c.challengeLister.Challenges(namespace).Get(name)

	if err != nil {
		if k8sErrors.IsNotFound(err) {
			log.Error(err, "challenge in work queue no longer exists")
			return nil
		}

		return err
	}

	ctx = logf.NewContext(ctx, logf.WithResource(log, ch))
	return c.Sync(ctx, ch)
}

const (
	ControllerName = "challenges"
)

func init() {
	controllerpkg.Register(ControllerName, func(ctx *controllerpkg.ContextFactory) (controllerpkg.Interface, error) {
		c := &controller{}
		return controllerpkg.NewBuilder(ctx, ControllerName).
			For(c).
			With(c.runScheduler, time.Second).
			Complete()
	})
}
<commit_msg>Fix the error is reported to null when it happens<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acmechallenges

import (
	"context"
	"time"

	"github.com/go-logr/logr"
	corev1 "k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	"github.com/cert-manager/cert-manager/internal/ingress"
	"github.com/cert-manager/cert-manager/pkg/acme/accounts"
	cmacme "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
	cmclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
	cmacmelisters "github.com/cert-manager/cert-manager/pkg/client/listers/acme/v1"
	cmlisters "github.com/cert-manager/cert-manager/pkg/client/listers/certmanager/v1"
	controllerpkg "github.com/cert-manager/cert-manager/pkg/controller"
	"github.com/cert-manager/cert-manager/pkg/controller/acmechallenges/scheduler"
	"github.com/cert-manager/cert-manager/pkg/issuer"
	"github.com/cert-manager/cert-manager/pkg/issuer/acme/dns"
	"github.com/cert-manager/cert-manager/pkg/issuer/acme/http"
	logf "github.com/cert-manager/cert-manager/pkg/logs"
)

type controller struct {
	// issuer helper is used to obtain references to issuers, used by Sync()
	helper issuer.Helper

	// used to fetch ACME clients used in the controller
	accountRegistry accounts.Getter

	// all the listers used by this controller
	challengeLister     cmacmelisters.ChallengeLister
	issuerLister        cmlisters.IssuerLister
	clusterIssuerLister cmlisters.ClusterIssuerLister
	secretLister        corelisters.SecretLister

	// fieldManager is the manager name used for the Apply operations.
	fieldManager string

	// ACME challenge solvers are instantiated once at the time of controller
	// construction.
	// This also allows for easy mocking of the different challenge mechanisms.
	dnsSolver  solver
	httpSolver solver
	// scheduler marks challenges as Processing=true if they can be scheduled
	// for processing. This job runs periodically every N seconds, so it cannot
	// be constructed as a traditional controller.
	scheduler *scheduler.Scheduler

	// used to record Events about resources to the API
	recorder record.EventRecorder
	// clientset used to update cert-manager API resources
	cmClient cmclient.Interface

	// maintain a reference to the workqueue for this controller
	// so the handleOwnedResource method can enqueue resources
	queue workqueue.RateLimitingInterface

	// logger to be used by this controller
	log logr.Logger

	dns01Nameservers []string

	DNS01CheckRetryPeriod time.Duration
}

func (c *controller) Register(ctx *controllerpkg.Context) (workqueue.RateLimitingInterface, []cache.InformerSynced, error) {
	// construct a new named logger to be reused throughout the controller
	c.log = logf.FromContext(ctx.RootContext, ControllerName)

	// create a queue used to queue up items to be processed
	c.queue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(time.Second*5, time.Minute*30), ControllerName)

	// obtain references to all the informers used by this controller
	challengeInformer := ctx.SharedInformerFactory.Acme().V1().Challenges()
	issuerInformer := ctx.SharedInformerFactory.Certmanager().V1().Issuers()
	secretInformer := ctx.KubeSharedInformerFactory.Core().V1().Secrets()
	// we register these informers here so the HTTP01 solver has a synced
	// cache when managing pod/service/ingress resources
	podInformer := ctx.KubeSharedInformerFactory.Core().V1().Pods()
	serviceInformer := ctx.KubeSharedInformerFactory.Core().V1().Services()

	_, ingressInformer, err := ingress.NewListerInformer(ctx)
	if err != nil {
		return nil, nil, err
	}

	// build a list of InformerSynced functions that will be returned by the Register method.
	// the controller will only begin processing items once all of these informers have synced.
	mustSync := []cache.InformerSynced{
		challengeInformer.Informer().HasSynced,
		issuerInformer.Informer().HasSynced,
		secretInformer.Informer().HasSynced,
		podInformer.Informer().HasSynced,
		serviceInformer.Informer().HasSynced,
		ingressInformer.HasSynced,
	}

	if ctx.GatewaySolverEnabled {
		gwAPIHTTPRouteInformer := ctx.GWShared.Gateway().V1alpha2().HTTPRoutes()
		mustSync = append(mustSync, gwAPIHTTPRouteInformer.Informer().HasSynced)
	}

	// set all the references to the listers for used by the Sync function
	c.challengeLister = challengeInformer.Lister()
	c.issuerLister = issuerInformer.Lister()
	c.secretLister = secretInformer.Lister()

	// if we are running in non-namespaced mode (i.e. --namespace=""), we also
	// register event handlers and obtain a lister for clusterissuers.
	if ctx.Namespace == "" {
		clusterIssuerInformer := ctx.SharedInformerFactory.Certmanager().V1().ClusterIssuers()
		mustSync = append(mustSync, clusterIssuerInformer.Informer().HasSynced)
		c.clusterIssuerLister = clusterIssuerInformer.Lister()
	}

	// register handler functions
	challengeInformer.Informer().AddEventHandler(&controllerpkg.QueuingEventHandler{Queue: c.queue})

	c.helper = issuer.NewHelper(c.issuerLister, c.clusterIssuerLister)
	c.scheduler = scheduler.New(logf.NewContext(ctx.RootContext, c.log), c.challengeLister, ctx.SchedulerOptions.MaxConcurrentChallenges)
	c.recorder = ctx.Recorder
	c.cmClient = ctx.CMClient
	c.fieldManager = ctx.FieldManager
	c.accountRegistry = ctx.ACMEOptions.AccountRegistry

	c.httpSolver, err = http.NewSolver(ctx)
	if err != nil {
		return nil, nil, err
	}
	c.dnsSolver, err = dns.NewSolver(ctx)
	if err != nil {
		return nil, nil, err
	}

	// read options from context
	c.dns01Nameservers = ctx.ACMEOptions.DNS01Nameservers
	c.DNS01CheckRetryPeriod = ctx.ACMEOptions.DNS01CheckRetryPeriod

	return c.queue, mustSync, nil
}

// MaxChallengesPerSchedule is the maximum number of challenges that can be
// scheduled with a single call to the scheduler.
// This provides a very crude rate limit on how many challenges we will schedule
// per second. It may be better to remove this altogether in favour of some
// other method of rate limiting creations.
// TODO: make this configurable
const MaxChallengesPerSchedule = 20

// runScheduler will execute the scheduler's ScheduleN function to determine
// which, if any, challenges should be rescheduled.
// TODO: it should also only re-run the scheduler if a change to challenges has
// been observed, to save needless work
func (c *controller) runScheduler(ctx context.Context) {
	log := logf.FromContext(ctx, "scheduler")

	toSchedule, err := c.scheduler.ScheduleN(MaxChallengesPerSchedule)
	if err != nil {
		log.Error(err, "error determining set of challenges that should be scheduled for processing")
		return
	}

	for _, ch := range toSchedule {
		log := logf.WithResource(log, ch)
		ch = ch.DeepCopy()
		// Apply a finalizer, to ensure that the challenge is not
		// garbage collected before cert-manager has a chance to clean
		// up resources created for the challenge.
		hasFinalizer := false
		for _, finalizer := range ch.Finalizers {
			if finalizer == cmacme.ACMEFinalizer {
				hasFinalizer = true
				break
			}
		}
		if !hasFinalizer {
			ch.Finalizers = append(ch.Finalizers, cmacme.ACMEFinalizer)
			_, updateErr := c.updateOrApply(ctx, ch)
			if updateErr != nil {
				log.Error(updateErr, "error applying finalizer to the challenge")
				return
			}
		}

		ch.Status.Processing = true
		_, err := c.updateStatusOrApply(ctx, ch)
		if err != nil {
			log.Error(err, "error scheduling challenge for processing")
			return
		}

		c.recorder.Event(ch, corev1.EventTypeNormal, "Started", "Challenge scheduled for processing")
	}

	if len(toSchedule) > 0 {
		log.V(logf.DebugLevel).Info("scheduled challenges for processing", "number_scheduled", len(toSchedule))
	}
}

func (c *controller) ProcessItem(ctx context.Context, key string) error {
	log := logf.FromContext(ctx)
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		log.Error(err, "invalid resource key")
		return nil
	}

	ch, err := c.challengeLister.Challenges(namespace).Get(name)

	if err != nil {
		if k8sErrors.IsNotFound(err) {
			log.Error(err, "challenge in work queue no longer exists")
			return nil
		}

		return err
	}

	ctx = logf.NewContext(ctx, logf.WithResource(log, ch))
	return c.Sync(ctx, ch)
}

const (
	ControllerName = "challenges"
)

func init() {
	controllerpkg.Register(ControllerName, func(ctx *controllerpkg.ContextFactory) (controllerpkg.Interface, error) {
		c := &controller{}
		return controllerpkg.NewBuilder(ctx, ControllerName).
			For(c).
			With(c.runScheduler, time.Second).
			Complete()
	})
}
<|endoftext|>


<commit_before>package sysregistriesv2

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/BurntSushi/toml"
	"github.com/containers/image/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/containers/image/docker/reference"
)

// systemRegistriesConfPath is the path to the system-wide registry
// configuration file and is used to add/subtract potential registries for
// obtaining images.  You can override this at build time with
// -ldflags '-X github.com/containers/image/sysregistries.systemRegistriesConfPath=$your_path'
var systemRegistriesConfPath = builtinRegistriesConfPath

// builtinRegistriesConfPath is the path to the registry configuration file.
// DO NOT change this, instead see systemRegistriesConfPath above.
const builtinRegistriesConfPath = "/etc/containers/registries.conf"

// Endpoint describes a remote location of a registry.
type Endpoint struct {
	// The endpoint's remote location.
	Location string `toml:"location"`
	// If true, certs verification will be skipped and HTTP (non-TLS)
	// connections will be allowed.
	Insecure bool `toml:"insecure"`
}

// RewriteReference will substitute the provided reference `prefix` to the
// endpoints `location` from the `ref` and creates a new named reference from it.
// The function errors if the newly created reference is not parsable.
func (e *Endpoint) RewriteReference(ref reference.Named, prefix string) (reference.Named, error) {
	refString := ref.String()
	if !refMatchesPrefix(refString, prefix) {
		return nil, fmt.Errorf("invalid prefix '%v' for reference '%v'", prefix, refString)
	}

	newNamedRef := strings.Replace(refString, prefix, e.Location, 1)
	newParsedRef, err := reference.ParseNamed(newNamedRef)
	if newParsedRef != nil {
		logrus.Debugf("reference rewritten from '%v' to '%v'", refString, newParsedRef.String())
	}
	if err != nil {
		return nil, errors.Wrapf(err, "error rewriting reference")
	}
	return newParsedRef, nil
}

// Registry represents a registry.
type Registry struct {
	// A registry is an Endpoint too
	Endpoint
	// The registry's mirrors.
	Mirrors []Endpoint `toml:"mirror"`
	// If true, pulling from the registry will be blocked.
	Blocked bool `toml:"blocked"`
	// If true, the registry can be used when pulling an unqualified image.
	Search bool `toml:"unqualified-search"`
	// Prefix is used for matching images, and to translate one namespace to
	// another.  If `Prefix="example.com/bar"`, `location="example.com/foo/bar"`
	// and we pull from "example.com/bar/myimage:latest", the image will
	// effectively be pulled from "example.com/foo/bar/myimage:latest".
	// If no Prefix is specified, it defaults to the specified location.
	Prefix string `toml:"prefix"`
}

// V1TOMLregistries is for backwards compatibility to sysregistries v1
type V1TOMLregistries struct {
	Registries []string `toml:"registries"`
}

// V1TOMLConfig is for backwards compatibility to sysregistries v1
type V1TOMLConfig struct {
	Search   V1TOMLregistries `toml:"search"`
	Insecure V1TOMLregistries `toml:"insecure"`
	Block    V1TOMLregistries `toml:"block"`
}

// tomlConfig is the data type used to unmarshal the toml config.
type tomlConfig struct {
	Registries []Registry `toml:"registry"`
	// backwards compatability to sysregistries v1
	V1TOMLConfig `toml:"registries"`
}

// InvalidRegistries represents an invalid registry configurations.  An example
// is when "registry.com" is defined multiple times in the configuration but
// with conflicting security settings.
type InvalidRegistries struct {
	s string
}

// Error returns the error string.
func (e *InvalidRegistries) Error() string {
	return e.s
}

// parseLocation parses the input string, performs some sanity checks and returns
// the sanitized input string.  An error is returned if the input string is
// empty or if contains an "http{s,}://" prefix.
func parseLocation(input string) (string, error) {
	trimmed := strings.TrimRight(input, "/")

	if trimmed == "" {
		return "", &InvalidRegistries{s: "invalid location: cannot be empty"}
	}

	if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") {
		msg := fmt.Sprintf("invalid location '%s': URI schemes are not supported", input)
		return "", &InvalidRegistries{s: msg}
	}

	return trimmed, nil
}

// getV1Registries transforms v1 registries in the config into an array of v2
// registries of type Registry.
func getV1Registries(config *tomlConfig) ([]Registry, error) {
	regMap := make(map[string]*Registry)
	// We must preserve the order of config.V1Registries.Search.Registries at least.  The order of the
	// other registries is not really important, but make it deterministic (the same for the same config file)
	// to minimize behavior inconsistency and not contribute to difficult-to-reproduce situations.
	registryOrder := []string{}

	getRegistry := func(location string) (*Registry, error) { // Note: _pointer_ to a long-lived object
		var err error
		location, err = parseLocation(location)
		if err != nil {
			return nil, err
		}
		reg, exists := regMap[location]
		if !exists {
			reg = &Registry{
				Endpoint: Endpoint{Location: location},
				Mirrors:  []Endpoint{},
				Prefix:   location,
			}
			regMap[location] = reg
			registryOrder = append(registryOrder, location)
		}
		return reg, nil
	}

	// Note: config.V1Registries.Search needs to be processed first to ensure registryOrder is populated in the right order
	// if one of the search registries is also in one of the other lists.
	for _, search := range config.V1TOMLConfig.Search.Registries {
		reg, err := getRegistry(search)
		if err != nil {
			return nil, err
		}
		reg.Search = true
	}
	for _, blocked := range config.V1TOMLConfig.Block.Registries {
		reg, err := getRegistry(blocked)
		if err != nil {
			return nil, err
		}
		reg.Blocked = true
	}
	for _, insecure := range config.V1TOMLConfig.Insecure.Registries {
		reg, err := getRegistry(insecure)
		if err != nil {
			return nil, err
		}
		reg.Insecure = true
	}

	registries := []Registry{}
	for _, location := range registryOrder {
		reg := regMap[location]
		registries = append(registries, *reg)
	}
	return registries, nil
}

// postProcessRegistries checks the consistency of all registries (e.g., set
// the Prefix to Location if not set) and applies conflict checks.  It returns an
// array of cleaned registries and error in case of conflicts.
func postProcessRegistries(regs []Registry) ([]Registry, error) {
	var registries []Registry
	regMap := make(map[string][]Registry)

	for _, reg := range regs {
		var err error

		// make sure Location and Prefix are valid
		reg.Location, err = parseLocation(reg.Location)
		if err != nil {
			return nil, err
		}

		if reg.Prefix == "" {
			reg.Prefix = reg.Location
		} else {
			reg.Prefix, err = parseLocation(reg.Prefix)
			if err != nil {
				return nil, err
			}
		}

		// make sure mirrors are valid
		for _, mir := range reg.Mirrors {
			mir.Location, err = parseLocation(mir.Location)
			if err != nil {
				return nil, err
			}
		}
		registries = append(registries, reg)
		regMap[reg.Location] = append(regMap[reg.Location], reg)
	}

	// Given a registry can be mentioned multiple times (e.g., to have
	// multiple prefixes backed by different mirrors), we need to make sure
	// there are no conflicts among them.
	//
	// Note: we need to iterate over the registries array to ensure a
	// deterministic behavior which is not guaranteed by maps.
	for _, reg := range registries {
		others, _ := regMap[reg.Location]
		for _, other := range others {
			if reg.Insecure != other.Insecure {
				msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'insecure' setting", reg.Location)

				return nil, &InvalidRegistries{s: msg}
			}
			if reg.Blocked != other.Blocked {
				msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'blocked' setting", reg.Location)
				return nil, &InvalidRegistries{s: msg}
			}
		}
	}

	return registries, nil
}

// getConfigPath returns the system-registries config path if specified.
// Otherwise, systemRegistriesConfPath is returned.
func getConfigPath(ctx *types.SystemContext) string {
	confPath := systemRegistriesConfPath
	if ctx != nil {
		if ctx.SystemRegistriesConfPath != "" {
			confPath = ctx.SystemRegistriesConfPath
		} else if ctx.RootForImplicitAbsolutePaths != "" {
			confPath = filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfPath)
		}
	}
	return confPath
}

// configMutex is used to synchronize concurrent accesses to configCache.
var configMutex = sync.Mutex{}

// configCache caches already loaded configs with config paths as keys and is
// used to avoid redudantly parsing configs. Concurrent accesses to the cache
// are synchronized via configMutex.
var configCache = make(map[string][]Registry)

// InvalidateCache invalidates the registry cache.  This function is meant to be
// used for long-running processes that need to reload potential changes made to
// the cached registry config files.
func InvalidateCache() {
	configMutex.Lock()
	defer configMutex.Unlock()
	configCache = make(map[string][]Registry)
}

// GetRegistries loads and returns the registries specified in the config.
// Note the parsed content of registry config files is cached.  For reloading,
// use `InvalidateCache` and re-call `GetRegistries`.
func GetRegistries(ctx *types.SystemContext) ([]Registry, error) {
	configPath := getConfigPath(ctx)

	configMutex.Lock()
	defer configMutex.Unlock()
	// if the config has already been loaded, return the cached registries
	if registries, inCache := configCache[configPath]; inCache {
		return registries, nil
	}

	// load the config
	config, err := loadRegistryConf(configPath)
	if err != nil {
		// Return an empty []Registry if we use the default config,
		// which implies that the config path of the SystemContext
		// isn't set.  Note: if ctx.SystemRegistriesConfPath points to
		// the default config, we will still return an error.
		if os.IsNotExist(err) && (ctx == nil || ctx.SystemRegistriesConfPath == "") {
			return []Registry{}, nil
		}
		return nil, err
	}

	registries := config.Registries

	// backwards compatibility for v1 configs
	v1Registries, err := getV1Registries(config)
	if err != nil {
		return nil, err
	}
	if len(v1Registries) > 0 {
		if len(registries) > 0 {
			return nil, &InvalidRegistries{s: "mixing sysregistry v1/v2 is not supported"}
		}
		registries = v1Registries
	}

	registries, err = postProcessRegistries(registries)
	if err != nil {
		return nil, err
	}

	// populate the cache
	configCache[configPath] = registries

	return registries, err
}

// FindUnqualifiedSearchRegistries returns all registries that are configured
// for unqualified image search (i.e., with Registry.Search == true).
func FindUnqualifiedSearchRegistries(ctx *types.SystemContext) ([]Registry, error) {
	registries, err := GetRegistries(ctx)
	if err != nil {
		return nil, err
	}

	unqualified := []Registry{}
	for _, reg := range registries {
		if reg.Search {
			unqualified = append(unqualified, reg)
		}
	}
	return unqualified, nil
}

// refMatchesPrefix returns true iff ref,
// which is a registry, repository namespace, repository or image reference (as formatted by
// reference.Domain(), reference.Named.Name() or reference.Reference.String()
// — note that this requires the name to start with an explicit hostname!),
// matches a Registry.Prefix value.
// (This is split from the caller primarily to make testing easier.)
func refMatchesPrefix(ref, prefix string) bool {
	switch {
	case len(ref) < len(prefix):
		return false
	case len(ref) == len(prefix):
		return ref == prefix
	case len(ref) > len(prefix):
		if !strings.HasPrefix(ref, prefix) {
			return false
		}
		c := ref[len(prefix)]
		// This allows "example.com:5000" to match "example.com",
		// which is unintended; that will get fixed eventually, DON'T RELY
		// ON THE CURRENT BEHAVIOR.
		return c == ':' || c == '/' || c == '@'
	default:
		panic("Internal error: impossible comparison outcome")
	}
}

// FindRegistry returns the Registry with the longest prefix for ref,
// which is a registry, repository namespace repository or image reference (as formatted by
// reference.Domain(), reference.Named.Name() or reference.Reference.String()
// — note that this requires the name to start with an explicit hostname!).
// If no Registry prefixes the image, nil is returned.
func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) {
	registries, err := GetRegistries(ctx)
	if err != nil {
		return nil, err
	}

	reg := Registry{}
	prefixLen := 0
	for _, r := range registries {
		if refMatchesPrefix(ref, r.Prefix) {
			length := len(r.Prefix)
			if length > prefixLen {
				reg = r
				prefixLen = length
			}
		}
	}
	if prefixLen != 0 {
		return &reg, nil
	}
	return nil, nil
}

// Reads the global registry file from the filesystem. Returns a byte array.
func readRegistryConf(configPath string) ([]byte, error) {
	configBytes, err := ioutil.ReadFile(configPath)
	return configBytes, err
}

// Used in unittests to parse custom configs without a types.SystemContext.
var readConf = readRegistryConf

// Loads the registry configuration file from the filesystem and then unmarshals
// it.  Returns the unmarshalled object.
func loadRegistryConf(configPath string) (*tomlConfig, error) {
	config := &tomlConfig{}

	configBytes, err := readConf(configPath)
	if err != nil {
		return nil, err
	}

	err = toml.Unmarshal(configBytes, &config)
	return config, err
}
<commit_msg>Only log the result of reference rewriting if the result is valid<commit_after>package sysregistriesv2

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/BurntSushi/toml"
	"github.com/containers/image/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/containers/image/docker/reference"
)

// systemRegistriesConfPath is the path to the system-wide registry
// configuration file and is used to add/subtract potential registries for
// obtaining images.  You can override this at build time with
// -ldflags '-X github.com/containers/image/sysregistries.systemRegistriesConfPath=$your_path'
var systemRegistriesConfPath = builtinRegistriesConfPath

// builtinRegistriesConfPath is the path to the registry configuration file.
// DO NOT change this, instead see systemRegistriesConfPath above.
const builtinRegistriesConfPath = "/etc/containers/registries.conf"

// Endpoint describes a remote location of a registry.
type Endpoint struct {
	// The endpoint's remote location.
	Location string `toml:"location"`
	// If true, certs verification will be skipped and HTTP (non-TLS)
	// connections will be allowed.
	Insecure bool `toml:"insecure"`
}

// RewriteReference will substitute the provided reference `prefix` to the
// endpoints `location` from the `ref` and creates a new named reference from it.
// The function errors if the newly created reference is not parsable.
func (e *Endpoint) RewriteReference(ref reference.Named, prefix string) (reference.Named, error) {
	refString := ref.String()
	if !refMatchesPrefix(refString, prefix) {
		return nil, fmt.Errorf("invalid prefix '%v' for reference '%v'", prefix, refString)
	}

	newNamedRef := strings.Replace(refString, prefix, e.Location, 1)
	newParsedRef, err := reference.ParseNamed(newNamedRef)
	if err != nil {
		return nil, errors.Wrapf(err, "error rewriting reference")
	}
	logrus.Debugf("reference rewritten from '%v' to '%v'", refString, newParsedRef.String())
	return newParsedRef, nil
}

// Registry represents a registry.
type Registry struct {
	// A registry is an Endpoint too
	Endpoint
	// The registry's mirrors.
	Mirrors []Endpoint `toml:"mirror"`
	// If true, pulling from the registry will be blocked.
	Blocked bool `toml:"blocked"`
	// If true, the registry can be used when pulling an unqualified image.
	Search bool `toml:"unqualified-search"`
	// Prefix is used for matching images, and to translate one namespace to
	// another.  If `Prefix="example.com/bar"`, `location="example.com/foo/bar"`
	// and we pull from "example.com/bar/myimage:latest", the image will
	// effectively be pulled from "example.com/foo/bar/myimage:latest".
	// If no Prefix is specified, it defaults to the specified location.
	Prefix string `toml:"prefix"`
}

// V1TOMLregistries is for backwards compatibility to sysregistries v1
type V1TOMLregistries struct {
	Registries []string `toml:"registries"`
}

// V1TOMLConfig is for backwards compatibility to sysregistries v1
type V1TOMLConfig struct {
	Search   V1TOMLregistries `toml:"search"`
	Insecure V1TOMLregistries `toml:"insecure"`
	Block    V1TOMLregistries `toml:"block"`
}

// tomlConfig is the data type used to unmarshal the toml config.
type tomlConfig struct {
	Registries []Registry `toml:"registry"`
	// backwards compatability to sysregistries v1
	V1TOMLConfig `toml:"registries"`
}

// InvalidRegistries represents an invalid registry configurations.  An example
// is when "registry.com" is defined multiple times in the configuration but
// with conflicting security settings.
type InvalidRegistries struct {
	s string
}

// Error returns the error string.
func (e *InvalidRegistries) Error() string {
	return e.s
}

// parseLocation parses the input string, performs some sanity checks and returns
// the sanitized input string.  An error is returned if the input string is
// empty or if contains an "http{s,}://" prefix.
func parseLocation(input string) (string, error) {
	trimmed := strings.TrimRight(input, "/")

	if trimmed == "" {
		return "", &InvalidRegistries{s: "invalid location: cannot be empty"}
	}

	if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") {
		msg := fmt.Sprintf("invalid location '%s': URI schemes are not supported", input)
		return "", &InvalidRegistries{s: msg}
	}

	return trimmed, nil
}

// getV1Registries transforms v1 registries in the config into an array of v2
// registries of type Registry.
func getV1Registries(config *tomlConfig) ([]Registry, error) {
	regMap := make(map[string]*Registry)
	// We must preserve the order of config.V1Registries.Search.Registries at least.  The order of the
	// other registries is not really important, but make it deterministic (the same for the same config file)
	// to minimize behavior inconsistency and not contribute to difficult-to-reproduce situations.
	registryOrder := []string{}

	getRegistry := func(location string) (*Registry, error) { // Note: _pointer_ to a long-lived object
		var err error
		location, err = parseLocation(location)
		if err != nil {
			return nil, err
		}
		reg, exists := regMap[location]
		if !exists {
			reg = &Registry{
				Endpoint: Endpoint{Location: location},
				Mirrors:  []Endpoint{},
				Prefix:   location,
			}
			regMap[location] = reg
			registryOrder = append(registryOrder, location)
		}
		return reg, nil
	}

	// Note: config.V1Registries.Search needs to be processed first to ensure registryOrder is populated in the right order
	// if one of the search registries is also in one of the other lists.
	for _, search := range config.V1TOMLConfig.Search.Registries {
		reg, err := getRegistry(search)
		if err != nil {
			return nil, err
		}
		reg.Search = true
	}
	for _, blocked := range config.V1TOMLConfig.Block.Registries {
		reg, err := getRegistry(blocked)
		if err != nil {
			return nil, err
		}
		reg.Blocked = true
	}
	for _, insecure := range config.V1TOMLConfig.Insecure.Registries {
		reg, err := getRegistry(insecure)
		if err != nil {
			return nil, err
		}
		reg.Insecure = true
	}

	registries := []Registry{}
	for _, location := range registryOrder {
		reg := regMap[location]
		registries = append(registries, *reg)
	}
	return registries, nil
}

// postProcessRegistries checks the consistency of all registries (e.g., set
// the Prefix to Location if not set) and applies conflict checks.  It returns an
// array of cleaned registries and error in case of conflicts.
func postProcessRegistries(regs []Registry) ([]Registry, error) {
	var registries []Registry
	regMap := make(map[string][]Registry)

	for _, reg := range regs {
		var err error

		// make sure Location and Prefix are valid
		reg.Location, err = parseLocation(reg.Location)
		if err != nil {
			return nil, err
		}

		if reg.Prefix == "" {
			reg.Prefix = reg.Location
		} else {
			reg.Prefix, err = parseLocation(reg.Prefix)
			if err != nil {
				return nil, err
			}
		}

		// make sure mirrors are valid
		for _, mir := range reg.Mirrors {
			mir.Location, err = parseLocation(mir.Location)
			if err != nil {
				return nil, err
			}
		}
		registries = append(registries, reg)
		regMap[reg.Location] = append(regMap[reg.Location], reg)
	}

	// Given a registry can be mentioned multiple times (e.g., to have
	// multiple prefixes backed by different mirrors), we need to make sure
	// there are no conflicts among them.
	//
	// Note: we need to iterate over the registries array to ensure a
	// deterministic behavior which is not guaranteed by maps.
	for _, reg := range registries {
		others, _ := regMap[reg.Location]
		for _, other := range others {
			if reg.Insecure != other.Insecure {
				msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'insecure' setting", reg.Location)

				return nil, &InvalidRegistries{s: msg}
			}
			if reg.Blocked != other.Blocked {
				msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'blocked' setting", reg.Location)
				return nil, &InvalidRegistries{s: msg}
			}
		}
	}

	return registries, nil
}

// getConfigPath returns the system-registries config path if specified.
// Otherwise, systemRegistriesConfPath is returned.
func getConfigPath(ctx *types.SystemContext) string {
	confPath := systemRegistriesConfPath
	if ctx != nil {
		if ctx.SystemRegistriesConfPath != "" {
			confPath = ctx.SystemRegistriesConfPath
		} else if ctx.RootForImplicitAbsolutePaths != "" {
			confPath = filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfPath)
		}
	}
	return confPath
}

// configMutex is used to synchronize concurrent accesses to configCache.
var configMutex = sync.Mutex{}

// configCache caches already loaded configs with config paths as keys and is
// used to avoid redudantly parsing configs. Concurrent accesses to the cache
// are synchronized via configMutex.
var configCache = make(map[string][]Registry)

// InvalidateCache invalidates the registry cache.  This function is meant to be
// used for long-running processes that need to reload potential changes made to
// the cached registry config files.
func InvalidateCache() {
	configMutex.Lock()
	defer configMutex.Unlock()
	configCache = make(map[string][]Registry)
}

// GetRegistries loads and returns the registries specified in the config.
// Note the parsed content of registry config files is cached.  For reloading,
// use `InvalidateCache` and re-call `GetRegistries`.
func GetRegistries(ctx *types.SystemContext) ([]Registry, error) {
	configPath := getConfigPath(ctx)

	configMutex.Lock()
	defer configMutex.Unlock()
	// if the config has already been loaded, return the cached registries
	if registries, inCache := configCache[configPath]; inCache {
		return registries, nil
	}

	// load the config
	config, err := loadRegistryConf(configPath)
	if err != nil {
		// Return an empty []Registry if we use the default config,
		// which implies that the config path of the SystemContext
		// isn't set.  Note: if ctx.SystemRegistriesConfPath points to
		// the default config, we will still return an error.
		if os.IsNotExist(err) && (ctx == nil || ctx.SystemRegistriesConfPath == "") {
			return []Registry{}, nil
		}
		return nil, err
	}

	registries := config.Registries

	// backwards compatibility for v1 configs
	v1Registries, err := getV1Registries(config)
	if err != nil {
		return nil, err
	}
	if len(v1Registries) > 0 {
		if len(registries) > 0 {
			return nil, &InvalidRegistries{s: "mixing sysregistry v1/v2 is not supported"}
		}
		registries = v1Registries
	}

	registries, err = postProcessRegistries(registries)
	if err != nil {
		return nil, err
	}

	// populate the cache
	configCache[configPath] = registries

	return registries, err
}

// FindUnqualifiedSearchRegistries returns all registries that are configured
// for unqualified image search (i.e., with Registry.Search == true).
func FindUnqualifiedSearchRegistries(ctx *types.SystemContext) ([]Registry, error) {
	registries, err := GetRegistries(ctx)
	if err != nil {
		return nil, err
	}

	unqualified := []Registry{}
	for _, reg := range registries {
		if reg.Search {
			unqualified = append(unqualified, reg)
		}
	}
	return unqualified, nil
}

// refMatchesPrefix returns true iff ref,
// which is a registry, repository namespace, repository or image reference (as formatted by
// reference.Domain(), reference.Named.Name() or reference.Reference.String()
// — note that this requires the name to start with an explicit hostname!),
// matches a Registry.Prefix value.
// (This is split from the caller primarily to make testing easier.)
func refMatchesPrefix(ref, prefix string) bool {
	switch {
	case len(ref) < len(prefix):
		return false
	case len(ref) == len(prefix):
		return ref == prefix
	case len(ref) > len(prefix):
		if !strings.HasPrefix(ref, prefix) {
			return false
		}
		c := ref[len(prefix)]
		// This allows "example.com:5000" to match "example.com",
		// which is unintended; that will get fixed eventually, DON'T RELY
		// ON THE CURRENT BEHAVIOR.
		return c == ':' || c == '/' || c == '@'
	default:
		panic("Internal error: impossible comparison outcome")
	}
}

// FindRegistry returns the Registry with the longest prefix for ref,
// which is a registry, repository namespace repository or image reference (as formatted by
// reference.Domain(), reference.Named.Name() or reference.Reference.String()
// — note that this requires the name to start with an explicit hostname!).
// If no Registry prefixes the image, nil is returned.
func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) {
	registries, err := GetRegistries(ctx)
	if err != nil {
		return nil, err
	}

	reg := Registry{}
	prefixLen := 0
	for _, r := range registries {
		if refMatchesPrefix(ref, r.Prefix) {
			length := len(r.Prefix)
			if length > prefixLen {
				reg = r
				prefixLen = length
			}
		}
	}
	if prefixLen != 0 {
		return &reg, nil
	}
	return nil, nil
}

// Reads the global registry file from the filesystem. Returns a byte array.
func readRegistryConf(configPath string) ([]byte, error) {
	configBytes, err := ioutil.ReadFile(configPath)
	return configBytes, err
}

// Used in unittests to parse custom configs without a types.SystemContext.
var readConf = readRegistryConf

// Loads the registry configuration file from the filesystem and then unmarshals
// it.  Returns the unmarshalled object.
func loadRegistryConf(configPath string) (*tomlConfig, error) {
	config := &tomlConfig{}

	configBytes, err := readConf(configPath)
	if err != nil {
		return nil, err
	}

	err = toml.Unmarshal(configBytes, &config)
	return config, err
}
<|endoftext|>