<commit_before>package main

import (
	"os"
	"fmt"
	"flag"
	"io"
	"log"
	"encoding/json"
	"link-select/add"
	"link-select/sel"
	"link-select/types"
)

var system types.ConfigRecord
var files types.ConfigRecord

var addLink string
var selectLink string

func init() {
	const (
		defaultAdd = "read"
		usageAdd = "add a link (to json file)"
		defaultSelect = "read"
		usageSelect = "select a link (from json file)"
	)

	flag.StringVar(&addLink, "add-link", defaultAdd, usageAdd)
	flag.StringVar(&addLink, "a", defaultAdd, usageAdd + " (shorthand)")
	flag.StringVar(&selectLink, "sel-link", defaultSelect, usageSelect)
	flag.StringVar(&selectLink, "s", defaultSelect, usageSelect + " (shorthand)")
}

// loadConfig loads the JSON configuration file (see link-select/types for
// further details).
func loadConfig() {
	configFile, err := os.Open("config.json")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error while opening config JSON file\n")
		log.Fatal(err)
		os.Exit(-1)
	}

	dec := json.NewDecoder(configFile)
	for {
		var c types.Config
		if err := dec.Decode(&c); err == io.EOF {
			break
		} else if err != nil {
			fmt.Fprintf(os.Stderr, "Error while parsing config JSON file\n")
			log.Fatal(err)
			os.Exit(-1)
		}

		system = c["system"]
		files = c["files"]
	}
}

// processArgs processes the command-line arguments (in essence, calls the
// functions dependent on the provided flags).
func processArgs(arg *flag.Flag) {
	fmt.Println(selectLink)

	var err error
	switch arg.Name {
	case "add-link":
		err = add.AddLink(files[addLink])
	case "a":
		err = add.AddLink(files[addLink])
	case "sel-link":
		err = sel.SelectLink(files[selectLink], system["browser"])
	case "s":
		err = sel.SelectLink(files[selectLink], system["browser"])
	default:
		fmt.Fprintf(os.Stderr, "USAGE: link-select " +
			"--add-link=WHAT | " +
			" --sel-link=WHAT\n")
		os.Exit(-1)
	}

	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "USAGE: link-select " +
			"--add-link=WHAT |" +
			" --sel-link=WHAT\n")
		os.Exit(-1)
	}

	flag.Parse()
	if flag.Parsed() {
		loadConfig()
    	flag.Visit(processArgs)
	} else {
		fmt.Fprintf(os.Stderr, "Error while parsing command-line arguments\n")
		os.Exit(-1)
	}
}<commit_msg>Remove debug output<commit_after>package main

import (
	"os"
	"fmt"
	"flag"
	"io"
	"log"
	"encoding/json"
	"link-select/add"
	"link-select/sel"
	"link-select/types"
)

var system types.ConfigRecord
var files types.ConfigRecord

var addLink string
var selectLink string

func init() {
	const (
		defaultAdd = "read"
		usageAdd = "add a link (to json file)"
		defaultSelect = "read"
		usageSelect = "select a link (from json file)"
	)

	flag.StringVar(&addLink, "add-link", defaultAdd, usageAdd)
	flag.StringVar(&addLink, "a", defaultAdd, usageAdd + " (shorthand)")
	flag.StringVar(&selectLink, "sel-link", defaultSelect, usageSelect)
	flag.StringVar(&selectLink, "s", defaultSelect, usageSelect + " (shorthand)")
}

// loadConfig loads the JSON configuration file (see link-select/types for
// further details).
func loadConfig() {
	configFile, err := os.Open("config.json")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error while opening config JSON file\n")
		log.Fatal(err)
		os.Exit(-1)
	}

	dec := json.NewDecoder(configFile)
	for {
		var c types.Config
		if err := dec.Decode(&c); err == io.EOF {
			break
		} else if err != nil {
			fmt.Fprintf(os.Stderr, "Error while parsing config JSON file\n")
			log.Fatal(err)
			os.Exit(-1)
		}

		system = c["system"]
		files = c["files"]
	}
}

// processArgs processes the command-line arguments (in essence, calls the
// functions dependent on the provided flags).
func processArgs(arg *flag.Flag) {
	var err error
	switch arg.Name {
	case "add-link":
		err = add.AddLink(files[addLink])
	case "a":
		err = add.AddLink(files[addLink])
	case "sel-link":
		err = sel.SelectLink(files[selectLink], system["browser"])
	case "s":
		err = sel.SelectLink(files[selectLink], system["browser"])
	default:
		fmt.Fprintf(os.Stderr, "USAGE: link-select " +
			"--add-link=WHAT | " +
			" --sel-link=WHAT\n")
		os.Exit(-1)
	}

	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "USAGE: link-select " +
			"--add-link=WHAT |" +
			" --sel-link=WHAT\n")
		os.Exit(-1)
	}

	flag.Parse()
	if flag.Parsed() {
		loadConfig()
    	flag.Visit(processArgs)
	} else {
		fmt.Fprintf(os.Stderr, "Error while parsing command-line arguments\n")
		os.Exit(-1)
	}
}<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rest

import (
	"context"
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/registry/rest"
	genericapiserver "k8s.io/apiserver/pkg/server"
	serverstorage "k8s.io/apiserver/pkg/server/storage"
	"k8s.io/client-go/informers"
	flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2"
	flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1beta2"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	"k8s.io/kubernetes/pkg/apis/flowcontrol"
	flowcontrolapisv1alpha1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1alpha1"
	flowcontrolapisv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1"
	flowcontrolapisv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2"
	"k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer"
	flowschemastore "k8s.io/kubernetes/pkg/registry/flowcontrol/flowschema/storage"
	prioritylevelconfigurationstore "k8s.io/kubernetes/pkg/registry/flowcontrol/prioritylevelconfiguration/storage"
)

var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{}

// RESTStorageProvider is a provider of REST storage
type RESTStorageProvider struct {
	InformerFactory informers.SharedInformerFactory
}

// PostStartHookName is the name of the post-start-hook provided by flow-control storage
const PostStartHookName = "priority-and-fairness-config-producer"

// NewRESTStorage creates a new rest storage for flow-control api models.
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool, error) {
	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(flowcontrol.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs)

	if apiResourceConfigSource.VersionEnabled(flowcontrolapisv1alpha1.SchemeGroupVersion) {
		flowControlStorage, err := p.storage(apiResourceConfigSource, restOptionsGetter)
		if err != nil {
			return genericapiserver.APIGroupInfo{}, false, err
		}
		apiGroupInfo.VersionedResourcesStorageMap[flowcontrolapisv1alpha1.SchemeGroupVersion.Version] = flowControlStorage
	}

	if apiResourceConfigSource.VersionEnabled(flowcontrolapisv1beta1.SchemeGroupVersion) {
		flowControlStorage, err := p.storage(apiResourceConfigSource, restOptionsGetter)
		if err != nil {
			return genericapiserver.APIGroupInfo{}, false, err
		}
		apiGroupInfo.VersionedResourcesStorageMap[flowcontrolapisv1beta1.SchemeGroupVersion.Version] = flowControlStorage
	}

	if apiResourceConfigSource.VersionEnabled(flowcontrolapisv1beta2.SchemeGroupVersion) {
		flowControlStorage, err := p.storage(apiResourceConfigSource, restOptionsGetter)
		if err != nil {
			return genericapiserver.APIGroupInfo{}, false, err
		}
		apiGroupInfo.VersionedResourcesStorageMap[flowcontrolapisv1beta2.SchemeGroupVersion.Version] = flowControlStorage
	}

	return apiGroupInfo, true, nil
}

func (p RESTStorageProvider) storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (map[string]rest.Storage, error) {
	storage := map[string]rest.Storage{}

	// flow-schema
	flowSchemaStorage, flowSchemaStatusStorage, err := flowschemastore.NewREST(restOptionsGetter)
	if err != nil {
		return nil, err
	}
	storage["flowschemas"] = flowSchemaStorage
	storage["flowschemas/status"] = flowSchemaStatusStorage

	// priority-level-configuration
	priorityLevelConfigurationStorage, priorityLevelConfigurationStatusStorage, err := prioritylevelconfigurationstore.NewREST(restOptionsGetter)
	if err != nil {
		return nil, err
	}
	storage["prioritylevelconfigurations"] = priorityLevelConfigurationStorage
	storage["prioritylevelconfigurations/status"] = priorityLevelConfigurationStatusStorage

	return storage, nil
}

// GroupName returns group name of the storage
func (p RESTStorageProvider) GroupName() string {
	return flowcontrol.GroupName
}

// PostStartHook returns the hook func that launches the config provider
func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStartHookFunc, error) {
	bce := &bootstrapConfigurationEnsurer{
		informersSynced: []cache.InformerSynced{
			p.InformerFactory.Flowcontrol().V1beta2().PriorityLevelConfigurations().Informer().HasSynced,
			p.InformerFactory.Flowcontrol().V1beta2().FlowSchemas().Informer().HasSynced,
		},
		fsLister:  p.InformerFactory.Flowcontrol().V1beta2().FlowSchemas().Lister(),
		plcLister: p.InformerFactory.Flowcontrol().V1beta2().PriorityLevelConfigurations().Lister(),
	}
	return PostStartHookName, bce.ensureAPFBootstrapConfiguration, nil
}

type bootstrapConfigurationEnsurer struct {
	informersSynced []cache.InformerSynced
	fsLister        flowcontrollisters.FlowSchemaLister
	plcLister       flowcontrollisters.PriorityLevelConfigurationLister
}

func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookContext genericapiserver.PostStartHookContext) error {
	clientset, err := flowcontrolclient.NewForConfig(hookContext.LoopbackClientConfig)
	if err != nil {
		return fmt.Errorf("failed to initialize clientset for APF - %w", err)
	}

	// get a derived context that gets cancelled after 5m or
	// when the StopCh gets closed, whichever happens first.
	ctx, cancel := contextFromChannelAndMaxWaitDuration(hookContext.StopCh, 5*time.Minute)
	defer cancel()

	if !cache.WaitForCacheSync(ctx.Done(), bce.informersSynced...) {
		return fmt.Errorf("APF bootstrap ensurer timed out waiting for cache sync")
	}

	err = wait.PollImmediateUntilWithContext(
		ctx,
		time.Second,
		func(context.Context) (bool, error) {
			if err := ensure(clientset, bce.fsLister, bce.plcLister); err != nil {
				klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later")
				return false, nil
			}
			return true, nil
		})
	if err != nil {
		return fmt.Errorf("unable to initialize APF bootstrap configuration")
	}

	// we have successfully initialized the bootstrap configuration, now we
	// spin up a goroutine which reconciles the bootstrap configuration periodically.
	go func() {
		wait.PollImmediateUntil(
			time.Minute,
			func() (bool, error) {
				if err := ensure(clientset, bce.fsLister, bce.plcLister); err != nil {
					klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later")
				}
				// always auto update both suggested and mandatory configuration
				return false, nil
			}, hookContext.StopCh)
		klog.Info("APF bootstrap ensurer is exiting")
	}()

	return nil
}

func ensure(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	if err := ensureSuggestedConfiguration(clientset, fsLister, plcLister); err != nil {
		// We should not attempt creation of mandatory objects if ensuring the suggested
		// configuration resulted in an error.
		// This only happens when the stop channel is closed.
		return fmt.Errorf("failed ensuring suggested settings - %w", err)
	}

	if err := ensureMandatoryConfiguration(clientset, fsLister, plcLister); err != nil {
		return fmt.Errorf("failed ensuring mandatory settings - %w", err)
	}

	if err := removeDanglingBootstrapConfiguration(clientset, fsLister, plcLister); err != nil {
		return fmt.Errorf("failed to delete removed settings - %w", err)
	}

	return nil
}

func ensureSuggestedConfiguration(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	fsEnsurer := ensurer.NewSuggestedFlowSchemaEnsurer(clientset.FlowSchemas(), fsLister)
	if err := fsEnsurer.Ensure(flowcontrolbootstrap.SuggestedFlowSchemas); err != nil {
		return err
	}

	plEnsurer := ensurer.NewSuggestedPriorityLevelEnsurerEnsurer(clientset.PriorityLevelConfigurations(), plcLister)
	return plEnsurer.Ensure(flowcontrolbootstrap.SuggestedPriorityLevelConfigurations)
}

func ensureMandatoryConfiguration(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	fsEnsurer := ensurer.NewMandatoryFlowSchemaEnsurer(clientset.FlowSchemas(), fsLister)
	if err := fsEnsurer.Ensure(flowcontrolbootstrap.MandatoryFlowSchemas); err != nil {
		return err
	}

	plEnsurer := ensurer.NewMandatoryPriorityLevelEnsurer(clientset.PriorityLevelConfigurations(), plcLister)
	return plEnsurer.Ensure(flowcontrolbootstrap.MandatoryPriorityLevelConfigurations)
}

func removeDanglingBootstrapConfiguration(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	if err := removeDanglingBootstrapFlowSchema(clientset.FlowSchemas(), fsLister); err != nil {
		return err
	}

	return removeDanglingBootstrapPriorityLevel(clientset.PriorityLevelConfigurations(), plcLister)
}

func removeDanglingBootstrapFlowSchema(client flowcontrolclient.FlowSchemaInterface, lister flowcontrollisters.FlowSchemaLister) error {
	bootstrap := append(flowcontrolbootstrap.MandatoryFlowSchemas, flowcontrolbootstrap.SuggestedFlowSchemas...)
	candidates, err := ensurer.GetFlowSchemaRemoveCandidates(lister, bootstrap)
	if err != nil {
		return err
	}
	if len(candidates) == 0 {
		return nil
	}

	fsRemover := ensurer.NewFlowSchemaRemover(client, lister)
	return fsRemover.RemoveAutoUpdateEnabledObjects(candidates)
}

func removeDanglingBootstrapPriorityLevel(client flowcontrolclient.PriorityLevelConfigurationInterface, lister flowcontrollisters.PriorityLevelConfigurationLister) error {
	bootstrap := append(flowcontrolbootstrap.MandatoryPriorityLevelConfigurations, flowcontrolbootstrap.SuggestedPriorityLevelConfigurations...)
	candidates, err := ensurer.GetPriorityLevelRemoveCandidates(lister, bootstrap)
	if err != nil {
		return err
	}
	if len(candidates) == 0 {
		return nil
	}

	plRemover := ensurer.NewPriorityLevelRemover(client, lister)
	return plRemover.RemoveAutoUpdateEnabledObjects(candidates)
}

// contextFromChannelAndMaxWaitDuration returns a Context that is bound to the
// specified channel and the wait duration. The derived context will be
// cancelled when the specified channel stopCh is closed or the maximum wait
// duration specified in maxWait elapses, whichever happens first.
//
// Note the caller must *always* call the CancelFunc, otherwise resources may be leaked.
func contextFromChannelAndMaxWaitDuration(stopCh <-chan struct{}, maxWait time.Duration) (context.Context, context.CancelFunc) {
	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		defer cancel()

		select {
		case <-stopCh:
		case <-time.After(maxWait):

		// the caller can explicitly cancel the context which is an
		// indication to us to exit the goroutine immediately.
		// Note that we are calling cancel more than once when we are here,
		// CancelFunc is idempotent and we expect no ripple effects here.
		case <-ctx.Done():
		}
	}()
	return ctx, cancel
}
<commit_msg>apf: create suggested pl before flowschema<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rest

import (
	"context"
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/registry/rest"
	genericapiserver "k8s.io/apiserver/pkg/server"
	serverstorage "k8s.io/apiserver/pkg/server/storage"
	"k8s.io/client-go/informers"
	flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2"
	flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1beta2"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	"k8s.io/kubernetes/pkg/apis/flowcontrol"
	flowcontrolapisv1alpha1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1alpha1"
	flowcontrolapisv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1"
	flowcontrolapisv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2"
	"k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer"
	flowschemastore "k8s.io/kubernetes/pkg/registry/flowcontrol/flowschema/storage"
	prioritylevelconfigurationstore "k8s.io/kubernetes/pkg/registry/flowcontrol/prioritylevelconfiguration/storage"
)

var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{}

// RESTStorageProvider is a provider of REST storage
type RESTStorageProvider struct {
	InformerFactory informers.SharedInformerFactory
}

// PostStartHookName is the name of the post-start-hook provided by flow-control storage
const PostStartHookName = "priority-and-fairness-config-producer"

// NewRESTStorage creates a new rest storage for flow-control api models.
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool, error) {
	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(flowcontrol.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs)

	if apiResourceConfigSource.VersionEnabled(flowcontrolapisv1alpha1.SchemeGroupVersion) {
		flowControlStorage, err := p.storage(apiResourceConfigSource, restOptionsGetter)
		if err != nil {
			return genericapiserver.APIGroupInfo{}, false, err
		}
		apiGroupInfo.VersionedResourcesStorageMap[flowcontrolapisv1alpha1.SchemeGroupVersion.Version] = flowControlStorage
	}

	if apiResourceConfigSource.VersionEnabled(flowcontrolapisv1beta1.SchemeGroupVersion) {
		flowControlStorage, err := p.storage(apiResourceConfigSource, restOptionsGetter)
		if err != nil {
			return genericapiserver.APIGroupInfo{}, false, err
		}
		apiGroupInfo.VersionedResourcesStorageMap[flowcontrolapisv1beta1.SchemeGroupVersion.Version] = flowControlStorage
	}

	if apiResourceConfigSource.VersionEnabled(flowcontrolapisv1beta2.SchemeGroupVersion) {
		flowControlStorage, err := p.storage(apiResourceConfigSource, restOptionsGetter)
		if err != nil {
			return genericapiserver.APIGroupInfo{}, false, err
		}
		apiGroupInfo.VersionedResourcesStorageMap[flowcontrolapisv1beta2.SchemeGroupVersion.Version] = flowControlStorage
	}

	return apiGroupInfo, true, nil
}

func (p RESTStorageProvider) storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (map[string]rest.Storage, error) {
	storage := map[string]rest.Storage{}

	// flow-schema
	flowSchemaStorage, flowSchemaStatusStorage, err := flowschemastore.NewREST(restOptionsGetter)
	if err != nil {
		return nil, err
	}
	storage["flowschemas"] = flowSchemaStorage
	storage["flowschemas/status"] = flowSchemaStatusStorage

	// priority-level-configuration
	priorityLevelConfigurationStorage, priorityLevelConfigurationStatusStorage, err := prioritylevelconfigurationstore.NewREST(restOptionsGetter)
	if err != nil {
		return nil, err
	}
	storage["prioritylevelconfigurations"] = priorityLevelConfigurationStorage
	storage["prioritylevelconfigurations/status"] = priorityLevelConfigurationStatusStorage

	return storage, nil
}

// GroupName returns group name of the storage
func (p RESTStorageProvider) GroupName() string {
	return flowcontrol.GroupName
}

// PostStartHook returns the hook func that launches the config provider
func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStartHookFunc, error) {
	bce := &bootstrapConfigurationEnsurer{
		informersSynced: []cache.InformerSynced{
			p.InformerFactory.Flowcontrol().V1beta2().PriorityLevelConfigurations().Informer().HasSynced,
			p.InformerFactory.Flowcontrol().V1beta2().FlowSchemas().Informer().HasSynced,
		},
		fsLister:  p.InformerFactory.Flowcontrol().V1beta2().FlowSchemas().Lister(),
		plcLister: p.InformerFactory.Flowcontrol().V1beta2().PriorityLevelConfigurations().Lister(),
	}
	return PostStartHookName, bce.ensureAPFBootstrapConfiguration, nil
}

type bootstrapConfigurationEnsurer struct {
	informersSynced []cache.InformerSynced
	fsLister        flowcontrollisters.FlowSchemaLister
	plcLister       flowcontrollisters.PriorityLevelConfigurationLister
}

func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookContext genericapiserver.PostStartHookContext) error {
	clientset, err := flowcontrolclient.NewForConfig(hookContext.LoopbackClientConfig)
	if err != nil {
		return fmt.Errorf("failed to initialize clientset for APF - %w", err)
	}

	// get a derived context that gets cancelled after 5m or
	// when the StopCh gets closed, whichever happens first.
	ctx, cancel := contextFromChannelAndMaxWaitDuration(hookContext.StopCh, 5*time.Minute)
	defer cancel()

	if !cache.WaitForCacheSync(ctx.Done(), bce.informersSynced...) {
		return fmt.Errorf("APF bootstrap ensurer timed out waiting for cache sync")
	}

	err = wait.PollImmediateUntilWithContext(
		ctx,
		time.Second,
		func(context.Context) (bool, error) {
			if err := ensure(clientset, bce.fsLister, bce.plcLister); err != nil {
				klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later")
				return false, nil
			}
			return true, nil
		})
	if err != nil {
		return fmt.Errorf("unable to initialize APF bootstrap configuration")
	}

	// we have successfully initialized the bootstrap configuration, now we
	// spin up a goroutine which reconciles the bootstrap configuration periodically.
	go func() {
		wait.PollImmediateUntil(
			time.Minute,
			func() (bool, error) {
				if err := ensure(clientset, bce.fsLister, bce.plcLister); err != nil {
					klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later")
				}
				// always auto update both suggested and mandatory configuration
				return false, nil
			}, hookContext.StopCh)
		klog.Info("APF bootstrap ensurer is exiting")
	}()

	return nil
}

func ensure(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	if err := ensureSuggestedConfiguration(clientset, fsLister, plcLister); err != nil {
		// We should not attempt creation of mandatory objects if ensuring the suggested
		// configuration resulted in an error.
		// This only happens when the stop channel is closed.
		return fmt.Errorf("failed ensuring suggested settings - %w", err)
	}

	if err := ensureMandatoryConfiguration(clientset, fsLister, plcLister); err != nil {
		return fmt.Errorf("failed ensuring mandatory settings - %w", err)
	}

	if err := removeDanglingBootstrapConfiguration(clientset, fsLister, plcLister); err != nil {
		return fmt.Errorf("failed to delete removed settings - %w", err)
	}

	return nil
}

func ensureSuggestedConfiguration(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	plEnsurer := ensurer.NewSuggestedPriorityLevelEnsurerEnsurer(clientset.PriorityLevelConfigurations(), plcLister)
	if err := plEnsurer.Ensure(flowcontrolbootstrap.SuggestedPriorityLevelConfigurations); err != nil {
		return err
	}

	fsEnsurer := ensurer.NewSuggestedFlowSchemaEnsurer(clientset.FlowSchemas(), fsLister)
	return fsEnsurer.Ensure(flowcontrolbootstrap.SuggestedFlowSchemas)
}

func ensureMandatoryConfiguration(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	fsEnsurer := ensurer.NewMandatoryFlowSchemaEnsurer(clientset.FlowSchemas(), fsLister)
	if err := fsEnsurer.Ensure(flowcontrolbootstrap.MandatoryFlowSchemas); err != nil {
		return err
	}

	plEnsurer := ensurer.NewMandatoryPriorityLevelEnsurer(clientset.PriorityLevelConfigurations(), plcLister)
	return plEnsurer.Ensure(flowcontrolbootstrap.MandatoryPriorityLevelConfigurations)
}

func removeDanglingBootstrapConfiguration(clientset flowcontrolclient.FlowcontrolV1beta2Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error {
	if err := removeDanglingBootstrapFlowSchema(clientset.FlowSchemas(), fsLister); err != nil {
		return err
	}

	return removeDanglingBootstrapPriorityLevel(clientset.PriorityLevelConfigurations(), plcLister)
}

func removeDanglingBootstrapFlowSchema(client flowcontrolclient.FlowSchemaInterface, lister flowcontrollisters.FlowSchemaLister) error {
	bootstrap := append(flowcontrolbootstrap.MandatoryFlowSchemas, flowcontrolbootstrap.SuggestedFlowSchemas...)
	candidates, err := ensurer.GetFlowSchemaRemoveCandidates(lister, bootstrap)
	if err != nil {
		return err
	}
	if len(candidates) == 0 {
		return nil
	}

	fsRemover := ensurer.NewFlowSchemaRemover(client, lister)
	return fsRemover.RemoveAutoUpdateEnabledObjects(candidates)
}

func removeDanglingBootstrapPriorityLevel(client flowcontrolclient.PriorityLevelConfigurationInterface, lister flowcontrollisters.PriorityLevelConfigurationLister) error {
	bootstrap := append(flowcontrolbootstrap.MandatoryPriorityLevelConfigurations, flowcontrolbootstrap.SuggestedPriorityLevelConfigurations...)
	candidates, err := ensurer.GetPriorityLevelRemoveCandidates(lister, bootstrap)
	if err != nil {
		return err
	}
	if len(candidates) == 0 {
		return nil
	}

	plRemover := ensurer.NewPriorityLevelRemover(client, lister)
	return plRemover.RemoveAutoUpdateEnabledObjects(candidates)
}

// contextFromChannelAndMaxWaitDuration returns a Context that is bound to the
// specified channel and the wait duration. The derived context will be
// cancelled when the specified channel stopCh is closed or the maximum wait
// duration specified in maxWait elapses, whichever happens first.
//
// Note the caller must *always* call the CancelFunc, otherwise resources may be leaked.
func contextFromChannelAndMaxWaitDuration(stopCh <-chan struct{}, maxWait time.Duration) (context.Context, context.CancelFunc) {
	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		defer cancel()

		select {
		case <-stopCh:
		case <-time.After(maxWait):

		// the caller can explicitly cancel the context which is an
		// indication to us to exit the goroutine immediately.
		// Note that we are calling cancel more than once when we are here,
		// CancelFunc is idempotent and we expect no ripple effects here.
		case <-ctx.Done():
		}
	}()
	return ctx, cancel
}
<|endoftext|>


<commit_before>package guid

import (
	"time"
	"fmt"
	"sync"
	"errors"
	"net"
	"bytes"
	"crypto/md5"
	"os"
	"strconv"
)

const MAX_UINT uint16 = 0xFFFF
//const MAX_UINT uint16 = 3

type Guid struct {
	sync.Mutex
	workId uint16
	tick   uint16
	lastTime uint32
	lastTick uint16
}

/**
 * 只会用到这个workId的前三个字节
 */
func NewGuid() (*Guid, error) {
	workId, err := defaultWorkId()
	if err != nil {
		return nil, err
	}
	return &Guid{workId: workId}, nil
}

func defaultWorkId() (uint16, error) {
	var buf bytes.Buffer
	interfaces, err := net.Interfaces()
	if err != nil {
		fmt.Println(err)
		return 0, err
	}
	for _, inter := range interfaces {
		buf.Write(inter.HardwareAddr)
		buf.WriteByte(byte(0))
    }

	//fmt.Println("-------------------")

	inter2, err := net.InterfaceAddrs()
	if err != nil {
		fmt.Println(err)
		return 0, err
	}
	for _, i2 := range inter2 {
		buf.WriteString(i2.String())
		buf.WriteByte(byte(0))
    }

	buf.WriteString(strconv.Itoa(os.Getpid()))

	bs := md5.Sum(buf.Bytes())
	//fmt.Println(bs)

	//挑选16个字节的md5只的第1和第9个
	ret := uint16(bs[0]) << 8 + uint16(bs[8])
	//fmt.Println(ret)

	return ret, nil
}

// GUID = TimeStamp(32bit) + workId(16bit) + IncNo(16bit)
func (this *Guid) Generate() (uint64, error) {
	cur := (uint32)(time.Now().Unix())

	this.Lock()
	defer this.Unlock()

	// TODO: 如果修改了系统时间，时间提前，会导致不可生成。
	if cur > this.lastTime {
		this.lastTime = cur
		this.lastTick = this.tick
	} else {
		if this.lastTick == 0 {
			if this.tick == MAX_UINT {
				return 0, errors.New("meet max id count in 1 second")
			}
		} else if this.tick + 1 == this.lastTick {
			return 0, errors.New("meet max id count in 1 second")
		}
	}
	thatTick := this.tick
	//fmt.Printf("thatTick=%d, this.tick=%d, this.lastTick=%d\n", thatTick, this.tick, this.lastTick)
	if this.tick == MAX_UINT {
		this.tick = 0
	} else {
		this.tick++
	}
	//fmt.Printf("cur=%d, this.lastTime=%d, this.lastTick=%d, this.tick=%d\n", cur, this.lastTime, this.lastTick, this.tick)

	return uint64(cur) << 32 + (uint64)(this.workId) << 16 + uint64(thatTick), nil
}
<commit_msg>修改访问属性<commit_after>package guid

import (
	"time"
	"fmt"
	"sync"
	"errors"
	"net"
	"bytes"
	"crypto/md5"
	"os"
	"strconv"
)

const MAX_UINT uint16 = 0xFFFF
//const MAX_UINT uint16 = 3

type Guid struct {
	lock sync.Mutex
	workId uint16
	tick   uint16
	lastTime uint32
	lastTick uint16
}

/**
 * 只会用到这个workId的前三个字节
 */
func NewGuid() (*Guid, error) {
	workId, err := defaultWorkId()
	if err != nil {
		return nil, err
	}
	return &Guid{workId: workId}, nil
}

func defaultWorkId() (uint16, error) {
	var buf bytes.Buffer
	interfaces, err := net.Interfaces()
	if err != nil {
		fmt.Println(err)
		return 0, err
	}
	for _, inter := range interfaces {
		buf.Write(inter.HardwareAddr)
		buf.WriteByte(byte(0))
    }

	//fmt.Println("-------------------")

	inter2, err := net.InterfaceAddrs()
	if err != nil {
		fmt.Println(err)
		return 0, err
	}
	for _, i2 := range inter2 {
		buf.WriteString(i2.String())
		buf.WriteByte(byte(0))
    }

	buf.WriteString(strconv.Itoa(os.Getpid()))

	bs := md5.Sum(buf.Bytes())
	//fmt.Println(bs)

	//挑选16个字节的md5只的第1和第9个
	ret := uint16(bs[0]) << 8 + uint16(bs[8])
	//fmt.Println(ret)

	return ret, nil
}

// GUID = TimeStamp(32bit) + workId(16bit) + IncNo(16bit)
func (this *Guid) Generate() (uint64, error) {
	cur := (uint32)(time.Now().Unix())

	this.lock.Lock()
	defer this.lock.Unlock()

	// TODO: 如果修改了系统时间，时间提前，会导致不可生成。
	if cur > this.lastTime {
		this.lastTime = cur
		this.lastTick = this.tick
	} else {
		if this.lastTick == 0 {
			if this.tick == MAX_UINT {
				return 0, errors.New("meet max id count in 1 second")
			}
		} else if this.tick + 1 == this.lastTick {
			return 0, errors.New("meet max id count in 1 second")
		}
	}
	thatTick := this.tick
	//fmt.Printf("thatTick=%d, this.tick=%d, this.lastTick=%d\n", thatTick, this.tick, this.lastTick)
	if this.tick == MAX_UINT {
		this.tick = 0
	} else {
		this.tick++
	}
	//fmt.Printf("cur=%d, this.lastTime=%d, this.lastTick=%d, this.tick=%d\n", cur, this.lastTime, this.lastTick, this.tick)

	return uint64(cur) << 32 + (uint64)(this.workId) << 16 + uint64(thatTick), nil
}
<|endoftext|>


<commit_before>package lists

const defaultConfig = `
# For keeping simple shared lists of things
Help:
- Keywords: [ "list", "lists", "add" ]
  Helptext: [ "(bot), add <item> to the <type> list - add something to a list" ]
- Keywords: [ "list", "lists", "remove" ]
  Helptext: [ "(bot), remove <item> from the <type> list - remove something from a list" ]
- Keywords: [ "list", "lists", "empty" ]
  Helptext: [ "(bot), empty the <type> list - remove all items from a list" ]
- Keywords: [ "list", "lists", "delete" ]
  Helptext: [ "(bot), delete the <type> list - remove the list altogether" ]
- Keywords: [ "list", "lists" ]
  Helptext: [ "(bot), list lists - give a list of all the lists the robot knows about" ]
- Keywords: [ "list", "lists", "email", "send" ]
  Helptext: [ "(bot), send me the <type> list - send a copy of the list by email" ]
- Keywords: [ "list", "lists", "show", "view" ]
  Helptext: [ "(bot), show the <type> list - show the contents of a list" ]
CommandMatchers:
- Command: 'add'
  Regex: '(?i:add ([\w\d- ]+) to the ([\w]+) list)'
- Command: 'list'
  Regex: '(?i:list lists)'
- Command: 'remove'
  Regex: '(?i:remove ([\w\d- ]+) from the ([\w]+) list)'
- Command: 'empty'
  Regex: '(?i:empty the ([\w]+) list)'
- Command: 'delete'
  Regex: '(?i:delete the ([\w]+) list)'
- Command: 'show'
  Regex: '(?i:show the ([\w]+) list)'
- Command: 'send'
  Regex: '(?i:(?:send me|email) the ([\w]+) list)'
`
<commit_msg>Allow dashes and underscores in list names<commit_after>package lists

const defaultConfig = `
# For keeping simple shared lists of things
Help:
- Keywords: [ "list", "lists", "add" ]
  Helptext: [ "(bot), add <item> to the <type> list - add something to a list" ]
- Keywords: [ "list", "lists", "remove" ]
  Helptext: [ "(bot), remove <item> from the <type> list - remove something from a list" ]
- Keywords: [ "list", "lists", "empty" ]
  Helptext: [ "(bot), empty the <type> list - remove all items from a list" ]
- Keywords: [ "list", "lists", "delete" ]
  Helptext: [ "(bot), delete the <type> list - remove the list altogether" ]
- Keywords: [ "list", "lists" ]
  Helptext: [ "(bot), list lists - give a list of all the lists the robot knows about" ]
- Keywords: [ "list", "lists", "email", "send" ]
  Helptext: [ "(bot), send me the <type> list - send a copy of the list by email" ]
- Keywords: [ "list", "lists", "show", "view" ]
  Helptext: [ "(bot), show the <type> list - show the contents of a list" ]
CommandMatchers:
- Command: 'add'
  Regex: '(?i:add ([\w\d- ]+) to the ([\w-_]+) list)'
- Command: 'list'
  Regex: '(?i:list lists)'
- Command: 'remove'
  Regex: '(?i:remove ([\w\d- ]+) from the ([\w]+) list)'
- Command: 'empty'
  Regex: '(?i:empty the ([\w]+) list)'
- Command: 'delete'
  Regex: '(?i:delete the ([\w]+) list)'
- Command: 'show'
  Regex: '(?i:show the ([\w]+) list)'
- Command: 'send'
  Regex: '(?i:(?:send me|email) the ([\w]+) list)'
`
<|endoftext|>


<commit_before>package extensions

import (
	"github.com/pawelszydlo/papa-bot"
	"github.com/pawelszydlo/papa-bot/utils"
	"regexp"
	"strings"
)

// ExtensionMeta - extension for getting title and description from html links.
type ExtensionMeta struct {
	Extension
	titleRe, metaRe, descRe *regexp.Regexp
}

// Init inits the extension.
func (ext *ExtensionMeta) Init(bot *papaBot.Bot) error {
	ext.titleRe = regexp.MustCompile("(?is)<title.*?>(.+?)</title>")
	ext.metaRe = regexp.MustCompile(`(?is)<\s*?meta.*?content\s*?=\s*?"(.*?)".*?>`)
	ext.descRe = regexp.MustCompile(`(?is)(property|name)\s*?=.*?description`)
	return nil
}

// getTitle find the title and description.
func (ext *ExtensionMeta) getTitle(body string) (string, string, error) {
	// Iterate over meta tags to get the description
	description := ""
	metas := ext.metaRe.FindAllStringSubmatch(string(body), -1)
	for i := range metas {
		if len(metas[i]) > 1 {
			isDesc := ext.descRe.FindString(metas[i][0])
			if isDesc != "" && (len(metas[i][1]) > len(description)) {
				description = utils.CleanString(metas[i][1], true)
			}
		}
	}
	// Get the title
	match := ext.titleRe.FindStringSubmatch(string(body))
	if len(match) > 1 {
		title := utils.CleanString(match[1], true)
		return title, description, nil
	}

	return "", "", nil
}

// ProcessURL will try to get the title and description.
func (ext *ExtensionMeta) ProcessURL(bot *papaBot.Bot, channel, sender, msg string, urlinfo *papaBot.UrlInfo) {
	if len(urlinfo.Body) == 0 || !strings.Contains(urlinfo.ContentType, "html") {
		return
	}
	// Get the title
	// Temporary - YouTube title finding problem
	bot.Log.Debugf("YOUTUBE BODY:\n%s\n", urlinfo.Body)
	title, description, err := ext.getTitle(string(urlinfo.Body))
	if err != nil {
		bot.Log.Warningf("Error getting title: %s", err)
	} else {
		bot.Log.Debugf("Title: %s", title)
	}
	urlinfo.Title = title
	urlinfo.ShortInfo = title
	urlinfo.LongInfo = description
}
<commit_msg>Remove temporary debug info.<commit_after>package extensions

import (
	"github.com/pawelszydlo/papa-bot"
	"github.com/pawelszydlo/papa-bot/utils"
	"regexp"
	"strings"
)

// ExtensionMeta - extension for getting title and description from html links.
type ExtensionMeta struct {
	Extension
	titleRe, metaRe, descRe *regexp.Regexp
}

// Init inits the extension.
func (ext *ExtensionMeta) Init(bot *papaBot.Bot) error {
	ext.titleRe = regexp.MustCompile("(?is)<title.*?>(.+?)</title>")
	ext.metaRe = regexp.MustCompile(`(?is)<\s*?meta.*?content\s*?=\s*?"(.*?)".*?>`)
	ext.descRe = regexp.MustCompile(`(?is)(property|name)\s*?=.*?description`)
	return nil
}

// getTitle find the title and description.
func (ext *ExtensionMeta) getTitle(body string) (string, string, error) {
	// Iterate over meta tags to get the description
	description := ""
	metas := ext.metaRe.FindAllStringSubmatch(string(body), -1)
	for i := range metas {
		if len(metas[i]) > 1 {
			isDesc := ext.descRe.FindString(metas[i][0])
			if isDesc != "" && (len(metas[i][1]) > len(description)) {
				description = utils.CleanString(metas[i][1], true)
			}
		}
	}
	// Get the title
	match := ext.titleRe.FindStringSubmatch(string(body))
	if len(match) > 1 {
		title := utils.CleanString(match[1], true)
		return title, description, nil
	}

	return "", "", nil
}

// ProcessURL will try to get the title and description.
func (ext *ExtensionMeta) ProcessURL(bot *papaBot.Bot, channel, sender, msg string, urlinfo *papaBot.UrlInfo) {
	if len(urlinfo.Body) == 0 || !strings.Contains(urlinfo.ContentType, "html") {
		return
	}
	// Get the title
	title, description, err := ext.getTitle(string(urlinfo.Body))
	if err != nil {
		bot.Log.Warningf("Error getting title: %s", err)
	} else {
		bot.Log.Debugf("Title: %s", title)
	}
	urlinfo.Title = title
	urlinfo.ShortInfo = title
	urlinfo.LongInfo = description
}
<|endoftext|>


<commit_before>package alerting

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"strconv"
	"time"

	"github.com/golang/glog"
	metrics "github.com/rcrowley/go-metrics"

	"skia.googlesource.com/buildbot.git/go/metadata"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/perf/go/clustering"
	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/db"
	"skia.googlesource.com/buildbot.git/perf/go/types"
)

const (
	APIKEY_METADATA_KEY = "apikey"
	CLUSTER_SIZE        = 50
	CLUSTER_STDDEV      = 0.001
)

var (
	// The number of clusters with a status of "New".
	newClustersGauge = metrics.NewRegisteredGauge("alerting.new", metrics.DefaultRegistry)

	// The number of times we've successfully done alert clustering.
	runsCounter = metrics.NewRegisteredCounter("alerting.runs", metrics.DefaultRegistry)

	// How long it takes to do a clustering run.
	alertingLatency = metrics.NewRegisteredTimer("alerting.latency", metrics.DefaultRegistry)
)

// CombineClusters combines freshly found clusters with existing clusters.
//
//  Algorithm:
//    Run clustering and pick out the "Interesting" clusters.
//    Compare all the Interesting clusters to all the existing relevant clusters,
//      where "relevant" clusters are ones whose Hash/timestamp of the step
//      exists in the current tile.
//    Start with an empty "list".
//    For each cluster:
//      For each relevant existing cluster:
//        Take the top 20 keys from the existing cluster and count how many appear
//        in the cluster.
//      If there are no matches then this is a new cluster, add it to the "list".
//      If there are matches, possibly to multiple existing clusters, find the
//      existing cluster with the most matches.
//        Take the cluster (old/new) with the most members, or the best fit if
//        they have the same number of matches.
//    Return all the updated clusters.
func CombineClusters(freshSummaries, oldSummaries []*types.ClusterSummary) []*types.ClusterSummary {
	ret := []*types.ClusterSummary{}

	stillFresh := []*types.ClusterSummary{}
	// If two cluster summaries have the same hash and same Regression direction
	// then they are the same, merge them together.
	for _, fresh := range freshSummaries {
		for _, old := range oldSummaries {
			if fresh.Hash == old.Hash && math.Signbit(fresh.StepFit.Regression) == math.Signbit(old.StepFit.Regression) {
				old.Merge(fresh)
				ret = append(ret, old)
				break
			}
		}
		stillFresh = append(stillFresh, fresh)
	}

	// Even if a summary has a different hash it might still be the same event if
	// there is an overlap in the traces each summary contains.
	for _, fresh := range stillFresh {
		var bestMatch *types.ClusterSummary = nil
		bestMatchHits := 0
		for _, old := range oldSummaries {
			hits := 0
			for _, key := range util.AtMost(old.Keys, 20) {
				if util.In(key, fresh.Keys) {
					hits += 1
				}
			}
			if hits > bestMatchHits {
				bestMatchHits = hits
				bestMatch = old
			}
		}
		if bestMatch != nil {
			keysLengthEqual := len(fresh.Keys) == len(bestMatch.Keys)
			regressionInSameDirection := math.Signbit(fresh.StepFit.Regression) == math.Signbit(bestMatch.StepFit.Regression)
			freshHasBetterFit := math.Abs(fresh.StepFit.Regression) > math.Abs(bestMatch.StepFit.Regression)
			freshHasMoreKeys := len(fresh.Keys) > len(bestMatch.Keys)
			if freshHasMoreKeys || (keysLengthEqual && regressionInSameDirection && freshHasBetterFit) {
				fresh.Status = bestMatch.Status
				fresh.Message = bestMatch.Message
				fresh.ID = bestMatch.ID
				fresh.Bugs = bestMatch.Bugs
				ret = append(ret, fresh)
				// Find the bestMatch in oldSummaries and replace it with fresh.
				for i, oldBest := range oldSummaries {
					if oldBest == bestMatch {
						oldSummaries[i] = fresh
						break
					}
				}
			}
		} else {
			ret = append(ret, fresh)
		}
	}
	return ret
}

// processRows reads all the rows from the clusters table and constructs a
// slice of ClusterSummary's from them.
func processRows(rows *sql.Rows, err error) ([]*types.ClusterSummary, error) {
	if err != nil {
		return nil, fmt.Errorf("Failed to read from database: %s", err)
	}
	defer rows.Close()

	glog.Infof("Found rows %v", rows)

	ret := []*types.ClusterSummary{}

	for rows.Next() {
		var body string
		var id int64
		if err := rows.Scan(&id, &body); err != nil {
			return nil, fmt.Errorf("Failed to read row from database: %s", err)
		}
		c := &types.ClusterSummary{}
		if err := json.Unmarshal([]byte(body), c); err != nil {
			glog.Errorf("Found invalid JSON in clusters table: %s %s", id, err)
			return nil, fmt.Errorf("Failed to read row from database: %s", err)
		}
		c.ID = id
		glog.Infof("ID: %d", id)
		ret = append(ret, c)
	}

	return ret, nil
}

// ListFrom returns all clusters that have a step that occur after the given
// timestamp.
func ListFrom(ts int64) ([]*types.ClusterSummary, error) {
	rows, err := db.DB.Query("SELECT id, cluster FROM clusters WHERE ts>=? ORDER BY status DESC, ts DESC", ts)
	return processRows(rows, err)
}

// ListByStatus returns all clusters that match the given status.
func ListByStatus(status string) ([]*types.ClusterSummary, error) {
	rows, err := db.DB.Query("SELECT id, cluster FROM clusters WHERE status=?", status)
	return processRows(rows, err)
}

// Get returns the cluster that matches the given id.
func Get(id int64) (*types.ClusterSummary, error) {
	rows, err := db.DB.Query("SELECT id, cluster FROM clusters WHERE id=?", id)
	matches, err := processRows(rows, err)
	if len(matches) == 0 {
		return nil, fmt.Errorf("Failed to find cluster summary with id: %d", id)
	}
	return matches[0], nil
}

// Write writes a ClusterSummary to the datastore.
//
// If the ID is set to -1 then write it as a new entry, otherwise update the
// existing entry.
func Write(c *types.ClusterSummary) error {
	b, err := json.Marshal(c)
	if err != nil {
		return fmt.Errorf("Failed to encode to JSON: %s", err)
	}
	if c.ID == -1 {
		_, err := db.DB.Exec(
			"INSERT INTO clusters (ts, hash, regression, cluster, status, message) VALUES (?, ?, ?, ?, ?, ?)",
			c.Timestamp, c.Hash, c.StepFit.Regression, string(b), c.Status, c.Message)
		if err != nil {
			return fmt.Errorf("Failed to write to database: %s", err)
		}
	} else {
		_, err := db.DB.Exec(
			"UPDATE clusters SET ts=?, hash=?, regression=?, cluster=?, status=?, message=? WHERE id=?",
			c.Timestamp, c.Hash, c.StepFit.Regression, string(b), c.Status, c.Message, c.ID)
		if err != nil {
			return fmt.Errorf("Failed to update database: %s", err)
		}
	}
	return nil
}

// Reset removes all non-Bug alerts from the database.
func Reset() error {
	_, err := db.DB.Exec("DELETE FROM clusters WHERE status!='Bug'")
	if err != nil {
		return fmt.Errorf("Failed to write to database: %s", err)
	}
	return nil
}

func skpOnly(_ string, tr *types.PerfTrace) bool {
	return tr.Params()["source_type"] == "skp"
}

// apiKeyFromFlag returns the key that it was passed if the key isn't empty,
// otherwise it tries to fetch the key from the metadata server.
//
// Returns the API Key, or "" if it failed to fetch the key from the metadata
// server.
func apiKeyFromFlag(apiKeyFlag string) string {
	apiKey := apiKeyFlag
	// If apiKey isn't passed in then read it from the metadata server.
	if apiKey == "" {
		var err error
		if apiKey, err = metadata.Get(APIKEY_METADATA_KEY); err != nil {
			glog.Errorf("Retrieving API Key failed: %s", err)
			return ""
		}
	}
	return apiKey
}

// Issue is an individual issue returned from the project hosting response.
//
// It is used in IssueResponse.
type Issue struct {
	ID int64 `json:"id"`
}

// IssueResponse is used to decode JSON responses from the project hosting API.
type IssueResponse struct {
	Items []*Issue `json:"items"`
}

// updateBugs will find all the bugs the reference the alerting cluster will
// write them into the ClusterSummary and save it back to the store.
func updateBugs(c *types.ClusterSummary, apiKey string) {
	// All issues reported through skiaperf will contain a URL of the form:
	//
	//   http://skiaperf.com/cl/NNN.
	//
	// Where NNN is the alerting cluster ID.

	// Search through the project hosting API for all issues that match that URI.
	url := "https://www.googleapis.com/projecthosting/v2/projects/skia/issues?q=%3A%2F%2Fskiaperf.com%2Fcl%2F" + strconv.Itoa(int(c.ID)) + ".&fields=items%2Fid,items%2Fstate&key=" + apiKey

	//  This will return a JSON response of the form:
	//
	//  {
	//   "items": [
	//    {
	//     "id": 2874,
	//     "state": "open"
	//    }
	//   ]
	//  }
	//
	// We don't currently use "state".

	resp, err := http.Get(url)
	if err != nil {
		glog.Errorf("Request to project hosting failed: %s", err)
		return
	}
	defer resp.Body.Close()

	issueResponse := &IssueResponse{
		Items: []*Issue{},
	}
	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(&issueResponse); err != nil {
		glog.Errorf("Failed to decode project hosting response: %s", err)
		return
	}
	glog.Infof("For %d Got %#v", c.ID, issueResponse)
	bugs := []int64{}
	for _, issue := range issueResponse.Items {
		bugs = append(bugs, issue.ID)
	}
	if !util.Int64Equal(bugs, c.Bugs) {
		c.Bugs = bugs
		if err := Write(c); err != nil {
			glog.Errorf("Alerting: Failed to write updated cluster with bugs: %s", err)
		}
	}
}

// trimTime trims the Tile down to at most the last config.MAX_CLUSTER_COMMITS
// commits, less if there aren't that many commits in the Tile.
func trimTile(tile *types.Tile) (*types.Tile, error) {
	end := tile.LastCommitIndex() + 1
	begin := end - config.MAX_CLUSTER_COMMITS
	if begin < 0 {
		begin = 0
	}
	return tile.Trim(begin, end)
}

// singleStep does a single round of alerting.
func singleStep(tileStore types.TileStore, apiKey string) {
	latencyBegin := time.Now()
	tile, err := tileStore.Get(0, -1)
	if err != nil {
		glog.Errorf("Alerting: Failed to get tile: %s", err)
		return
	}

	tile, err = trimTile(tile)
	if err != nil {
		glog.Errorf("Alerting: Failed to Trim tile down to size: %s", err)
		return
	}

	summary, err := clustering.CalculateClusterSummaries(tile, CLUSTER_SIZE, CLUSTER_STDDEV, skpOnly)
	if err != nil {
		glog.Errorf("Alerting: Failed to calculate clusters: %s", err)
		return
	}
	fresh := []*types.ClusterSummary{}
	for _, c := range summary.Clusters {
		if math.Abs(c.StepFit.Regression) > clustering.INTERESTING_THRESHHOLD {
			fresh = append(fresh, c)
		}
	}
	old, err := ListFrom(tile.Commits[0].CommitTime)
	if err != nil {
		glog.Errorf("Alerting: Failed to get existing clusters: %s", err)
		return
	}
	glog.Infof("Found %d old", len(old))
	glog.Infof("Found %d fresh", len(fresh))
	updated := CombineClusters(fresh, old)
	for _, c := range updated {
		if c.Status == "" {
			c.Status = "New"
		}
		if err := Write(c); err != nil {
			glog.Errorf("Alerting: Failed to write updated cluster: %s", err)
		}
	}

	current, err := ListFrom(tile.Commits[0].CommitTime)
	if err != nil {
		glog.Errorf("Alerting: Failed to get existing clusters: %s", err)
		return
	}
	count := 0
	for _, c := range current {
		if c.Status == "New" {
			count++
		}
		if apiKey != "" {
			updateBugs(c, apiKey)
		} else {
			glog.Infof("Skipping ClusterSummary.Bugs update because apiKey is missing.")
			return
		}
	}
	newClustersGauge.Update(int64(count))
	runsCounter.Inc(1)
	alertingLatency.UpdateSince(latencyBegin)
}

// Start kicks off a go routine the periodically refreshes the current alerting clusters.
func Start(tileStore types.TileStore, apiKeyFlag string) {
	apiKey := apiKeyFromFlag(apiKeyFlag)
	go func() {
		for _ = range time.Tick(config.RECLUSTER_DURATION) {
			singleStep(tileStore, apiKey)
		}
	}()
}
<commit_msg>Trim down cluster summaries before storing them in the database.<commit_after>package alerting

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"strconv"
	"time"

	"github.com/golang/glog"
	metrics "github.com/rcrowley/go-metrics"

	"skia.googlesource.com/buildbot.git/go/metadata"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/perf/go/clustering"
	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/db"
	"skia.googlesource.com/buildbot.git/perf/go/types"
)

const (
	APIKEY_METADATA_KEY = "apikey"
	CLUSTER_SIZE        = 50
	CLUSTER_STDDEV      = 0.001
)

var (
	// The number of clusters with a status of "New".
	newClustersGauge = metrics.NewRegisteredGauge("alerting.new", metrics.DefaultRegistry)

	// The number of times we've successfully done alert clustering.
	runsCounter = metrics.NewRegisteredCounter("alerting.runs", metrics.DefaultRegistry)

	// How long it takes to do a clustering run.
	alertingLatency = metrics.NewRegisteredTimer("alerting.latency", metrics.DefaultRegistry)
)

// CombineClusters combines freshly found clusters with existing clusters.
//
//  Algorithm:
//    Run clustering and pick out the "Interesting" clusters.
//    Compare all the Interesting clusters to all the existing relevant clusters,
//      where "relevant" clusters are ones whose Hash/timestamp of the step
//      exists in the current tile.
//    Start with an empty "list".
//    For each cluster:
//      For each relevant existing cluster:
//        Take the top 20 keys from the existing cluster and count how many appear
//        in the cluster.
//      If there are no matches then this is a new cluster, add it to the "list".
//      If there are matches, possibly to multiple existing clusters, find the
//      existing cluster with the most matches.
//        Take the cluster (old/new) with the most members, or the best fit if
//        they have the same number of matches.
//    Return all the updated clusters.
func CombineClusters(freshSummaries, oldSummaries []*types.ClusterSummary) []*types.ClusterSummary {
	ret := []*types.ClusterSummary{}

	stillFresh := []*types.ClusterSummary{}
	// If two cluster summaries have the same hash and same Regression direction
	// then they are the same, merge them together.
	for _, fresh := range freshSummaries {
		for _, old := range oldSummaries {
			if fresh.Hash == old.Hash && math.Signbit(fresh.StepFit.Regression) == math.Signbit(old.StepFit.Regression) {
				old.Merge(fresh)
				ret = append(ret, old)
				break
			}
		}
		stillFresh = append(stillFresh, fresh)
	}

	// Even if a summary has a different hash it might still be the same event if
	// there is an overlap in the traces each summary contains.
	for _, fresh := range stillFresh {
		var bestMatch *types.ClusterSummary = nil
		bestMatchHits := 0
		for _, old := range oldSummaries {
			hits := 0
			for _, key := range util.AtMost(old.Keys, 20) {
				if util.In(key, fresh.Keys) {
					hits += 1
				}
			}
			if hits > bestMatchHits {
				bestMatchHits = hits
				bestMatch = old
			}
		}
		if bestMatch != nil {
			keysLengthEqual := len(fresh.Keys) == len(bestMatch.Keys)
			regressionInSameDirection := math.Signbit(fresh.StepFit.Regression) == math.Signbit(bestMatch.StepFit.Regression)
			freshHasBetterFit := math.Abs(fresh.StepFit.Regression) > math.Abs(bestMatch.StepFit.Regression)
			freshHasMoreKeys := len(fresh.Keys) > len(bestMatch.Keys)
			if freshHasMoreKeys || (keysLengthEqual && regressionInSameDirection && freshHasBetterFit) {
				fresh.Status = bestMatch.Status
				fresh.Message = bestMatch.Message
				fresh.ID = bestMatch.ID
				fresh.Bugs = bestMatch.Bugs
				ret = append(ret, fresh)
				// Find the bestMatch in oldSummaries and replace it with fresh.
				for i, oldBest := range oldSummaries {
					if oldBest == bestMatch {
						oldSummaries[i] = fresh
						break
					}
				}
			}
		} else {
			ret = append(ret, fresh)
		}
	}
	return ret
}

// processRows reads all the rows from the clusters table and constructs a
// slice of ClusterSummary's from them.
func processRows(rows *sql.Rows, err error) ([]*types.ClusterSummary, error) {
	if err != nil {
		return nil, fmt.Errorf("Failed to read from database: %s", err)
	}
	defer rows.Close()

	glog.Infof("Found rows %v", rows)

	ret := []*types.ClusterSummary{}

	for rows.Next() {
		var body string
		var id int64
		if err := rows.Scan(&id, &body); err != nil {
			return nil, fmt.Errorf("Failed to read row from database: %s", err)
		}
		c := &types.ClusterSummary{}
		if err := json.Unmarshal([]byte(body), c); err != nil {
			glog.Errorf("Found invalid JSON in clusters table: %s %s", id, err)
			return nil, fmt.Errorf("Failed to read row from database: %s", err)
		}
		c.ID = id
		glog.Infof("ID: %d", id)
		ret = append(ret, c)
	}

	return ret, nil
}

// ListFrom returns all clusters that have a step that occur after the given
// timestamp.
func ListFrom(ts int64) ([]*types.ClusterSummary, error) {
	rows, err := db.DB.Query("SELECT id, cluster FROM clusters WHERE ts>=? ORDER BY status DESC, ts DESC", ts)
	return processRows(rows, err)
}

// ListByStatus returns all clusters that match the given status.
func ListByStatus(status string) ([]*types.ClusterSummary, error) {
	rows, err := db.DB.Query("SELECT id, cluster FROM clusters WHERE status=?", status)
	return processRows(rows, err)
}

// Get returns the cluster that matches the given id.
func Get(id int64) (*types.ClusterSummary, error) {
	rows, err := db.DB.Query("SELECT id, cluster FROM clusters WHERE id=?", id)
	matches, err := processRows(rows, err)
	if len(matches) == 0 {
		return nil, fmt.Errorf("Failed to find cluster summary with id: %d", id)
	}
	return matches[0], nil
}

// Write writes a ClusterSummary to the datastore.
//
// If the ID is set to -1 then write it as a new entry, otherwise update the
// existing entry.
func Write(c *types.ClusterSummary) error {
	// First trim down c.Traces to just the first entry, which is the centroid.
	c.Traces = c.Traces[:1]

	b, err := json.Marshal(c)
	if err != nil {
		return fmt.Errorf("Failed to encode to JSON: %s", err)
	}
	if c.ID == -1 {
		_, err := db.DB.Exec(
			"INSERT INTO clusters (ts, hash, regression, cluster, status, message) VALUES (?, ?, ?, ?, ?, ?)",
			c.Timestamp, c.Hash, c.StepFit.Regression, string(b), c.Status, c.Message)
		if err != nil {
			return fmt.Errorf("Failed to write to database: %s", err)
		}
	} else {
		_, err := db.DB.Exec(
			"UPDATE clusters SET ts=?, hash=?, regression=?, cluster=?, status=?, message=? WHERE id=?",
			c.Timestamp, c.Hash, c.StepFit.Regression, string(b), c.Status, c.Message, c.ID)
		if err != nil {
			return fmt.Errorf("Failed to update database: %s", err)
		}
	}
	return nil
}

// Reset removes all non-Bug alerts from the database.
func Reset() error {
	_, err := db.DB.Exec("DELETE FROM clusters WHERE status!='Bug'")
	if err != nil {
		return fmt.Errorf("Failed to write to database: %s", err)
	}
	return nil
}

func skpOnly(_ string, tr *types.PerfTrace) bool {
	return tr.Params()["source_type"] == "skp"
}

// apiKeyFromFlag returns the key that it was passed if the key isn't empty,
// otherwise it tries to fetch the key from the metadata server.
//
// Returns the API Key, or "" if it failed to fetch the key from the metadata
// server.
func apiKeyFromFlag(apiKeyFlag string) string {
	apiKey := apiKeyFlag
	// If apiKey isn't passed in then read it from the metadata server.
	if apiKey == "" {
		var err error
		if apiKey, err = metadata.Get(APIKEY_METADATA_KEY); err != nil {
			glog.Errorf("Retrieving API Key failed: %s", err)
			return ""
		}
	}
	return apiKey
}

// Issue is an individual issue returned from the project hosting response.
//
// It is used in IssueResponse.
type Issue struct {
	ID int64 `json:"id"`
}

// IssueResponse is used to decode JSON responses from the project hosting API.
type IssueResponse struct {
	Items []*Issue `json:"items"`
}

// updateBugs will find all the bugs the reference the alerting cluster will
// write them into the ClusterSummary and save it back to the store.
func updateBugs(c *types.ClusterSummary, apiKey string) {
	// All issues reported through skiaperf will contain a URL of the form:
	//
	//   http://skiaperf.com/cl/NNN.
	//
	// Where NNN is the alerting cluster ID.

	// Search through the project hosting API for all issues that match that URI.
	url := "https://www.googleapis.com/projecthosting/v2/projects/skia/issues?q=%3A%2F%2Fskiaperf.com%2Fcl%2F" + strconv.Itoa(int(c.ID)) + ".&fields=items%2Fid,items%2Fstate&key=" + apiKey

	//  This will return a JSON response of the form:
	//
	//  {
	//   "items": [
	//    {
	//     "id": 2874,
	//     "state": "open"
	//    }
	//   ]
	//  }
	//
	// We don't currently use "state".

	resp, err := http.Get(url)
	if err != nil {
		glog.Errorf("Request to project hosting failed: %s", err)
		return
	}
	defer resp.Body.Close()

	issueResponse := &IssueResponse{
		Items: []*Issue{},
	}
	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(&issueResponse); err != nil {
		glog.Errorf("Failed to decode project hosting response: %s", err)
		return
	}
	glog.Infof("For %d Got %#v", c.ID, issueResponse)
	bugs := []int64{}
	for _, issue := range issueResponse.Items {
		bugs = append(bugs, issue.ID)
	}
	if !util.Int64Equal(bugs, c.Bugs) {
		c.Bugs = bugs
		if err := Write(c); err != nil {
			glog.Errorf("Alerting: Failed to write updated cluster with bugs: %s", err)
		}
	}
}

// trimTime trims the Tile down to at most the last config.MAX_CLUSTER_COMMITS
// commits, less if there aren't that many commits in the Tile.
func trimTile(tile *types.Tile) (*types.Tile, error) {
	end := tile.LastCommitIndex() + 1
	begin := end - config.MAX_CLUSTER_COMMITS
	if begin < 0 {
		begin = 0
	}
	return tile.Trim(begin, end)
}

// singleStep does a single round of alerting.
func singleStep(tileStore types.TileStore, apiKey string) {
	latencyBegin := time.Now()
	tile, err := tileStore.Get(0, -1)
	if err != nil {
		glog.Errorf("Alerting: Failed to get tile: %s", err)
		return
	}

	tile, err = trimTile(tile)
	if err != nil {
		glog.Errorf("Alerting: Failed to Trim tile down to size: %s", err)
		return
	}

	summary, err := clustering.CalculateClusterSummaries(tile, CLUSTER_SIZE, CLUSTER_STDDEV, skpOnly)
	if err != nil {
		glog.Errorf("Alerting: Failed to calculate clusters: %s", err)
		return
	}
	fresh := []*types.ClusterSummary{}
	for _, c := range summary.Clusters {
		if math.Abs(c.StepFit.Regression) > clustering.INTERESTING_THRESHHOLD {
			fresh = append(fresh, c)
		}
	}
	old, err := ListFrom(tile.Commits[0].CommitTime)
	if err != nil {
		glog.Errorf("Alerting: Failed to get existing clusters: %s", err)
		return
	}
	glog.Infof("Found %d old", len(old))
	glog.Infof("Found %d fresh", len(fresh))
	updated := CombineClusters(fresh, old)
	for _, c := range updated {
		if c.Status == "" {
			c.Status = "New"
		}
		if err := Write(c); err != nil {
			glog.Errorf("Alerting: Failed to write updated cluster: %s", err)
		}
	}

	current, err := ListFrom(tile.Commits[0].CommitTime)
	if err != nil {
		glog.Errorf("Alerting: Failed to get existing clusters: %s", err)
		return
	}
	count := 0
	for _, c := range current {
		if c.Status == "New" {
			count++
		}
		if apiKey != "" {
			updateBugs(c, apiKey)
		} else {
			glog.Infof("Skipping ClusterSummary.Bugs update because apiKey is missing.")
			return
		}
	}
	newClustersGauge.Update(int64(count))
	runsCounter.Inc(1)
	alertingLatency.UpdateSince(latencyBegin)
}

// Start kicks off a go routine the periodically refreshes the current alerting clusters.
func Start(tileStore types.TileStore, apiKeyFlag string) {
	apiKey := apiKeyFromFlag(apiKeyFlag)
	go func() {
		for _ = range time.Tick(config.RECLUSTER_DURATION) {
			singleStep(tileStore, apiKey)
		}
	}()
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package net provides a portable interface for network I/O, including
TCP/IP, UDP, domain name resolution, and Unix domain sockets.

Although the package provides access to low-level networking
primitives, most clients will need only the basic interface
provided by the Dial, Listen, and Accept functions.

The Dial function connects to a server:

	conn, err := net.Dial("tcp", "google.com:80")
	if err != nil {
		// handle error
	}
	fmt.Fprintf(conn, "GET / HTTP/1.0\r\n\r\n")
	status, err := bufio.NewReader(conn).ReadString('\n')
	// ...

The Listen function creates servers:

	ln, err := net.Listen("tcp", ":8080")
	if err != nil {
		// handle error
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			// handle error
			continue
		}
		go handleConnection(conn)
	}
*/
package net

// TODO(rsc):
//	support for raw ethernet sockets

import (
	"errors"
	"time"
)

// Addr represents a network end point address.
type Addr interface {
	Network() string // name of the network
	String() string  // string form of address
}

// Conn is a generic stream-oriented network connection.
type Conn interface {
	// Read reads data from the connection.
	// Read can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetReadDeadline.
	Read(b []byte) (n int, err error)

	// Write writes data to the connection.
	// Write can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetWriteDeadline.
	Write(b []byte) (n int, err error)

	// Close closes the connection.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// RemoteAddr returns the remote network address.
	RemoteAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for all Read calls to return.
	// If the deadline is reached, Read will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for all Write calls to return.
	// If the deadline is reached, Write will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Write will not time out.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	SetWriteDeadline(t time.Time) error
}

// An Error represents a network error.
type Error interface {
	error
	Timeout() bool   // Is the error a timeout?
	Temporary() bool // Is the error temporary?
}

// PacketConn is a generic packet-oriented network connection.
type PacketConn interface {
	// ReadFrom reads a packet from the connection,
	// copying the payload into b.  It returns the number of
	// bytes copied into b and the return address that
	// was on the packet.
	// ReadFrom can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetReadDeadline.
	ReadFrom(b []byte) (n int, addr Addr, err error)

	// WriteTo writes a packet with payload b to addr.
	// WriteTo can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetWriteDeadline.
	// On packet-oriented connections, write timeouts are rare.
	WriteTo(b []byte, addr Addr) (n int, err error)

	// Close closes the connection.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for all Read calls to return.
	// If the deadline is reached, Read will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for all Write calls to return.
	// If the deadline is reached, Write will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Write will not time out.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	SetWriteDeadline(t time.Time) error
}

// A Listener is a generic network listener for stream-oriented protocols.
type Listener interface {
	// Accept waits for and returns the next connection to the listener.
	Accept() (c Conn, err error)

	// Close closes the listener.
	Close() error

	// Addr returns the listener's network address.
	Addr() Addr
}

var errMissingAddress = errors.New("missing address")

type OpError struct {
	Op   string
	Net  string
	Addr Addr
	Err  error
}

func (e *OpError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Op
	if e.Net != "" {
		s += " " + e.Net
	}
	if e.Addr != nil {
		s += " " + e.Addr.String()
	}
	s += ": " + e.Err.Error()
	return s
}

type temporary interface {
	Temporary() bool
}

func (e *OpError) Temporary() bool {
	t, ok := e.Err.(temporary)
	return ok && t.Temporary()
}

type timeout interface {
	Timeout() bool
}

func (e *OpError) Timeout() bool {
	t, ok := e.Err.(timeout)
	return ok && t.Timeout()
}

type timeoutError struct{}

func (e *timeoutError) Error() string   { return "i/o timeout" }
func (e *timeoutError) Timeout() bool   { return true }
func (e *timeoutError) Temporary() bool { return true }

var errTimeout error = &timeoutError{}

type AddrError struct {
	Err  string
	Addr string
}

func (e *AddrError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Err
	if e.Addr != "" {
		s += " " + e.Addr
	}
	return s
}

func (e *AddrError) Temporary() bool {
	return false
}

func (e *AddrError) Timeout() bool {
	return false
}

type UnknownNetworkError string

func (e UnknownNetworkError) Error() string   { return "unknown network " + string(e) }
func (e UnknownNetworkError) Temporary() bool { return false }
func (e UnknownNetworkError) Timeout() bool   { return false }

// DNSConfigError represents an error reading the machine's DNS configuration.
type DNSConfigError struct {
	Err error
}

func (e *DNSConfigError) Error() string {
	return "error reading DNS config: " + e.Err.Error()
}

func (e *DNSConfigError) Timeout() bool   { return false }
func (e *DNSConfigError) Temporary() bool { return false }
<commit_msg>net: more docs on deadlines<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package net provides a portable interface for network I/O, including
TCP/IP, UDP, domain name resolution, and Unix domain sockets.

Although the package provides access to low-level networking
primitives, most clients will need only the basic interface provided
by the Dial, Listen, and Accept functions and the associated
Conn and Listener interfaces. The crypto/tls package uses
the same interfaces and similar Dial and Listen functions.

The Dial function connects to a server:

	conn, err := net.Dial("tcp", "google.com:80")
	if err != nil {
		// handle error
	}
	fmt.Fprintf(conn, "GET / HTTP/1.0\r\n\r\n")
	status, err := bufio.NewReader(conn).ReadString('\n')
	// ...

The Listen function creates servers:

	ln, err := net.Listen("tcp", ":8080")
	if err != nil {
		// handle error
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			// handle error
			continue
		}
		go handleConnection(conn)
	}
*/
package net

// TODO(rsc):
//	support for raw ethernet sockets

import (
	"errors"
	"time"
)

// Addr represents a network end point address.
type Addr interface {
	Network() string // name of the network
	String() string  // string form of address
}

// Conn is a generic stream-oriented network connection.
type Conn interface {
	// Read reads data from the connection.
	// Read can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetReadDeadline.
	Read(b []byte) (n int, err error)

	// Write writes data to the connection.
	// Write can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetWriteDeadline.
	Write(b []byte) (n int, err error)

	// Close closes the connection.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// RemoteAddr returns the remote network address.
	RemoteAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection. It is equivalent to calling both
	// SetReadDeadline and SetWriteDeadline.
	//
	// A deadline is an absolute time after which I/O operations
	// fail with a timeout (see type Error) instead of
	// blocking. The deadline applies to all future I/O, not just
	// the immediately following call to Read or Write.
	//
	// An idle timeout can be implemented by repeatedly extending
	// the deadline after successful Read or Write calls.
	//
	// A zero value for t means I/O operations will not time out.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for Read calls.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for Write calls.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	// A zero value for t means Write will not time out.
	SetWriteDeadline(t time.Time) error
}

// An Error represents a network error.
type Error interface {
	error
	Timeout() bool   // Is the error a timeout?
	Temporary() bool // Is the error temporary?
}

// PacketConn is a generic packet-oriented network connection.
type PacketConn interface {
	// ReadFrom reads a packet from the connection,
	// copying the payload into b.  It returns the number of
	// bytes copied into b and the return address that
	// was on the packet.
	// ReadFrom can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetReadDeadline.
	ReadFrom(b []byte) (n int, addr Addr, err error)

	// WriteTo writes a packet with payload b to addr.
	// WriteTo can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetWriteDeadline.
	// On packet-oriented connections, write timeouts are rare.
	WriteTo(b []byte, addr Addr) (n int, err error)

	// Close closes the connection.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for all Read calls to return.
	// If the deadline is reached, Read will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for all Write calls to return.
	// If the deadline is reached, Write will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Write will not time out.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	SetWriteDeadline(t time.Time) error
}

// A Listener is a generic network listener for stream-oriented protocols.
type Listener interface {
	// Accept waits for and returns the next connection to the listener.
	Accept() (c Conn, err error)

	// Close closes the listener.
	Close() error

	// Addr returns the listener's network address.
	Addr() Addr
}

var errMissingAddress = errors.New("missing address")

type OpError struct {
	Op   string
	Net  string
	Addr Addr
	Err  error
}

func (e *OpError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Op
	if e.Net != "" {
		s += " " + e.Net
	}
	if e.Addr != nil {
		s += " " + e.Addr.String()
	}
	s += ": " + e.Err.Error()
	return s
}

type temporary interface {
	Temporary() bool
}

func (e *OpError) Temporary() bool {
	t, ok := e.Err.(temporary)
	return ok && t.Temporary()
}

type timeout interface {
	Timeout() bool
}

func (e *OpError) Timeout() bool {
	t, ok := e.Err.(timeout)
	return ok && t.Timeout()
}

type timeoutError struct{}

func (e *timeoutError) Error() string   { return "i/o timeout" }
func (e *timeoutError) Timeout() bool   { return true }
func (e *timeoutError) Temporary() bool { return true }

var errTimeout error = &timeoutError{}

type AddrError struct {
	Err  string
	Addr string
}

func (e *AddrError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Err
	if e.Addr != "" {
		s += " " + e.Addr
	}
	return s
}

func (e *AddrError) Temporary() bool {
	return false
}

func (e *AddrError) Timeout() bool {
	return false
}

type UnknownNetworkError string

func (e UnknownNetworkError) Error() string   { return "unknown network " + string(e) }
func (e UnknownNetworkError) Temporary() bool { return false }
func (e UnknownNetworkError) Timeout() bool   { return false }

// DNSConfigError represents an error reading the machine's DNS configuration.
type DNSConfigError struct {
	Err error
}

func (e *DNSConfigError) Error() string {
	return "error reading DNS config: " + e.Err.Error()
}

func (e *DNSConfigError) Timeout() bool   { return false }
func (e *DNSConfigError) Temporary() bool { return false }
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"errors"
	"flag"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
)

func main() {
	var args arguments
	flag.StringVar(&args.algorithm, "a", "sha256", "hashing algorithm to use")
	flag.StringVar(&args.outputFile, "o", "", "output result to a file")
	flag.BoolVar(&args.hex, "h", true, "output base-16 encoded bytes")
	flag.Parse()
	args.files = flag.Args()
	if err := args.run(); err != nil {
		fmt.Fprintf(os.Stderr, "%+v\n", err)
		os.Exit(1)
	}
}

type arguments struct {
	algorithm  string
	outputFile string
	files      []string
	hex        bool
}

func (a arguments) run() error {
	var h hash.Hash
	switch a.algorithm {
	case "md5":
		h = md5.New()
	case "sha1":
		h = sha1.New()
	case "sha256":
		h = sha256.New()
	case "sha512":
		h = sha512.New()
	default:
		return fmt.Errorf("unsupported hashing algorithm %q", a.algorithm)
	}

	for _, file := range a.files {
		err := hashFile(file, h)
		if err != nil {
			return fmt.Errorf("failed to hash file %s", file)
		}
	}

	result := h.Sum(nil)
	if a.hex {
		result = []byte(fmt.Sprintf("%x", result))
	}

	err := ioutil.WriteFile(a.outputFile, result, 0644)
	if err != nil {
		return fmt.Errorf("failed to save result in %s", a.outputFile)
	}
	return nil
}

func hashFile(file string, h hash.Hash) error {
	f, err := os.Open(file)
	if err != nil {
		return errors.New("failed to open file")
	}
	defer f.Close() // ignore errors from file opened for reading

	_, err = io.Copy(h, f)

	return err
}
<commit_msg>Write separator byte between files<commit_after>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"errors"
	"flag"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
)

func main() {
	var args arguments
	flag.StringVar(&args.algorithm, "a", "sha256", "hashing algorithm to use")
	flag.StringVar(&args.outputFile, "o", "", "output result to a file")
	flag.BoolVar(&args.hex, "h", true, "output base-16 encoded bytes")
	flag.Parse()
	args.files = flag.Args()
	if err := args.run(); err != nil {
		fmt.Fprintf(os.Stderr, "%+v\n", err)
		os.Exit(1)
	}
}

type arguments struct {
	algorithm  string
	outputFile string
	files      []string
	hex        bool
}

func (a arguments) run() error {
	var h hash.Hash
	switch a.algorithm {
	case "md5":
		h = md5.New()
	case "sha1":
		h = sha1.New()
	case "sha256":
		h = sha256.New()
	case "sha512":
		h = sha512.New()
	default:
		return fmt.Errorf("unsupported hashing algorithm %q", a.algorithm)
	}

	first := true
	separator := []byte{0}
	for _, file := range a.files {
		if first {
			first = false
		} else {
			h.Write(separator) // ignore error. Hash.Write() promises to never return an error
		}
		err := hashFile(file, h)
		if err != nil {
			return fmt.Errorf("failed to hash file %s", file)
		}
	}

	result := h.Sum(nil)
	if a.hex {
		result = []byte(fmt.Sprintf("%x", result))
	}

	err := ioutil.WriteFile(a.outputFile, result, 0644)
	if err != nil {
		return fmt.Errorf("failed to save result in %s", a.outputFile)
	}
	return nil
}

func hashFile(file string, h hash.Hash) error {
	f, err := os.Open(file)
	if err != nil {
		return errors.New("failed to open file")
	}
	defer f.Close() // ignore errors from file opened for reading

	_, err = io.Copy(h, f)

	return err
}
<|endoftext|>


<commit_before>// Copyright 2017 The Serulian Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package grok

import (
	"github.com/serulian/compiler/compilercommon"
	"github.com/serulian/compiler/graphs/scopegraph/proto"
	"github.com/serulian/compiler/packageloader"
	"github.com/serulian/compiler/parser"
)

// GetSignatureForPosition returns the parameter signature information for the given activationString at the given position.
// If the activation string does not end in a signature activation character, returns nothing.
func (gh Handle) GetSignatureForPosition(activationString string, source compilercommon.InputSource, lineNumber int, colPosition int) (SignatureInformation, error) {
	// Note: We cannot use PositionFromLineAndColumn here, as it will lookup the position in the *tracked* source, which
	// may be different than the current live source.
	liveRune, err := gh.scopeResult.SourceTracker.LineAndColToRunePosition(lineNumber, colPosition, source, compilercommon.SourceMapCurrent)
	if err != nil {
		return SignatureInformation{}, err
	}

	sourcePosition := source.PositionForRunePosition(liveRune, gh.scopeResult.SourceTracker)
	return gh.GetSignature(activationString, sourcePosition)
}

// GetSignature returns the parameter signature information for the given activationString at the given position.
// If the activation string does not end in a signature activation character, returns nothing.
func (gh Handle) GetSignature(activationString string, sourcePosition compilercommon.SourcePosition) (SignatureInformation, error) {
	updatedPosition, err := gh.scopeResult.SourceTracker.GetPositionOffset(sourcePosition, packageloader.CurrentFilePosition)
	if err != nil {
		return SignatureInformation{}, err
	}

	// Find the node or a nearby node at the position.
	sourceGraph := gh.scopeResult.Graph.SourceGraph()
	node, found := sourceGraph.FindNearbyNodeForPosition(updatedPosition)
	if !found {
		return SignatureInformation{}, nil
	}

	// Extract the expression over which we are activating and the parameter position.
	expressionString, signatureIndex, ok := extractCalled(activationString)
	if !ok {
		return SignatureInformation{}, nil
	}

	// Parse the expression string into an expression.
	source := compilercommon.InputSource(node.Get(parser.NodePredicateSource))
	startRune := node.GetValue(parser.NodePredicateStartRune).Int()

	parsed, ok := gh.scopeResult.Graph.SourceGraph().ParseExpression(expressionString, source, startRune)
	if !ok {
		return SignatureInformation{}, nil
	}

	parentImplementable, hasParentImplementable := gh.structureFinder.TryGetContainingImplemented(node)
	if !hasParentImplementable {
		return SignatureInformation{}, nil
	}

	// Scope the expression underneath the context of the node found.
	scopeInfo, isValid := gh.scopeResult.Graph.BuildTransientScope(parsed, parentImplementable)
	if !isValid {
		return SignatureInformation{}, nil
	}

	// Find the parameter information for the scoped item, which must either refer to a type member
	// with parameter types *or* must have type function.
	var functionType = gh.scopeResult.Graph.TypeGraph().VoidTypeReference()

	switch scopeInfo.Kind {
	case proto.ScopeKind_VALUE:
		functionType = scopeInfo.ResolvedTypeRef(gh.scopeResult.Graph.TypeGraph())

	case proto.ScopeKind_STATIC:
		return SignatureInformation{}, nil

	case proto.ScopeKind_GENERIC:
		return SignatureInformation{}, nil

	default:
		panic("Unknown scope kind")
	}

	// Check for a named reference. If we find a member, use its information directly. This allows
	// us to get parameter names and documentation, in addition to just the type information.
	// If a value referring to a type, then the access is treated as an indexer.
	referencedName, isNamedScope := gh.scopeResult.Graph.GetReferencedName(scopeInfo)
	if isNamedScope {
		member, isMember := referencedName.Member()

		if !isMember {
			if functionType.IsNormal() {
				member, isMember = functionType.ReferredType().GetOperator("index")
			}
		}

		if isMember {
			name := member.Name()
			documentation, _ := member.Documentation()

			parameters := member.Parameters()
			parameterInfo := make([]ParameterInformation, len(parameters))
			for index, parameter := range parameters {
				parameterName, _ := parameter.Name()
				parameterDocumentation, _ := parameter.Documentation()

				parameterInfo[index] = ParameterInformation{
					Name:          parameterName,
					TypeReference: parameter.DeclaredType(),
					Documentation: parameterDocumentation,
				}
			}

			return SignatureInformation{
				TypeReference:        functionType,
				Name:                 name,
				Documentation:        documentation,
				ActiveParameterIndex: signatureIndex,
				Member:               nil,
				Parameters:           parameterInfo,
			}, nil
		}
	}

	// Otherwise, ensure we have a function type.
	if !functionType.IsDirectReferenceTo(gh.scopeResult.Graph.TypeGraph().FunctionType()) {
		// Not a function, so not callable.
		return SignatureInformation{}, nil
	}

	// Generate signature information based on the type information found.
	parameterTypes := functionType.Parameters()
	parameterInfo := make([]ParameterInformation, len(parameterTypes))
	for index, parameterType := range parameterTypes {
		parameterInfo[index] = ParameterInformation{
			Name:          "", // Anonymous
			TypeReference: parameterType,
			Documentation: "",
		}
	}

	return SignatureInformation{
		TypeReference:        functionType,
		Name:                 "", // Anonymous
		Documentation:        "",
		ActiveParameterIndex: signatureIndex,
		Member:               nil,
		Parameters:           parameterInfo,
	}, nil
}
<commit_msg>Add missing member reference<commit_after>// Copyright 2017 The Serulian Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package grok

import (
	"github.com/serulian/compiler/compilercommon"
	"github.com/serulian/compiler/graphs/scopegraph/proto"
	"github.com/serulian/compiler/packageloader"
	"github.com/serulian/compiler/parser"
)

// GetSignatureForPosition returns the parameter signature information for the given activationString at the given position.
// If the activation string does not end in a signature activation character, returns nothing.
func (gh Handle) GetSignatureForPosition(activationString string, source compilercommon.InputSource, lineNumber int, colPosition int) (SignatureInformation, error) {
	// Note: We cannot use PositionFromLineAndColumn here, as it will lookup the position in the *tracked* source, which
	// may be different than the current live source.
	liveRune, err := gh.scopeResult.SourceTracker.LineAndColToRunePosition(lineNumber, colPosition, source, compilercommon.SourceMapCurrent)
	if err != nil {
		return SignatureInformation{}, err
	}

	sourcePosition := source.PositionForRunePosition(liveRune, gh.scopeResult.SourceTracker)
	return gh.GetSignature(activationString, sourcePosition)
}

// GetSignature returns the parameter signature information for the given activationString at the given position.
// If the activation string does not end in a signature activation character, returns nothing.
func (gh Handle) GetSignature(activationString string, sourcePosition compilercommon.SourcePosition) (SignatureInformation, error) {
	updatedPosition, err := gh.scopeResult.SourceTracker.GetPositionOffset(sourcePosition, packageloader.CurrentFilePosition)
	if err != nil {
		return SignatureInformation{}, err
	}

	// Find the node or a nearby node at the position.
	sourceGraph := gh.scopeResult.Graph.SourceGraph()
	node, found := sourceGraph.FindNearbyNodeForPosition(updatedPosition)
	if !found {
		return SignatureInformation{}, nil
	}

	// Extract the expression over which we are activating and the parameter position.
	expressionString, signatureIndex, ok := extractCalled(activationString)
	if !ok {
		return SignatureInformation{}, nil
	}

	// Parse the expression string into an expression.
	source := compilercommon.InputSource(node.Get(parser.NodePredicateSource))
	startRune := node.GetValue(parser.NodePredicateStartRune).Int()

	parsed, ok := gh.scopeResult.Graph.SourceGraph().ParseExpression(expressionString, source, startRune)
	if !ok {
		return SignatureInformation{}, nil
	}

	parentImplementable, hasParentImplementable := gh.structureFinder.TryGetContainingImplemented(node)
	if !hasParentImplementable {
		return SignatureInformation{}, nil
	}

	// Scope the expression underneath the context of the node found.
	scopeInfo, isValid := gh.scopeResult.Graph.BuildTransientScope(parsed, parentImplementable)
	if !isValid {
		return SignatureInformation{}, nil
	}

	// Find the parameter information for the scoped item, which must either refer to a type member
	// with parameter types *or* must have type function.
	var functionType = gh.scopeResult.Graph.TypeGraph().VoidTypeReference()

	switch scopeInfo.Kind {
	case proto.ScopeKind_VALUE:
		functionType = scopeInfo.ResolvedTypeRef(gh.scopeResult.Graph.TypeGraph())

	case proto.ScopeKind_STATIC:
		return SignatureInformation{}, nil

	case proto.ScopeKind_GENERIC:
		return SignatureInformation{}, nil

	default:
		panic("Unknown scope kind")
	}

	// Check for a named reference. If we find a member, use its information directly. This allows
	// us to get parameter names and documentation, in addition to just the type information.
	// If a value referring to a type, then the access is treated as an indexer.
	referencedName, isNamedScope := gh.scopeResult.Graph.GetReferencedName(scopeInfo)
	if isNamedScope {
		member, isMember := referencedName.Member()

		if !isMember {
			if functionType.IsNormal() {
				member, isMember = functionType.ReferredType().GetOperator("index")
			}
		}

		if isMember {
			name := member.Name()
			documentation, _ := member.Documentation()

			parameters := member.Parameters()
			parameterInfo := make([]ParameterInformation, len(parameters))
			for index, parameter := range parameters {
				parameterName, _ := parameter.Name()
				parameterDocumentation, _ := parameter.Documentation()

				parameterInfo[index] = ParameterInformation{
					Name:          parameterName,
					TypeReference: parameter.DeclaredType(),
					Documentation: parameterDocumentation,
				}
			}

			return SignatureInformation{
				TypeReference:        functionType,
				Name:                 name,
				Documentation:        documentation,
				ActiveParameterIndex: signatureIndex,
				Member:               &member,
				Parameters:           parameterInfo,
			}, nil
		}
	}

	// Otherwise, ensure we have a function type.
	if !functionType.IsDirectReferenceTo(gh.scopeResult.Graph.TypeGraph().FunctionType()) {
		// Not a function, so not callable.
		return SignatureInformation{}, nil
	}

	// Generate signature information based on the type information found.
	parameterTypes := functionType.Parameters()
	parameterInfo := make([]ParameterInformation, len(parameterTypes))
	for index, parameterType := range parameterTypes {
		parameterInfo[index] = ParameterInformation{
			Name:          "", // Anonymous
			TypeReference: parameterType,
			Documentation: "",
		}
	}

	return SignatureInformation{
		TypeReference:        functionType,
		Name:                 "", // Anonymous
		Documentation:        "",
		ActiveParameterIndex: signatureIndex,
		Member:               nil,
		Parameters:           parameterInfo,
	}, nil
}
<|endoftext|>


<commit_before>package guestbook

import (
        "html/template"
        "net/http"
        "time"

        "appengine"
        "appengine/datastore"
        "appengine/user"
)

// [START greeting_struct]
type Greeting struct {
        Author  string
        Content string
        Date    time.Time
}
// [END greeting_struct]

func init() {
        http.HandleFunc("/", root)
        http.HandleFunc("/sign", sign)
}

// guestbookKey returns the key used for all guestbook entries.
func guestbookKey(c appengine.Context) *datastore.Key {
        // The string "default_guestbook" here could be varied to have multiple guestbooks.
        return datastore.NewKey(c, "Guestbook", "default_guestbook", 0, nil)
}

// [START func_root]
func root(w http.ResponseWriter, r *http.Request) {
        c := appengine.NewContext(r)
        // Ancestor queries, as shown here, are strongly consistent with the High
        // Replication Datastore. Queries that span entity groups are eventually
        // consistent. If we omitted the .Ancestor from this query there would be
        // a slight chance that Greeting that had just been written would not
        // show up in a query.
        // [START query]
        q := datastore.NewQuery("Greeting").Ancestor(guestbookKey(c)).Order("-Date").Limit(10)
        // [END query]
        // [START getall]
        greetings := make([]Greeting, 0, 10)
        if _, err := q.GetAll(c, &greetings); err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
                return
        }
        // [END getall]
        if err := guestbookTemplate.Execute(w, greetings); err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
        }
}
// [END func_root]

var guestbookTemplate = template.Must(template.New("book").Parse(`

    <title>Go Guestbook</title>

    {{range .}}
      {{with .Author}}

{{.}} wrote:

      {{else}}

An anonymous person wrote:

      {{end}}

{{.Content}}

    {{end}}
    <form action="/sign" method="post">

<textarea name="content" rows="3" cols="60"></textarea>

<input type="submit" value="Sign Guestbook">

    </form>

`))

// [START func_sign]
func sign(w http.ResponseWriter, r *http.Request) {
        // [START new_context]
        c := appengine.NewContext(r)
        // [END new_context]
        g := Greeting{
                Content: r.FormValue("content"),
                Date:    time.Now(),
        }
        // [START if_user]
        if u := user.Current(c); u != nil {
                g.Author = u.String()
        }
        // We set the same parent key on every Greeting entity to ensure each Greeting
        // is in the same entity group. Queries across the single entity group
        // will be consistent. However, the write rate to a single entity group
        // should be limited to ~1/second.
        key := datastore.NewIncompleteKey(c, "Greeting", guestbookKey(c))
        _, err := datastore.Put(c, key, &g)
        if err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
                return
        }
        http.Redirect(w, r, "/", http.StatusFound)
        // [END if_user]
}
// [END func_sign]
<commit_msg>Removed form components<commit_after>package guestbook

import (
        "html/template"
        "net/http"
        "time"

        "appengine"
        "appengine/datastore"
        "appengine/user"
)

// [START greeting_struct]
type Greeting struct {
        Author  string
        Content string
        Date    time.Time
}
// [END greeting_struct]

func init() {
        http.HandleFunc("/", root)
        http.HandleFunc("/sign", sign)
}

// guestbookKey returns the key used for all guestbook entries.
func guestbookKey(c appengine.Context) *datastore.Key {
        // The string "default_guestbook" here could be varied to have multiple guestbooks.
        return datastore.NewKey(c, "Guestbook", "default_guestbook", 0, nil)
}

// [START func_root]
func root(w http.ResponseWriter, r *http.Request) {
        c := appengine.NewContext(r)
        // Ancestor queries, as shown here, are strongly consistent with the High
        // Replication Datastore. Queries that span entity groups are eventually
        // consistent. If we omitted the .Ancestor from this query there would be
        // a slight chance that Greeting that had just been written would not
        // show up in a query.
        // [START query]
        q := datastore.NewQuery("Greeting").Ancestor(guestbookKey(c)).Order("-Date").Limit(10)
        // [END query]
        // [START getall]
        greetings := make([]Greeting, 0, 10)
        if _, err := q.GetAll(c, &greetings); err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
                return
        }
        // [END getall]
        if err := guestbookTemplate.Execute(w, greetings); err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
        }
}
// [END func_root]

var guestbookTemplate = template.Must(template.New("book").Parse(`

    <title>Go Guestbook Viewer</title>

    {{range .}}
      {{with .Author}}

{{.}} wrote:

      {{else}}

An anonymous person wrote:

      {{end}}

{{.Content}}

    {{end}}

`))

// [START func_sign]
func sign(w http.ResponseWriter, r *http.Request) {
        // [START new_context]
        c := appengine.NewContext(r)
        // [END new_context]
        g := Greeting{
                Content: r.FormValue("content"),
                Date:    time.Now(),
        }
        // [START if_user]
        if u := user.Current(c); u != nil {
                g.Author = u.String()
        }
        // We set the same parent key on every Greeting entity to ensure each Greeting
        // is in the same entity group. Queries across the single entity group
        // will be consistent. However, the write rate to a single entity group
        // should be limited to ~1/second.
        key := datastore.NewIncompleteKey(c, "Greeting", guestbookKey(c))
        _, err := datastore.Put(c, key, &g)
        if err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
                return
        }
        http.Redirect(w, r, "/", http.StatusFound)
        // [END if_user]
}
// [END func_sign]
<|endoftext|>