<commit_before>package gorethink

import (
	"encoding/base64"
	"math"
	"strconv"
	"time"

	"github.com/dancannon/gorethink/types"

	"fmt"
)

func convertPseudotype(obj map[string]interface{}, opts map[string]interface{}) (interface{}, error) {
	if reqlType, ok := obj["$reql_type$"]; ok {
		if reqlType == "TIME" {
			// load timeFormat, set to native if the option was not set
			timeFormat := "native"
			if opt, ok := opts["time_format"]; ok {
				if sopt, ok := opt.(string); ok {
					timeFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid time_format run option \"%s\".", opt)
				}
			}

			if timeFormat == "native" {
				return reqlTimeToNativeTime(obj["epoch_time"].(float64), obj["timezone"].(string))
			} else if timeFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown time_format run option \"%s\".", reqlType)
			}
		} else if reqlType == "GROUPED_DATA" {
			// load groupFormat, set to native if the option was not set
			groupFormat := "native"
			if opt, ok := opts["group_format"]; ok {
				if sopt, ok := opt.(string); ok {
					groupFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid group_format run option \"%s\".", opt)
				}
			}

			if groupFormat == "native" {
				return reqlGroupedDataToObj(obj)
			} else if groupFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown group_format run option \"%s\".", reqlType)
			}
		} else if reqlType == "BINARY" {
			binaryFormat := "native"
			if opt, ok := opts["binary_format"]; ok {
				if sopt, ok := opt.(string); ok {
					binaryFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid binary_format run option \"%s\".", opt)
				}
			}

			if binaryFormat == "native" {
				return reqlBinaryToNativeBytes(obj)
			} else if binaryFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown binary_format run option \"%s\".", reqlType)
			}
		} else if reqlType == "GEOMETRY" {
			geometryFormat := "native"
			if opt, ok := opts["geometry_format"]; ok {
				if sopt, ok := opt.(string); ok {
					geometryFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid geometry_format run option \"%s\".", opt)
				}
			}

			if geometryFormat == "native" {
				return reqlGeometryToNativeGeometry(obj)
			} else if geometryFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown geometry_format run option \"%s\".", reqlType)
			}
		} else {
			return obj, nil
		}
	}

	return obj, nil
}

func recursivelyConvertPseudotype(obj interface{}, opts map[string]interface{}) (interface{}, error) {
	var err error

	switch obj := obj.(type) {
	case []interface{}:
		for key, val := range obj {
			obj[key], err = recursivelyConvertPseudotype(val, opts)
			if err != nil {
				return nil, err
			}
		}
	case map[string]interface{}:
		for key, val := range obj {
			obj[key], err = recursivelyConvertPseudotype(val, opts)
			if err != nil {
				return nil, err
			}
		}

		pobj, err := convertPseudotype(obj, opts)
		if err != nil {
			return nil, err
		}

		return pobj, nil
	}

	return obj, nil
}

// Pseudo-type helper functions

func reqlTimeToNativeTime(timestamp float64, timezone string) (time.Time, error) {
	sec, ms := math.Modf(timestamp)

	t := time.Unix(int64(sec), int64(ms*1000*1000*1000))

	// Caclulate the timezone
	if timezone != "" {
		hours, err := strconv.Atoi(timezone[1:3])
		if err != nil {
			return time.Time{}, err
		}
		minutes, err := strconv.Atoi(timezone[4:6])
		if err != nil {
			return time.Time{}, err
		}
		tzOffset := ((hours * 60) + minutes) * 60
		if timezone[:1] == "-" {
			tzOffset = 0 - tzOffset
		}

		t = t.In(time.FixedZone(timezone, tzOffset))
	}

	return t, nil
}

func reqlGroupedDataToObj(obj map[string]interface{}) (interface{}, error) {
	if data, ok := obj["data"]; ok {
		ret := []interface{}{}
		for _, v := range data.([]interface{}) {
			v := v.([]interface{})
			ret = append(ret, map[string]interface{}{
				"group":     v[0],
				"reduction": v[1],
			})
		}
		return ret, nil
	}
	return nil, fmt.Errorf("pseudo-type GROUPED_DATA object %v does not have the expected field \"data\"", obj)
}

func reqlBinaryToNativeBytes(obj map[string]interface{}) (interface{}, error) {
	if data, ok := obj["data"]; ok {
		if data, ok := data.(string); ok {
			b, err := base64.StdEncoding.DecodeString(data)
			if err != nil {
				return nil, fmt.Errorf("error decoding pseudo-type BINARY object %v", obj)
			}

			return b, nil
		}
		return nil, fmt.Errorf("pseudo-type BINARY object %v field \"data\" is not valid", obj)
	}
	return nil, fmt.Errorf("pseudo-type BINARY object %v does not have the expected field \"data\"", obj)
}

func reqlGeometryToNativeGeometry(obj map[string]interface{}) (interface{}, error) {
	if typ, ok := obj["type"]; !ok {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v does not have the expected field \"type\"", obj)
	} else if typ, ok := typ.(string); !ok {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v field \"type\" is not valid", obj)
	} else if coords, ok := obj["coordinates"]; !ok {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v does not have the expected field \"coordinates\"", obj)
	} else if typ == "Point" {
		point, err := types.UnmarshalPoint(coords)
		if err != nil {
			return nil, err
		}

		return types.Geometry{
			Type:  "Point",
			Point: point,
		}, nil
	} else if typ == "LineString" {
		line, err := types.UnmarshalLineString(coords)
		if err != nil {
			return nil, err
		}
		return types.Geometry{
			Type: "LineString",
			Line: line,
		}, nil
	} else if typ == "Polygon" {
		lines, err := types.UnmarshalPolygon(coords)
		if err != nil {
			return nil, err
		}
		return types.Geometry{
			Type:  "Polygon",
			Lines: lines,
		}, nil
	} else {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v field has unknown type %s", obj, typ)
	}
}
<commit_msg>A bug in conversion of milliseconds<commit_after>package gorethink

import (
	"encoding/base64"
	"math"
	"strconv"
	"time"

	"github.com/dancannon/gorethink/types"

	"fmt"
)

func convertPseudotype(obj map[string]interface{}, opts map[string]interface{}) (interface{}, error) {
	if reqlType, ok := obj["$reql_type$"]; ok {
		if reqlType == "TIME" {
			// load timeFormat, set to native if the option was not set
			timeFormat := "native"
			if opt, ok := opts["time_format"]; ok {
				if sopt, ok := opt.(string); ok {
					timeFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid time_format run option \"%s\".", opt)
				}
			}

			if timeFormat == "native" {
				return reqlTimeToNativeTime(obj["epoch_time"].(float64), obj["timezone"].(string))
			} else if timeFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown time_format run option \"%s\".", reqlType)
			}
		} else if reqlType == "GROUPED_DATA" {
			// load groupFormat, set to native if the option was not set
			groupFormat := "native"
			if opt, ok := opts["group_format"]; ok {
				if sopt, ok := opt.(string); ok {
					groupFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid group_format run option \"%s\".", opt)
				}
			}

			if groupFormat == "native" {
				return reqlGroupedDataToObj(obj)
			} else if groupFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown group_format run option \"%s\".", reqlType)
			}
		} else if reqlType == "BINARY" {
			binaryFormat := "native"
			if opt, ok := opts["binary_format"]; ok {
				if sopt, ok := opt.(string); ok {
					binaryFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid binary_format run option \"%s\".", opt)
				}
			}

			if binaryFormat == "native" {
				return reqlBinaryToNativeBytes(obj)
			} else if binaryFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown binary_format run option \"%s\".", reqlType)
			}
		} else if reqlType == "GEOMETRY" {
			geometryFormat := "native"
			if opt, ok := opts["geometry_format"]; ok {
				if sopt, ok := opt.(string); ok {
					geometryFormat = sopt
				} else {
					return nil, fmt.Errorf("Invalid geometry_format run option \"%s\".", opt)
				}
			}

			if geometryFormat == "native" {
				return reqlGeometryToNativeGeometry(obj)
			} else if geometryFormat == "raw" {
				return obj, nil
			} else {
				return nil, fmt.Errorf("Unknown geometry_format run option \"%s\".", reqlType)
			}
		} else {
			return obj, nil
		}
	}

	return obj, nil
}

func recursivelyConvertPseudotype(obj interface{}, opts map[string]interface{}) (interface{}, error) {
	var err error

	switch obj := obj.(type) {
	case []interface{}:
		for key, val := range obj {
			obj[key], err = recursivelyConvertPseudotype(val, opts)
			if err != nil {
				return nil, err
			}
		}
	case map[string]interface{}:
		for key, val := range obj {
			obj[key], err = recursivelyConvertPseudotype(val, opts)
			if err != nil {
				return nil, err
			}
		}

		pobj, err := convertPseudotype(obj, opts)
		if err != nil {
			return nil, err
		}

		return pobj, nil
	}

	return obj, nil
}

// Pseudo-type helper functions

func reqlTimeToNativeTime(timestamp float64, timezone string) (time.Time, error) {
	sec, ms := math.Modf(timestamp)

	// Convert to native time rounding to milliseconds
	t := time.Unix(int64(sec), int64(math.Floor(ms*1000+0.5))*1000*1000)

	// Caclulate the timezone
	if timezone != "" {
		hours, err := strconv.Atoi(timezone[1:3])
		if err != nil {
			return time.Time{}, err
		}
		minutes, err := strconv.Atoi(timezone[4:6])
		if err != nil {
			return time.Time{}, err
		}
		tzOffset := ((hours * 60) + minutes) * 60
		if timezone[:1] == "-" {
			tzOffset = 0 - tzOffset
		}

		t = t.In(time.FixedZone(timezone, tzOffset))
	}

	return t, nil
}

func reqlGroupedDataToObj(obj map[string]interface{}) (interface{}, error) {
	if data, ok := obj["data"]; ok {
		ret := []interface{}{}
		for _, v := range data.([]interface{}) {
			v := v.([]interface{})
			ret = append(ret, map[string]interface{}{
				"group":     v[0],
				"reduction": v[1],
			})
		}
		return ret, nil
	}
	return nil, fmt.Errorf("pseudo-type GROUPED_DATA object %v does not have the expected field \"data\"", obj)
}

func reqlBinaryToNativeBytes(obj map[string]interface{}) (interface{}, error) {
	if data, ok := obj["data"]; ok {
		if data, ok := data.(string); ok {
			b, err := base64.StdEncoding.DecodeString(data)
			if err != nil {
				return nil, fmt.Errorf("error decoding pseudo-type BINARY object %v", obj)
			}

			return b, nil
		}
		return nil, fmt.Errorf("pseudo-type BINARY object %v field \"data\" is not valid", obj)
	}
	return nil, fmt.Errorf("pseudo-type BINARY object %v does not have the expected field \"data\"", obj)
}

func reqlGeometryToNativeGeometry(obj map[string]interface{}) (interface{}, error) {
	if typ, ok := obj["type"]; !ok {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v does not have the expected field \"type\"", obj)
	} else if typ, ok := typ.(string); !ok {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v field \"type\" is not valid", obj)
	} else if coords, ok := obj["coordinates"]; !ok {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v does not have the expected field \"coordinates\"", obj)
	} else if typ == "Point" {
		point, err := types.UnmarshalPoint(coords)
		if err != nil {
			return nil, err
		}

		return types.Geometry{
			Type:  "Point",
			Point: point,
		}, nil
	} else if typ == "LineString" {
		line, err := types.UnmarshalLineString(coords)
		if err != nil {
			return nil, err
		}
		return types.Geometry{
			Type: "LineString",
			Line: line,
		}, nil
	} else if typ == "Polygon" {
		lines, err := types.UnmarshalPolygon(coords)
		if err != nil {
			return nil, err
		}
		return types.Geometry{
			Type:  "Polygon",
			Lines: lines,
		}, nil
	} else {
		return nil, fmt.Errorf("pseudo-type GEOMETRY object %v field has unknown type %s", obj, typ)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package updateconfig

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"unicode/utf8"

	"github.com/mattn/go-zglob"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
	coreapi "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/git/v2"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

const (
	pluginName    = "config-updater"
	bootstrapMode = false
)

func init() {
	plugins.RegisterPullRequestHandler(pluginName, handlePullRequest, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []config.OrgRepo) (*pluginhelp.PluginHelp, error) {
	var configInfo map[string]string
	if len(enabledRepos) == 1 {
		msg := ""
		for configFileName, configMapSpec := range config.ConfigUpdater.Maps {
			msg = msg + fmt.Sprintf(
				"Files matching %s/%s are used to populate the %s ConfigMap in ",
				enabledRepos[0],
				configFileName,
				configMapSpec.Name,
			)
			if len(configMapSpec.AdditionalNamespaces) == 0 {
				msg = msg + fmt.Sprintf("the %s namespace.\n", configMapSpec.Namespace)
			} else {
				for _, nameSpace := range configMapSpec.AdditionalNamespaces {
					msg = msg + fmt.Sprintf("%s, ", nameSpace)
				}
				msg = msg + fmt.Sprintf("and %s namespaces.\n", configMapSpec.Namespace)
			}
		}
		configInfo = map[string]string{"": msg}
	}
	return &pluginhelp.PluginHelp{
			Description: "The config-updater plugin automatically redeploys configuration and plugin configuration files when they change. The plugin watches for pull request merges that modify either of the config files and updates the cluster's configmap resources in response.",
			Config:      configInfo,
		},
		nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
	GetPullRequestChanges(org, repo string, number int) ([]github.PullRequestChange, error)
}

func handlePullRequest(pc plugins.Agent, pre github.PullRequestEvent) error {
	return handle(pc.GitHubClient, pc.GitClient, pc.KubernetesClient.CoreV1(), pc.BuildClusterCoreV1Clients, pc.Config.ProwJobNamespace, pc.Logger, pre, pc.PluginConfig.ConfigUpdater, pc.Metrics.ConfigMapGauges)
}

// FileGetter knows how to get the contents of a file by name
type FileGetter interface {
	GetFile(filename string) ([]byte, error)
}

type OSFileGetter struct {
	Root string
}

func (g *OSFileGetter) GetFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filepath.Join(g.Root, filename))
}

// Update updates the configmap with the data from the identified files.
// Existing configmap keys that are not included in the updates are left alone
// unless bootstrap is true in which case they are deleted.
func Update(fg FileGetter, kc corev1.ConfigMapInterface, name, namespace string, updates []ConfigMapUpdate, bootstrap bool, metrics *prometheus.GaugeVec, logger *logrus.Entry) error {
	cm, getErr := kc.Get(name, metav1.GetOptions{})
	isNotFound := errors.IsNotFound(getErr)
	if getErr != nil && !isNotFound {
		return fmt.Errorf("failed to fetch current state of configmap: %v", getErr)
	}

	if cm == nil || isNotFound {
		cm = &coreapi.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: namespace,
			},
		}
	}
	if cm.Data == nil || bootstrap {
		cm.Data = map[string]string{}
	}
	if cm.BinaryData == nil || bootstrap {
		cm.BinaryData = map[string][]byte{}
	}

	for _, upd := range updates {
		if upd.Filename == "" {
			logger.WithField("key", upd.Key).Debug("Deleting key.")
			delete(cm.Data, upd.Key)
			delete(cm.BinaryData, upd.Key)
			continue
		}

		content, err := fg.GetFile(upd.Filename)
		if err != nil {
			return fmt.Errorf("get file err: %v", err)
		}
		logger.WithFields(logrus.Fields{"key": upd.Key, "filename": upd.Filename}).Debug("Populating key.")
		value := content
		if upd.GZIP {
			buff := bytes.NewBuffer([]byte{})
			// TODO: this error is wildly unlikely for anything that
			// would actually fit in a configmap, we could just as well return
			// the error instead of falling back to the raw content
			z := gzip.NewWriter(buff)
			if _, err := z.Write(content); err != nil {
				logger.WithError(err).Error("failed to gzip content, falling back to raw")
			} else {
				if err := z.Close(); err != nil {
					logger.WithError(err).Error("failed to flush gzipped content (!?), falling back to raw")
				} else {
					value = buff.Bytes()
				}
			}
		}
		if utf8.ValidString(string(value)) {
			delete(cm.BinaryData, upd.Key)
			cm.Data[upd.Key] = string(value)
		} else {
			delete(cm.Data, upd.Key)
			cm.BinaryData[upd.Key] = value
		}
	}

	var updateErr error
	var verb string
	if getErr != nil && isNotFound {
		verb = "create"
		_, updateErr = kc.Create(cm)
	} else {
		verb = "update"
		_, updateErr = kc.Update(cm)
	}
	if updateErr != nil {
		return fmt.Errorf("%s config map err: %v", verb, updateErr)
	}
	if metrics != nil {
		var size float64
		for _, data := range cm.Data {
			size += float64(len(data))
		}
		// in a strict sense this can race to update the value with other goroutines
		// handling other events, but as events are serialized due to the fact that
		// merges are serial in repositories, this is effectively not an issue here
		metrics.WithLabelValues(cm.Name, cm.Namespace).Set(size)
	}
	return nil
}

// ConfigMapUpdate is populated with information about a config map that should
// be updated.
type ConfigMapUpdate struct {
	Key, Filename string
	GZIP          bool
}

// FilterChanges determines which of the changes are relevant for config updating, returning mapping of
// config map to key to filename to update that key from.
func FilterChanges(cfg plugins.ConfigUpdater, changes []github.PullRequestChange, defaultNamespace string, log *logrus.Entry) map[plugins.ConfigMapID][]ConfigMapUpdate {
	toUpdate := map[plugins.ConfigMapID][]ConfigMapUpdate{}
	for _, change := range changes {
		var cm plugins.ConfigMapSpec
		found := false

		for key, configMap := range cfg.Maps {
			var matchErr error
			found, matchErr = zglob.Match(key, change.Filename)
			if matchErr != nil {
				// Should not happen, log matchErr and continue
				log.WithError(matchErr).Info("key matching error")
				continue
			}

			if found {
				cm = configMap
				break
			}
		}

		if !found {
			continue // This file does not define a configmap
		}

		// Yes, update the configmap with the contents of this file
		for cluster, namespaces := range cm.Clusters {
			for _, ns := range namespaces {
				id := plugins.ConfigMapID{Name: cm.Name, Namespace: ns, Cluster: cluster}
				key := cm.Key
				if key == "" {
					key = path.Base(change.Filename)
					// if the key changed, we need to remove the old key
					if change.Status == github.PullRequestFileRenamed {
						oldKey := path.Base(change.PreviousFilename)
						// not setting the filename field will cause the key to be
						// deleted
						toUpdate[id] = append(toUpdate[id], ConfigMapUpdate{Key: oldKey})
					}
				}
				if change.Status == github.PullRequestFileRemoved {
					toUpdate[id] = append(toUpdate[id], ConfigMapUpdate{Key: key})
				} else {
					gzip := cfg.GZIP
					if cm.GZIP != nil {
						gzip = *cm.GZIP
					}
					toUpdate[id] = append(toUpdate[id], ConfigMapUpdate{Key: key, Filename: change.Filename, GZIP: gzip})
				}
			}
		}
	}
	return handleDefaultNamespace(toUpdate, defaultNamespace)
}

// handleDefaultNamespace ensures plugins.ConfigMapID.Namespace is not empty string
func handleDefaultNamespace(toUpdate map[plugins.ConfigMapID][]ConfigMapUpdate, defaultNamespace string) map[plugins.ConfigMapID][]ConfigMapUpdate {
	for cm, data := range toUpdate {
		if cm.Namespace == "" {
			key := plugins.ConfigMapID{Name: cm.Name, Namespace: defaultNamespace, Cluster: cm.Cluster}
			toUpdate[key] = append(toUpdate[key], data...)
			delete(toUpdate, cm)
		}
	}
	return toUpdate
}

func handle(gc githubClient, gitClient git.ClientFactory, kc corev1.ConfigMapsGetter, buildClusterCoreV1Clients map[string]corev1.CoreV1Interface, defaultNamespace string, log *logrus.Entry, pre github.PullRequestEvent, config plugins.ConfigUpdater, metrics *prometheus.GaugeVec) error {
	// Only consider newly merged PRs
	if pre.Action != github.PullRequestActionClosed {
		return nil
	}

	if len(config.Maps) == 0 { // Nothing to update
		return nil
	}

	pr := pre.PullRequest

	if !pr.Merged || pr.MergeSHA == nil || pr.Base.Repo.DefaultBranch != pr.Base.Ref {
		return nil
	}

	org := pr.Base.Repo.Owner.Login
	repo := pr.Base.Repo.Name

	// Which files changed in this PR?
	changes, err := gc.GetPullRequestChanges(org, repo, pr.Number)
	if err != nil {
		return err
	}

	message := func(name, cluster, namespace string, updates []ConfigMapUpdate, indent string) string {
		identifier := fmt.Sprintf("`%s` configmap", name)
		if namespace != "" {
			identifier = fmt.Sprintf("%s in namespace `%s`", identifier, namespace)
		}
		if cluster != "" {
			identifier = fmt.Sprintf("%s at cluster `%s`", identifier, cluster)
		}
		msg := fmt.Sprintf("%s using the following files:", identifier)
		for _, u := range updates {
			msg = fmt.Sprintf("%s\n%s- key `%s` using file `%s`", msg, indent, u.Key, u.Filename)
		}
		return msg
	}

	// Are any of the changes files ones that define a configmap we want to update?
	toUpdate := FilterChanges(config, changes, defaultNamespace, log)

	var updated []string
	indent := " " // one space
	if len(toUpdate) > 1 {
		indent = "   " // three spaces for sub bullets
	}

	gitRepo, err := gitClient.ClientFor(org, repo)
	if err != nil {
		return err
	}
	defer func() {
		if err := gitClient.Clean(); err != nil {
			log.WithError(err).Error("Could not clean up git client cache.")
		}
	}()
	if err := gitRepo.Checkout(*pr.MergeSHA); err != nil {
		return err
	}

	var errs []error
	for cm, data := range toUpdate {
		logger := log.WithFields(logrus.Fields{"configmap": map[string]string{"name": cm.Name, "namespace": cm.Namespace, "cluster": cm.Cluster}})
		configMapClient, err := GetConfigMapClient(kc, cm.Namespace, buildClusterCoreV1Clients, cm.Cluster)
		if err != nil {
			log.WithError(err).Errorf("Failed to find configMap client")
			continue
		}
		if err := Update(&OSFileGetter{Root: gitRepo.Directory()}, configMapClient, cm.Name, cm.Namespace, data, bootstrapMode, metrics, logger); err != nil {
			errs = append(errs, err)
			continue
		}
		updated = append(updated, message(cm.Name, cm.Cluster, cm.Namespace, data, indent))
	}

	var msg string
	switch n := len(updated); n {
	case 0:
		return nil
	case 1:
		msg = fmt.Sprintf("Updated the %s", updated[0])
	default:
		msg = fmt.Sprintf("Updated the following %d configmaps:\n", n)
		for _, updateMsg := range updated {
			msg += fmt.Sprintf(" * %s\n", updateMsg) // one space indent
		}
	}

	if err := gc.CreateComment(org, repo, pr.Number, plugins.FormatResponseRaw(pr.Body, pr.HTMLURL, pr.User.Login, msg)); err != nil {
		errs = append(errs, fmt.Errorf("comment err: %v", err))
	}
	return utilerrors.NewAggregate(errs)
}

// GetConfigMapClient returns a configMap interface according to the given cluster and namespace
func GetConfigMapClient(kc corev1.ConfigMapsGetter, namespace string, buildClusterCoreV1Clients map[string]corev1.CoreV1Interface, cluster string) (corev1.ConfigMapInterface, error) {
	configMapClient := kc.ConfigMaps(namespace)
	if cluster != kube.DefaultClusterAlias {
		if client, ok := buildClusterCoreV1Clients[cluster]; ok {
			configMapClient = client.ConfigMaps(namespace)
		} else {
			return nil, fmt.Errorf("no k8s client is found for build cluster: '%s'", cluster)
		}
	}
	return configMapClient, nil
}
<commit_msg>Add a debug log in updateconfig plugin<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package updateconfig

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"unicode/utf8"

	"github.com/mattn/go-zglob"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
	coreapi "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/git/v2"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

const (
	pluginName    = "config-updater"
	bootstrapMode = false
)

func init() {
	plugins.RegisterPullRequestHandler(pluginName, handlePullRequest, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []config.OrgRepo) (*pluginhelp.PluginHelp, error) {
	var configInfo map[string]string
	if len(enabledRepos) == 1 {
		msg := ""
		for configFileName, configMapSpec := range config.ConfigUpdater.Maps {
			msg = msg + fmt.Sprintf(
				"Files matching %s/%s are used to populate the %s ConfigMap in ",
				enabledRepos[0],
				configFileName,
				configMapSpec.Name,
			)
			if len(configMapSpec.AdditionalNamespaces) == 0 {
				msg = msg + fmt.Sprintf("the %s namespace.\n", configMapSpec.Namespace)
			} else {
				for _, nameSpace := range configMapSpec.AdditionalNamespaces {
					msg = msg + fmt.Sprintf("%s, ", nameSpace)
				}
				msg = msg + fmt.Sprintf("and %s namespaces.\n", configMapSpec.Namespace)
			}
		}
		configInfo = map[string]string{"": msg}
	}
	return &pluginhelp.PluginHelp{
			Description: "The config-updater plugin automatically redeploys configuration and plugin configuration files when they change. The plugin watches for pull request merges that modify either of the config files and updates the cluster's configmap resources in response.",
			Config:      configInfo,
		},
		nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
	GetPullRequestChanges(org, repo string, number int) ([]github.PullRequestChange, error)
}

func handlePullRequest(pc plugins.Agent, pre github.PullRequestEvent) error {
	return handle(pc.GitHubClient, pc.GitClient, pc.KubernetesClient.CoreV1(), pc.BuildClusterCoreV1Clients, pc.Config.ProwJobNamespace, pc.Logger, pre, pc.PluginConfig.ConfigUpdater, pc.Metrics.ConfigMapGauges)
}

// FileGetter knows how to get the contents of a file by name
type FileGetter interface {
	GetFile(filename string) ([]byte, error)
}

type OSFileGetter struct {
	Root string
}

func (g *OSFileGetter) GetFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filepath.Join(g.Root, filename))
}

// Update updates the configmap with the data from the identified files.
// Existing configmap keys that are not included in the updates are left alone
// unless bootstrap is true in which case they are deleted.
func Update(fg FileGetter, kc corev1.ConfigMapInterface, name, namespace string, updates []ConfigMapUpdate, bootstrap bool, metrics *prometheus.GaugeVec, logger *logrus.Entry) error {
	cm, getErr := kc.Get(name, metav1.GetOptions{})
	isNotFound := errors.IsNotFound(getErr)
	if getErr != nil && !isNotFound {
		return fmt.Errorf("failed to fetch current state of configmap: %v", getErr)
	}

	if cm == nil || isNotFound {
		cm = &coreapi.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: namespace,
			},
		}
	}
	if cm.Data == nil || bootstrap {
		cm.Data = map[string]string{}
	}
	if cm.BinaryData == nil || bootstrap {
		cm.BinaryData = map[string][]byte{}
	}

	for _, upd := range updates {
		if upd.Filename == "" {
			logger.WithField("key", upd.Key).Debug("Deleting key.")
			delete(cm.Data, upd.Key)
			delete(cm.BinaryData, upd.Key)
			continue
		}

		content, err := fg.GetFile(upd.Filename)
		if err != nil {
			return fmt.Errorf("get file err: %v", err)
		}
		logger.WithFields(logrus.Fields{"key": upd.Key, "filename": upd.Filename}).Debug("Populating key.")
		value := content
		if upd.GZIP {
			buff := bytes.NewBuffer([]byte{})
			// TODO: this error is wildly unlikely for anything that
			// would actually fit in a configmap, we could just as well return
			// the error instead of falling back to the raw content
			z := gzip.NewWriter(buff)
			if _, err := z.Write(content); err != nil {
				logger.WithError(err).Error("failed to gzip content, falling back to raw")
			} else {
				if err := z.Close(); err != nil {
					logger.WithError(err).Error("failed to flush gzipped content (!?), falling back to raw")
				} else {
					value = buff.Bytes()
				}
			}
		}
		if utf8.ValidString(string(value)) {
			delete(cm.BinaryData, upd.Key)
			cm.Data[upd.Key] = string(value)
		} else {
			delete(cm.Data, upd.Key)
			cm.BinaryData[upd.Key] = value
		}
	}

	var updateErr error
	var verb string
	if getErr != nil && isNotFound {
		verb = "create"
		_, updateErr = kc.Create(cm)
	} else {
		verb = "update"
		_, updateErr = kc.Update(cm)
	}
	if updateErr != nil {
		return fmt.Errorf("%s config map err: %v", verb, updateErr)
	}
	if metrics != nil {
		var size float64
		for _, data := range cm.Data {
			size += float64(len(data))
		}
		// in a strict sense this can race to update the value with other goroutines
		// handling other events, but as events are serialized due to the fact that
		// merges are serial in repositories, this is effectively not an issue here
		metrics.WithLabelValues(cm.Name, cm.Namespace).Set(size)
	}
	return nil
}

// ConfigMapUpdate is populated with information about a config map that should
// be updated.
type ConfigMapUpdate struct {
	Key, Filename string
	GZIP          bool
}

// FilterChanges determines which of the changes are relevant for config updating, returning mapping of
// config map to key to filename to update that key from.
func FilterChanges(cfg plugins.ConfigUpdater, changes []github.PullRequestChange, defaultNamespace string, log *logrus.Entry) map[plugins.ConfigMapID][]ConfigMapUpdate {
	toUpdate := map[plugins.ConfigMapID][]ConfigMapUpdate{}
	for _, change := range changes {
		var cm plugins.ConfigMapSpec
		found := false

		for key, configMap := range cfg.Maps {
			var matchErr error
			found, matchErr = zglob.Match(key, change.Filename)
			if matchErr != nil {
				// Should not happen, log matchErr and continue
				log.WithError(matchErr).Info("key matching error")
				continue
			}

			if found {
				cm = configMap
				break
			}
		}

		if !found {
			continue // This file does not define a configmap
		}

		// Yes, update the configmap with the contents of this file
		for cluster, namespaces := range cm.Clusters {
			for _, ns := range namespaces {
				id := plugins.ConfigMapID{Name: cm.Name, Namespace: ns, Cluster: cluster}
				key := cm.Key
				if key == "" {
					key = path.Base(change.Filename)
					// if the key changed, we need to remove the old key
					if change.Status == github.PullRequestFileRenamed {
						oldKey := path.Base(change.PreviousFilename)
						// not setting the filename field will cause the key to be
						// deleted
						toUpdate[id] = append(toUpdate[id], ConfigMapUpdate{Key: oldKey})
					}
				}
				if change.Status == github.PullRequestFileRemoved {
					toUpdate[id] = append(toUpdate[id], ConfigMapUpdate{Key: key})
				} else {
					gzip := cfg.GZIP
					if cm.GZIP != nil {
						gzip = *cm.GZIP
					}
					toUpdate[id] = append(toUpdate[id], ConfigMapUpdate{Key: key, Filename: change.Filename, GZIP: gzip})
				}
			}
		}
	}
	return handleDefaultNamespace(toUpdate, defaultNamespace)
}

// handleDefaultNamespace ensures plugins.ConfigMapID.Namespace is not empty string
func handleDefaultNamespace(toUpdate map[plugins.ConfigMapID][]ConfigMapUpdate, defaultNamespace string) map[plugins.ConfigMapID][]ConfigMapUpdate {
	for cm, data := range toUpdate {
		if cm.Namespace == "" {
			key := plugins.ConfigMapID{Name: cm.Name, Namespace: defaultNamespace, Cluster: cm.Cluster}
			toUpdate[key] = append(toUpdate[key], data...)
			delete(toUpdate, cm)
		}
	}
	return toUpdate
}

func handle(gc githubClient, gitClient git.ClientFactory, kc corev1.ConfigMapsGetter, buildClusterCoreV1Clients map[string]corev1.CoreV1Interface, defaultNamespace string, log *logrus.Entry, pre github.PullRequestEvent, config plugins.ConfigUpdater, metrics *prometheus.GaugeVec) error {
	// Only consider newly merged PRs
	if pre.Action != github.PullRequestActionClosed {
		return nil
	}

	if len(config.Maps) == 0 { // Nothing to update
		return nil
	}

	pr := pre.PullRequest

	if !pr.Merged || pr.MergeSHA == nil || pr.Base.Repo.DefaultBranch != pr.Base.Ref {
		return nil
	}

	org := pr.Base.Repo.Owner.Login
	repo := pr.Base.Repo.Name

	// Which files changed in this PR?
	changes, err := gc.GetPullRequestChanges(org, repo, pr.Number)
	if err != nil {
		return err
	}

	message := func(name, cluster, namespace string, updates []ConfigMapUpdate, indent string) string {
		identifier := fmt.Sprintf("`%s` configmap", name)
		if namespace != "" {
			identifier = fmt.Sprintf("%s in namespace `%s`", identifier, namespace)
		}
		if cluster != "" {
			identifier = fmt.Sprintf("%s at cluster `%s`", identifier, cluster)
		}
		msg := fmt.Sprintf("%s using the following files:", identifier)
		for _, u := range updates {
			msg = fmt.Sprintf("%s\n%s- key `%s` using file `%s`", msg, indent, u.Key, u.Filename)
		}
		return msg
	}

	// Are any of the changes files ones that define a configmap we want to update?
	toUpdate := FilterChanges(config, changes, defaultNamespace, log)
	log.WithFields(logrus.Fields{
		"configmaps_to_update": len(toUpdate),
		"changes":              len(changes),
	}).Debug("Identified configmaps to update")

	var updated []string
	indent := " " // one space
	if len(toUpdate) > 1 {
		indent = "   " // three spaces for sub bullets
	}

	gitRepo, err := gitClient.ClientFor(org, repo)
	if err != nil {
		return err
	}
	defer func() {
		if err := gitClient.Clean(); err != nil {
			log.WithError(err).Error("Could not clean up git client cache.")
		}
	}()
	if err := gitRepo.Checkout(*pr.MergeSHA); err != nil {
		return err
	}

	var errs []error
	for cm, data := range toUpdate {
		logger := log.WithFields(logrus.Fields{"configmap": map[string]string{"name": cm.Name, "namespace": cm.Namespace, "cluster": cm.Cluster}})
		configMapClient, err := GetConfigMapClient(kc, cm.Namespace, buildClusterCoreV1Clients, cm.Cluster)
		if err != nil {
			log.WithError(err).Errorf("Failed to find configMap client")
			continue
		}
		if err := Update(&OSFileGetter{Root: gitRepo.Directory()}, configMapClient, cm.Name, cm.Namespace, data, bootstrapMode, metrics, logger); err != nil {
			errs = append(errs, err)
			continue
		}
		updated = append(updated, message(cm.Name, cm.Cluster, cm.Namespace, data, indent))
	}

	var msg string
	switch n := len(updated); n {
	case 0:
		return nil
	case 1:
		msg = fmt.Sprintf("Updated the %s", updated[0])
	default:
		msg = fmt.Sprintf("Updated the following %d configmaps:\n", n)
		for _, updateMsg := range updated {
			msg += fmt.Sprintf(" * %s\n", updateMsg) // one space indent
		}
	}

	if err := gc.CreateComment(org, repo, pr.Number, plugins.FormatResponseRaw(pr.Body, pr.HTMLURL, pr.User.Login, msg)); err != nil {
		errs = append(errs, fmt.Errorf("comment err: %v", err))
	}
	return utilerrors.NewAggregate(errs)
}

// GetConfigMapClient returns a configMap interface according to the given cluster and namespace
func GetConfigMapClient(kc corev1.ConfigMapsGetter, namespace string, buildClusterCoreV1Clients map[string]corev1.CoreV1Interface, cluster string) (corev1.ConfigMapInterface, error) {
	configMapClient := kc.ConfigMaps(namespace)
	if cluster != kube.DefaultClusterAlias {
		if client, ok := buildClusterCoreV1Clients[cluster]; ok {
			configMapClient = client.ConfigMaps(namespace)
		} else {
			return nil, fmt.Errorf("no k8s client is found for build cluster: '%s'", cluster)
		}
	}
	return configMapClient, nil
}
<|endoftext|>


<commit_before>package nschecker

var Sources = []Source{
	{
		Name:          "Amazon - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:           "https://www.amazon.co.jp/%E4%BB%BB%E5%A4%A9%E5%A0%82-Nintendo-Switch-Joy-Con-%E3%82%B0%E3%83%AC%E3%83%BC/dp/B01N5QLLT3/",
		AvailableText: `この商品は、<a href="/gp/help/customer/display.html?ie=UTF8&nodeId=643004">Amazon.co.jp</a> が販売、発送します。`,
	},
	{
		Name:          "Amazon - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:           "https://www.amazon.co.jp/Nintendo-Switch-Joy-Con-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%96%E3%83%AB%E3%83%BC-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%AC%E3%83%83%E3%83%89/dp/B01NCXFWIZ/",
		AvailableText: `この商品は、<a href="/gp/help/customer/display.html?ie=UTF8&nodeId=643004">Amazon.co.jp</a> が販売、発送します。`,
	},
	{
		Name:        "My Nintendo Store",
		URL:         "https://store.nintendo.co.jp/customize.html",
		SoldOutText: `<button class="btn btn__primary_soldout to_cart" type="submit"><span>SOLD OUT</span></button>`,
	},
	{
		Name:        "Yodobashi - Nintendo Switch Joy-Con(L)/(R)グレー [Nintendo Switch本体]",
		URL:         "http://www.yodobashi.com/product/100000001003431565/",
		SoldOutText: `<div class="salesInfo"><p>予定数の販売を終了しました</p></div>`,
	},
	{
		Name:        "Yodobashi - Nintendo Switch Joy-Con(L)ネオンブルー/(R)ネオンレッド [Nintendo Switch本体]",
		URL:         "http://www.yodobashi.com/product/100000001003431566/",
		SoldOutText: `<div class="salesInfo"><p>予定数の販売を終了しました</p></div>`,
	},
	{
		Name:        "Joshin - Nintendo Switch 本体【Joy-Con(L)/(R) グレー】",
		URL:         "http://joshinweb.jp/game/40519/4902370535709.html",
		SoldOutText: `<span class="fsL"><font color="blue"><b>販売休止中です</b></font><br></span>`,
	},
	{
		Name:        "Joshin - Nintendo Switch 本体【Joy-Con(L) ネオンブルー/(R) ネオンレッド】",
		URL:         "http://joshinweb.jp/game/40519/4902370535716.html",
		SoldOutText: `<span class="fsL"><font color="blue"><b>販売休止中です</b></font><br></span>`,
	},
	{
		Name:        "omni7(7net) - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "http://7net.omni7.jp/detail/2110595636",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "omni7(7net) - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "http://7net.omni7.jp/detail/2110595637",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "omni7(iyec) - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "http://iyec.omni7.jp/detail/4902370535709",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "omni7(iyec) - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "http://iyec.omni7.jp/detail/4902370535716",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "nojima - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "https://online.nojima.co.jp/Nintendo-HAC-S-KAAAA-%E3%80%90NSW%E3%80%91-%E3%83%8B%E3%83%B3%E3%83%86%E3%83%B3%E3%83%89%E3%83%BC%E3%82%B9%E3%82%A4%E3%83%83%E3%83%81%E6%9C%AC%E4%BD%93-Joy-Con%28L%29-%28R%29-%E3%82%B0%E3%83%AC%E3%83%BC/4902370535709/1/cd/",
		SoldOutText: `<span>完売御礼</span>`,
	},
	{
		Name:        "nojima - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "https://online.nojima.co.jp/Nintendo-HAC-S-KABAA-%E3%80%90NSW%E3%80%91-%E3%83%8B%E3%83%B3%E3%83%86%E3%83%B3%E3%83%89%E3%83%BC%E3%82%B9%E3%82%A4%E3%83%83%E3%83%81%E6%9C%AC%E4%BD%93-Joy-Con%28L%29-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%96%E3%83%AB%E3%83%BC-%28R%29-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%AC%E3%83%83%E3%83%89/4902370535716/1/cd/",
		SoldOutText: `<span>完売御礼</span>`,
	},
	{
		Name:        "toysrus - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "https://www.toysrus.co.jp/s/dsg-572182200",
		SoldOutText: `<span id="isStock_c" >在庫なし/入荷予定あり</span>`,
	},
	{
		Name:        "toysrus - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "https://www.toysrus.co.jp/s/dsg-572186500",
		SoldOutText: `<span id="isStock_c" >在庫なし/入荷予定あり</span>`,
	},
	{
		Name:        "tsutaya - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "http://shop.tsutaya.co.jp/Nintendo-Switch-Joy-Con-L-R-%E3%82%B0%E3%83%AC%E3%83%BC-HACSKAAAA/product-game-4902370535709/",
		SoldOutText: `<img src="/library/img/base/ic/btn_nostockL.png" alt="在庫なし" />`,
	},
	{
		Name:        "tsutaya - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "http://shop.tsutaya.co.jp/Nintendo-Switch-Joy-Con-L-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%96%E3%83%AB%E3%83%BC-R-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%AC%E3%83%83%E3%83%89-HACSKABAA/product-game-4902370535716/",
		SoldOutText: `<img src="/library/img/base/ic/btn_nostockL.png" alt="在庫なし" />`,
	},
	{
		Name:        "sofmap - Nintendo Switch",
		URL:         "http://www.sofmap.com/topics/exec/?id=5500",
		SoldOutText: `<IMG src="/images/system_icon/zaiko06.gif" alt="在庫切れ" border="0">`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー + ゼルダの伝説　ブレス オブ ザ ワイルド Nintendo Switch版",
		URL:         "http://books.rakuten.co.jp/rb/14779136/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー + マリオカート8 デラックス",
		URL:         "http://books.rakuten.co.jp/rb/14785337/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー 楽天あんしん延長保証",
		URL:         "http://books.rakuten.co.jp/rb/14655634/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド 楽天あんしん延長保証",
		URL:         "http://books.rakuten.co.jp/rb/14655635/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド + マリオカート8 デラックス",
		URL:         "http://books.rakuten.co.jp/rb/14787497/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド + 1-2-Switch ",
		URL:         "http://books.rakuten.co.jp/rb/14779141/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー",
		URL:         "http://books.rakuten.co.jp/rb/14647221/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド",
		URL:         "http://books.rakuten.co.jp/rb/14647222/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
}
<commit_msg>Yamada Revived<commit_after>package nschecker

var Sources = []Source{
	{
		Name:          "Amazon - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:           "https://www.amazon.co.jp/%E4%BB%BB%E5%A4%A9%E5%A0%82-Nintendo-Switch-Joy-Con-%E3%82%B0%E3%83%AC%E3%83%BC/dp/B01N5QLLT3/",
		AvailableText: `この商品は、<a href="/gp/help/customer/display.html?ie=UTF8&nodeId=643004">Amazon.co.jp</a> が販売、発送します。`,
	},
	{
		Name:          "Amazon - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:           "https://www.amazon.co.jp/Nintendo-Switch-Joy-Con-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%96%E3%83%AB%E3%83%BC-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%AC%E3%83%83%E3%83%89/dp/B01NCXFWIZ/",
		AvailableText: `この商品は、<a href="/gp/help/customer/display.html?ie=UTF8&nodeId=643004">Amazon.co.jp</a> が販売、発送します。`,
	},
	{
		Name:        "My Nintendo Store",
		URL:         "https://store.nintendo.co.jp/customize.html",
		SoldOutText: `<button class="btn btn__primary_soldout to_cart" type="submit"><span>SOLD OUT</span></button>`,
	},
	{
		Name:        "Yodobashi - Nintendo Switch Joy-Con(L)/(R)グレー [Nintendo Switch本体]",
		URL:         "http://www.yodobashi.com/product/100000001003431565/",
		SoldOutText: `<div class="salesInfo"><p>予定数の販売を終了しました</p></div>`,
	},
	{
		Name:        "Yodobashi - Nintendo Switch Joy-Con(L)ネオンブルー/(R)ネオンレッド [Nintendo Switch本体]",
		URL:         "http://www.yodobashi.com/product/100000001003431566/",
		SoldOutText: `<div class="salesInfo"><p>予定数の販売を終了しました</p></div>`,
	},
	{
		Name:        "Joshin - Nintendo Switch 本体【Joy-Con(L)/(R) グレー】",
		URL:         "http://joshinweb.jp/game/40519/4902370535709.html",
		SoldOutText: `<span class="fsL"><font color="blue"><b>販売休止中です</b></font><br></span>`,
	},
	{
		Name:        "Joshin - Nintendo Switch 本体【Joy-Con(L) ネオンブルー/(R) ネオンレッド】",
		URL:         "http://joshinweb.jp/game/40519/4902370535716.html",
		SoldOutText: `<span class="fsL"><font color="blue"><b>販売休止中です</b></font><br></span>`,
	},
	{
		Name:        "omni7(7net) - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "http://7net.omni7.jp/detail/2110595636",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "omni7(7net) - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "http://7net.omni7.jp/detail/2110595637",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "omni7(iyec) - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "http://iyec.omni7.jp/detail/4902370535709",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "omni7(iyec) - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "http://iyec.omni7.jp/detail/4902370535716",
		SoldOutText: `<input class="linkBtn js-pressTwice" type="submit" value="在庫切れ" title="在庫切れ"`,
	},
	{
		Name:        "nojima - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "https://online.nojima.co.jp/Nintendo-HAC-S-KAAAA-%E3%80%90NSW%E3%80%91-%E3%83%8B%E3%83%B3%E3%83%86%E3%83%B3%E3%83%89%E3%83%BC%E3%82%B9%E3%82%A4%E3%83%83%E3%83%81%E6%9C%AC%E4%BD%93-Joy-Con%28L%29-%28R%29-%E3%82%B0%E3%83%AC%E3%83%BC/4902370535709/1/cd/",
		SoldOutText: `<span>完売御礼</span>`,
	},
	{
		Name:        "nojima - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "https://online.nojima.co.jp/Nintendo-HAC-S-KABAA-%E3%80%90NSW%E3%80%91-%E3%83%8B%E3%83%B3%E3%83%86%E3%83%B3%E3%83%89%E3%83%BC%E3%82%B9%E3%82%A4%E3%83%83%E3%83%81%E6%9C%AC%E4%BD%93-Joy-Con%28L%29-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%96%E3%83%AB%E3%83%BC-%28R%29-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%AC%E3%83%83%E3%83%89/4902370535716/1/cd/",
		SoldOutText: `<span>完売御礼</span>`,
	},
	{
		Name:        "yamada - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "http://www.yamada-denkiweb.com/1177991016",
		SoldOutText: `<button type="submit" class="btn btn-disabled btn-block" disabled="disabled">売り切れました</button>`,
	},
	{
		Name:        "yamada - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "http://www.yamada-denkiweb.com/1177992013",
		SoldOutText: `<button type="submit" class="btn btn-disabled btn-block" disabled="disabled">売り切れました</button>`,
	},
	{
		Name:        "toysrus - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "https://www.toysrus.co.jp/s/dsg-572182200",
		SoldOutText: `<span id="isStock_c" >在庫なし/入荷予定あり</span>`,
	},
	{
		Name:        "toysrus - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "https://www.toysrus.co.jp/s/dsg-572186500",
		SoldOutText: `<span id="isStock_c" >在庫なし/入荷予定あり</span>`,
	},
	{
		Name:        "tsutaya - Nintendo Switch Joy-Con (L) / (R) グレー",
		URL:         "http://shop.tsutaya.co.jp/Nintendo-Switch-Joy-Con-L-R-%E3%82%B0%E3%83%AC%E3%83%BC-HACSKAAAA/product-game-4902370535709/",
		SoldOutText: `<img src="/library/img/base/ic/btn_nostockL.png" alt="在庫なし" />`,
	},
	{
		Name:        "tsutaya - Nintendo Switch Joy-Con (L) ネオンブルー/ (R) ネオンレッド",
		URL:         "http://shop.tsutaya.co.jp/Nintendo-Switch-Joy-Con-L-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%96%E3%83%AB%E3%83%BC-R-%E3%83%8D%E3%82%AA%E3%83%B3%E3%83%AC%E3%83%83%E3%83%89-HACSKABAA/product-game-4902370535716/",
		SoldOutText: `<img src="/library/img/base/ic/btn_nostockL.png" alt="在庫なし" />`,
	},
	{
		Name:        "sofmap - Nintendo Switch",
		URL:         "http://www.sofmap.com/topics/exec/?id=5500",
		SoldOutText: `<IMG src="/images/system_icon/zaiko06.gif" alt="在庫切れ" border="0">`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー + ゼルダの伝説　ブレス オブ ザ ワイルド Nintendo Switch版",
		URL:         "http://books.rakuten.co.jp/rb/14779136/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー + マリオカート8 デラックス",
		URL:         "http://books.rakuten.co.jp/rb/14785337/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー 楽天あんしん延長保証",
		URL:         "http://books.rakuten.co.jp/rb/14655634/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド 楽天あんしん延長保証",
		URL:         "http://books.rakuten.co.jp/rb/14655635/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド + マリオカート8 デラックス",
		URL:         "http://books.rakuten.co.jp/rb/14787497/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド + 1-2-Switch ",
		URL:         "http://books.rakuten.co.jp/rb/14779141/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L)/(R) グレー",
		URL:         "http://books.rakuten.co.jp/rb/14647221/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
	{
		Name:        "rakuten - Nintendo Switch Joy-Con(L) ネオンブルー/(R) ネオンレッド",
		URL:         "http://books.rakuten.co.jp/rb/14647222/",
		SoldOutText: `<span class="status">ご注文できない商品`,
	},
}
<|endoftext|>


<commit_before>/*
Déjà vu: Quickly detect already witnessed data.

Limited memory of witnessed data, oldest are forgotten. Library is thread safe.
Offers deterministic and probabilistic (over an order of magnatude less memory
consuming) implementation.
*/
package dejavu

import (
	"crypto/sha256"
	"github.com/willf/bloom"
	"sync"
)

// DejaVu witnesses data and recalls if seen before.
type DejaVu interface {

	// Witness data and add to memory. Returns true if previously seen.
	Witness(data []byte) bool

	// WitnessDigest is equivalent to the Winness method but bypasses hashing
	// the data. Use this to improve performance if you already happen
	// to have the sha256 digest.
	WitnessDigest(digest [sha256.Size]byte) bool
}

//////////////////////////////////
// DETERMINISTIC IMPLEMENTATION //
//////////////////////////////////

type deterministic struct {
	buffer [][sha256.Size]byte       // ring buffer
	size   int                       // ring buffer size
	index  int                       // current ring buffer index
	lookup map[[sha256.Size]byte]int // digest -> newest index (optimization)
	mutex  *sync.Mutex
}

// NewDeterministic creates a deterministic DejaVu memory. Will remember
// most recent entries within given entrie limit and forget older entries.
func NewDeterministic(entrieLimit uint32) DejaVu {
	return &deterministic{
		buffer: make([][sha256.Size]byte, entrieLimit),
		size:   int(entrieLimit),
		index:  0,
		lookup: make(map[[sha256.Size]byte]int),
		mutex:  new(sync.Mutex),
	}
}

func (d *deterministic) WitnessDigest(digest [sha256.Size]byte) bool {
	d.mutex.Lock()

	_, familiar := d.lookup[digest] // check if previously seen

	// rm oldest lookup key if no newer entry
	maxed := len(d.buffer) == d.size // overwriting oldest entry
	if maxed && (d.lookup[d.buffer[d.index]] == d.index) {
		delete(d.lookup, d.buffer[d.index]) // no newer entries
	}

	// add entry and update index/lookup
	d.buffer[d.index] = digest
	d.lookup[digest] = d.index
	d.index = (d.index + 1) % d.size

	d.mutex.Unlock()
	return familiar
}

func (d *deterministic) Witness(data []byte) bool {
	return d.WitnessDigest(sha256.Sum256(data))
}

//////////////////////////////////
// PROBABILISTIC IMPLEMENTATION //
//////////////////////////////////

type probabilistic struct {
	filters            [2]*bloom.BloomFilter // alternatingly replaced when maxed
	entrieLimit        uint32                // filter size
	falsePositiveRatio float64               // remember for buffer switch
	index              int                   // current filter index
	entries            uint32                // entries in currently indexed filter
	mutex              *sync.Mutex
}

// NewProbabilistic creates a probabilistic DejaVu memory. Probably remembers
// most recent entries within given entrie limit and false positive ratio.
// False positive ratio should be between 0.0 and 1.0.
func NewProbabilistic(entrieLimit uint32, falsePositiveRatio float64) DejaVu {
	a := bloom.NewWithEstimates(uint(entrieLimit), falsePositiveRatio)
	b := bloom.NewWithEstimates(uint(entrieLimit), falsePositiveRatio)
	return &probabilistic{
		filters:            [2]*bloom.BloomFilter{a, b},
		entrieLimit:        entrieLimit,
		falsePositiveRatio: falsePositiveRatio,
		index:              0,
		entries:            0,
		mutex:              new(sync.Mutex),
	}
}

func (p *probabilistic) WitnessDigest(digest [sha256.Size]byte) bool {
	p.mutex.Lock()

	// check if exists
	d := digest[:]
	familiar := p.filters[0].Test(d) || p.filters[1].Test(d)

	// always add in case its from the old buffer
	p.filters[p.index].Add(d)
	p.entries++

	// switch buffers if current is maxed
	if p.entries >= p.entrieLimit {
		p.entries = 0
		p.index = (p.index + 1) % 2
		f := bloom.NewWithEstimates(uint(p.entrieLimit), p.falsePositiveRatio)
		p.filters[p.index] = f // replace old filter
	}

	p.mutex.Unlock()
	return familiar
}

func (p *probabilistic) Witness(data []byte) bool {
	return p.WitnessDigest(sha256.Sum256(data))
}
<commit_msg>fix package doc header<commit_after>/*
Package dejavu offers quick detection of already witnessed data.

Limited memory of witnessed data, oldest are forgotten. Library is thread safe.
Offers deterministic and probabilistic (over an order of magnatude less memory
consuming) implementation.
*/
package dejavu

import (
	"crypto/sha256"
	"github.com/willf/bloom"
	"sync"
)

// DejaVu witnesses data and recalls if seen before.
type DejaVu interface {

	// Witness data and add to memory. Returns true if previously seen.
	Witness(data []byte) bool

	// WitnessDigest is equivalent to the Winness method but bypasses hashing
	// the data. Use this to improve performance if you already happen
	// to have the sha256 digest.
	WitnessDigest(digest [sha256.Size]byte) bool
}

//////////////////////////////////
// DETERMINISTIC IMPLEMENTATION //
//////////////////////////////////

type deterministic struct {
	buffer [][sha256.Size]byte       // ring buffer
	size   int                       // ring buffer size
	index  int                       // current ring buffer index
	lookup map[[sha256.Size]byte]int // digest -> newest index (optimization)
	mutex  *sync.Mutex
}

// NewDeterministic creates a deterministic DejaVu memory. Will remember
// most recent entries within given entrie limit and forget older entries.
func NewDeterministic(entrieLimit uint32) DejaVu {
	return &deterministic{
		buffer: make([][sha256.Size]byte, entrieLimit),
		size:   int(entrieLimit),
		index:  0,
		lookup: make(map[[sha256.Size]byte]int),
		mutex:  new(sync.Mutex),
	}
}

func (d *deterministic) WitnessDigest(digest [sha256.Size]byte) bool {
	d.mutex.Lock()

	_, familiar := d.lookup[digest] // check if previously seen

	// rm oldest lookup key if no newer entry
	maxed := len(d.buffer) == d.size // overwriting oldest entry
	if maxed && (d.lookup[d.buffer[d.index]] == d.index) {
		delete(d.lookup, d.buffer[d.index]) // no newer entries
	}

	// add entry and update index/lookup
	d.buffer[d.index] = digest
	d.lookup[digest] = d.index
	d.index = (d.index + 1) % d.size

	d.mutex.Unlock()
	return familiar
}

func (d *deterministic) Witness(data []byte) bool {
	return d.WitnessDigest(sha256.Sum256(data))
}

//////////////////////////////////
// PROBABILISTIC IMPLEMENTATION //
//////////////////////////////////

type probabilistic struct {
	filters            [2]*bloom.BloomFilter // alternatingly replaced when maxed
	entrieLimit        uint32                // filter size
	falsePositiveRatio float64               // remember for buffer switch
	index              int                   // current filter index
	entries            uint32                // entries in currently indexed filter
	mutex              *sync.Mutex
}

// NewProbabilistic creates a probabilistic DejaVu memory. Probably remembers
// most recent entries within given entrie limit and false positive ratio.
// False positive ratio should be between 0.0 and 1.0.
func NewProbabilistic(entrieLimit uint32, falsePositiveRatio float64) DejaVu {
	a := bloom.NewWithEstimates(uint(entrieLimit), falsePositiveRatio)
	b := bloom.NewWithEstimates(uint(entrieLimit), falsePositiveRatio)
	return &probabilistic{
		filters:            [2]*bloom.BloomFilter{a, b},
		entrieLimit:        entrieLimit,
		falsePositiveRatio: falsePositiveRatio,
		index:              0,
		entries:            0,
		mutex:              new(sync.Mutex),
	}
}

func (p *probabilistic) WitnessDigest(digest [sha256.Size]byte) bool {
	p.mutex.Lock()

	// check if exists
	d := digest[:]
	familiar := p.filters[0].Test(d) || p.filters[1].Test(d)

	// always add in case its from the old buffer
	p.filters[p.index].Add(d)
	p.entries++

	// switch buffers if current is maxed
	if p.entries >= p.entrieLimit {
		p.entries = 0
		p.index = (p.index + 1) % 2
		f := bloom.NewWithEstimates(uint(p.entrieLimit), p.falsePositiveRatio)
		p.filters[p.index] = f // replace old filter
	}

	p.mutex.Unlock()
	return familiar
}

func (p *probabilistic) Witness(data []byte) bool {
	return p.WitnessDigest(sha256.Sum256(data))
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"strings"
	"time"
)

// Currencies from database
type Currencies struct {
	Base  string             `json:"base"`
	Date  string             `json:"date"`
	Rates map[string]float32 `json:"rates"`
}

// LatestRates query db for most updated results for each currency
// supports different queries based on request parameters (base, symbols)
func LatestRates(baseParam, symbolsParam string) (*Currencies, error) {
	base := strings.ToUpper(baseParam)
	symbols := strings.ToUpper(symbolsParam)
	if base == "EUR" {
		base = ""
	}

	var rows *sql.Rows
	var err error

	switch {
	case base != "" && symbols != "":
		if strings.Contains(symbols, "EUR") {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT max(ratedate) FROM rates)
			UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
			WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1)`, base, symbols)
			if err != nil {
				return nil, err
			}
		} else {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT max(ratedate) FROM rates)`, base, symbols)
			if err != nil {
				return nil, err
			}
		}
	case base != "" && symbols == "":
		rows, err = db.Query(`SELECT currency,
		rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
		AND currency = $1) AS rate FROM rates WHERE currency !=$1
		AND ratedate = (SELECT max(ratedate) FROM rates)
		UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
		WHERE ratedate = (SELECT max(ratedate) FROM rates) AND currency = $1)`, base)
		if err != nil {
			return nil, err
		}
	case base == "" && symbols != "":
		rows, err = db.Query(`SELECT currency, rate
		FROM rates WHERE currency = ANY(string_to_array($1, ','))
		AND ratedate = (SELECT max(ratedate) FROM rates)`, symbols)
		if err != nil {
			return nil, err
		}
	default:
		rows, err = db.Query(`SELECT currency, rate FROM rates
		WHERE ratedate = (SELECT max(ratedate) FROM rates)`)
		if err != nil {
			return nil, err
		}
	}
	defer rows.Close()

	currencies := &Currencies{Rates: make(map[string]float32)}
	for rows.Next() {
		var (
			currency string
			rate     float32
		)
		if err := rows.Scan(&currency, &rate); err != nil {
			return nil, err
		}
		currencies.Rates[currency] = rate
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	var date time.Time
	err = db.QueryRow("SELECT max(ratedate) FROM rates").Scan(&date)
	currencies.Date = date.Format("2006-01-02")

	if base != "" {
		currencies.Base = base
	} else {
		currencies.Base = "EUR"
	}

	return currencies, nil
}

// HistoricalRates query db for most updated results for each currency
// supports different queries based on request parameters (base, symbols)
func HistoricalRates(baseParam, symbolsParam, dateParam string) (*Currencies, error) {
	base := strings.ToUpper(baseParam)
	symbols := strings.ToUpper(symbolsParam)
	if base == "EUR" {
		base = ""
	}

	var rows *sql.Rows
	var err error

	switch {
	case base != "" && symbols != "":
		if strings.Contains(symbols, "EUR") {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates
			WHERE ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $3 LIMIT 1)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $3 LIMIT 1)
			UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
			WHERE ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $3 LIMIT 1)
			AND currency = $1)`, base, symbols, dateParam)
			if err != nil {
				return nil, err
			}
		} else {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates
			WHERE ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $3 LIMIT 1)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT max(ratedate) FROM rates)`, base, symbols, dateParam)
			if err != nil {
				return nil, err
			}
		}
	case base != "" && symbols == "":
		rows, err = db.Query(`SELECT currency,
		rate / (SELECT rate FROM rates WHERE ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $2 LIMIT 1)
		AND currency = $1) AS rate FROM rates WHERE currency !=$1
		AND ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $2 LIMIT 1)
		UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
		WHERE ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $2 LIMIT 1)`, base, dateParam)
		if err != nil {
			return nil, err
		}
	case base == "" && symbols != "":
		rows, err = db.Query(`SELECT currency, rate
		FROM rates WHERE currency = ANY(string_to_array($1, ','))
		AND ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $2 LIMIT 1)`, symbols, dateParam)
		if err != nil {
			return nil, err
		}
	default:
		rows, err = db.Query(`SELECT currency, rate FROM rates
		WHERE ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $1 LIMIT 1)`,
			dateParam)
		if err != nil {
			return nil, err
		}
	}
	defer rows.Close()

	currencies := &Currencies{Rates: make(map[string]float32)}
	for rows.Next() {
		var (
			currency string
			rate     float32
		)
		if err := rows.Scan(&currency, &rate); err != nil {
			return nil, err
		}
		currencies.Rates[currency] = rate
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	var date string
	err = db.QueryRow("SELECT ratedate FROM rates WHERE ratedate <= $1 LIMIT 1", date).Scan(&date)
	currencies.Date = date

	if base != "" {
		currencies.Base = base
	} else {
		currencies.Base = "EUR"
	}

	return currencies, nil
}
<commit_msg>Fix sql queries<commit_after>package models

import (
	"database/sql"
	"strings"
	"time"
)

// Currencies from database
type Currencies struct {
	Base  string             `json:"base"`
	Date  string             `json:"date"`
	Rates map[string]float32 `json:"rates"`
}

// LatestRates query db for most updated results for each currency
// supports different queries based on request parameters (base, symbols)
func LatestRates(baseParam, symbolsParam string) (*Currencies, error) {
	base := strings.ToUpper(baseParam)
	symbols := strings.ToUpper(symbolsParam)
	if base == "EUR" {
		base = ""
	}

	var rows *sql.Rows
	var err error

	switch {
	case base != "" && symbols != "":
		if strings.Contains(symbols, "EUR") {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT max(ratedate) FROM rates)
			UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
			WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1)`, base, symbols)
			if err != nil {
				return nil, err
			}
		} else {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT max(ratedate) FROM rates)`, base, symbols)
			if err != nil {
				return nil, err
			}
		}
	case base != "" && symbols == "":
		rows, err = db.Query(`SELECT currency,
		rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
		AND currency = $1) AS rate FROM rates WHERE currency !=$1
		AND ratedate = (SELECT max(ratedate) FROM rates)
		UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
		WHERE ratedate = (SELECT max(ratedate) FROM rates) AND currency = $1)`, base)
		if err != nil {
			return nil, err
		}
	case base == "" && symbols != "":
		rows, err = db.Query(`SELECT currency, rate
		FROM rates WHERE currency = ANY(string_to_array($1, ','))
		AND ratedate = (SELECT max(ratedate) FROM rates)`, symbols)
		if err != nil {
			return nil, err
		}
	default:
		rows, err = db.Query(`SELECT currency, rate FROM rates
		WHERE ratedate = (SELECT max(ratedate) FROM rates)`)
		if err != nil {
			return nil, err
		}
	}
	defer rows.Close()

	currencies := &Currencies{Rates: make(map[string]float32)}
	for rows.Next() {
		var (
			currency string
			rate     float32
		)
		if err := rows.Scan(&currency, &rate); err != nil {
			return nil, err
		}
		currencies.Rates[currency] = rate
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	var date time.Time
	err = db.QueryRow("SELECT max(ratedate) FROM rates").Scan(&date)
	currencies.Date = date.Format("2006-01-02")

	if base != "" {
		currencies.Base = base
	} else {
		currencies.Base = "EUR"
	}

	return currencies, nil
}

// HistoricalRates query db for historical results for each currency
// supports different queries based on request parameters (base, symbols, date)
func HistoricalRates(baseParam, symbolsParam, dateParam string) (*Currencies, error) {
	base := strings.ToUpper(baseParam)
	symbols := strings.ToUpper(symbolsParam)
	if base == "EUR" {
		base = ""
	}

	var rows *sql.Rows
	var err error

	switch {
	case base != "" && symbols != "":
		if strings.Contains(symbols, "EUR") {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT max(ratedate) FROM rates)
			UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
			WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1)`, base, symbols)
			if err != nil {
				return nil, err
			}
		} else {
			rows, err = db.Query(`SELECT currency,
			rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
			AND currency = $1) AS rate FROM rates WHERE currency !=$1
			AND currency = ANY(string_to_array($2, ','))
			AND ratedate = (SELECT max(ratedate) FROM rates)`, base, symbols)
			if err != nil {
				return nil, err
			}
		}
	case base != "" && symbols == "":
		rows, err = db.Query(`SELECT currency,
		rate / (SELECT rate FROM rates WHERE ratedate = (SELECT max(ratedate) FROM rates)
		AND currency = $1) AS rate FROM rates WHERE currency !=$1
		AND ratedate = (SELECT max(ratedate) FROM rates)
		UNION ALL SELECT 'EUR', 1 / (SELECT rate FROM rates
		WHERE ratedate = (SELECT max(ratedate) FROM rates) AND currency = $1)`, base)
		if err != nil {
			return nil, err
		}
	case base == "" && symbols != "":
		rows, err = db.Query(`SELECT currency, rate
		FROM rates WHERE currency = ANY(string_to_array($1, ','))
		AND ratedate = (SELECT max(ratedate) FROM rates)`, symbols)
		if err != nil {
			return nil, err
		}
	default:
		rows, err = db.Query(`SELECT currency, rate FROM rates
		WHERE ratedate = (SELECT ratedate FROM rates WHERE ratedate <= $1 LIMIT 1)`,
			dateParam)
		if err != nil {
			return nil, err
		}
	}
	defer rows.Close()

	currencies := &Currencies{Rates: make(map[string]float32)}
	for rows.Next() {
		var (
			currency string
			rate     float32
		)
		if err := rows.Scan(&currency, &rate); err != nil {
			return nil, err
		}
		currencies.Rates[currency] = rate
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	var date string
	err = db.QueryRow("SELECT ratedate FROM rates WHERE ratedate <= $1 LIMIT 1", dateParam).Scan(&date)
	currencies.Date = date

	if base != "" {
		currencies.Base = base
	} else {
		currencies.Base = "EUR"
	}

	return currencies, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package customresource

import (
	"context"

	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)

type statusStrategy struct {
	customResourceStrategy
}

func NewStatusStrategy(strategy customResourceStrategy) statusStrategy {
	return statusStrategy{strategy}
}

// GetResetFields returns the set of fields that get reset by the strategy
// and should not be modified by the user.
func (a statusStrategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
	fields := map[fieldpath.APIVersion]*fieldpath.Set{
		fieldpath.APIVersion(a.customResourceStrategy.kind.GroupVersion().String()): fieldpath.NewSet(
			// Note that if there are other top level fields unique to CRDs,
			// those will also get removed by the apiserver prior to persisting,
			// but wont be added to the resetFields set.

			// This isn't an issue now, but if it becomes an issue in the future
			// we might need a mechanism that is the inverse of resetFields where
			// you specify only the fields to be kept rather than the fields to be wiped
			// that way you could wipe everything but the status in this case.
			fieldpath.MakePathOrDie("spec"),
		),
	}

	return fields
}

func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	// update is only allowed to set status
	newCustomResourceObject := obj.(*unstructured.Unstructured)
	newCustomResource := newCustomResourceObject.UnstructuredContent()
	status, ok := newCustomResource["status"]

	// managedFields must be preserved since it's been modified to
	// track changed fields in the status update.
	managedFields := newCustomResourceObject.GetManagedFields()

	// copy old object into new object
	oldCustomResourceObject := old.(*unstructured.Unstructured)
	// overridding the resourceVersion in metadata is safe here, we have already checked that
	// new object and old object have the same resourceVersion.
	*newCustomResourceObject = *oldCustomResourceObject.DeepCopy()

	// set status
	newCustomResourceObject.SetManagedFields(managedFields)
	newCustomResource = newCustomResourceObject.UnstructuredContent()
	if ok {
		newCustomResource["status"] = status
	} else {
		delete(newCustomResource, "status")
	}
}

// ValidateUpdate is the default update validation for an end user updating status.
func (a statusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return a.customResourceStrategy.validator.ValidateStatusUpdate(ctx, obj, old, a.scale)
}
<commit_msg>Fixed several spelling mistakes<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package customresource

import (
	"context"

	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)

type statusStrategy struct {
	customResourceStrategy
}

func NewStatusStrategy(strategy customResourceStrategy) statusStrategy {
	return statusStrategy{strategy}
}

// GetResetFields returns the set of fields that get reset by the strategy
// and should not be modified by the user.
func (a statusStrategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
	fields := map[fieldpath.APIVersion]*fieldpath.Set{
		fieldpath.APIVersion(a.customResourceStrategy.kind.GroupVersion().String()): fieldpath.NewSet(
			// Note that if there are other top level fields unique to CRDs,
			// those will also get removed by the apiserver prior to persisting,
			// but won't be added to the resetFields set.

			// This isn't an issue now, but if it becomes an issue in the future
			// we might need a mechanism that is the inverse of resetFields where
			// you specify only the fields to be kept rather than the fields to be wiped
			// that way you could wipe everything but the status in this case.
			fieldpath.MakePathOrDie("spec"),
		),
	}

	return fields
}

func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	// update is only allowed to set status
	newCustomResourceObject := obj.(*unstructured.Unstructured)
	newCustomResource := newCustomResourceObject.UnstructuredContent()
	status, ok := newCustomResource["status"]

	// managedFields must be preserved since it's been modified to
	// track changed fields in the status update.
	managedFields := newCustomResourceObject.GetManagedFields()

	// copy old object into new object
	oldCustomResourceObject := old.(*unstructured.Unstructured)
	// overridding the resourceVersion in metadata is safe here, we have already checked that
	// new object and old object have the same resourceVersion.
	*newCustomResourceObject = *oldCustomResourceObject.DeepCopy()

	// set status
	newCustomResourceObject.SetManagedFields(managedFields)
	newCustomResource = newCustomResourceObject.UnstructuredContent()
	if ok {
		newCustomResource["status"] = status
	} else {
		delete(newCustomResource, "status")
	}
}

// ValidateUpdate is the default update validation for an end user updating status.
func (a statusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return a.customResourceStrategy.validator.ValidateStatusUpdate(ctx, obj, old, a.scale)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package buildpacks

import "sync"

type pulledImages struct {
	images map[builderRunnerPair]bool
	lock   sync.Mutex
}

type builderRunnerPair struct {
	builder string
	runner  string
}

func (p *pulledImages) AreAlreadyPulled(builder, runImage string) bool {
	p.lock.Lock()
	defer p.lock.Unlock()

	return p.images[builderRunnerPair{
		builder: builder,
		runner:  runImage,
	}]
}

func (p *pulledImages) MarkAsPulled(builder, runImage string) {
	p.lock.Lock()
	defer p.lock.Unlock()

	if p.images == nil {
		p.images = map[builderRunnerPair]bool{}
	}

	p.images[builderRunnerPair{
		builder: builder,
		runner:  runImage,
	}] = true
}
<commit_msg>Feedback<commit_after>/*
Copyright 2020 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package buildpacks

import "sync"

type pulledImages struct {
	images map[imageTuple]bool
	lock   sync.Mutex
}

type imageTuple struct {
	builder string
	runner  string
}

func (p *pulledImages) AreAlreadyPulled(builder, runImage string) bool {
	p.lock.Lock()
	defer p.lock.Unlock()

	return p.images[tuple(builder, runImage)]
}

func (p *pulledImages) MarkAsPulled(builder, runImage string) {
	p.lock.Lock()
	defer p.lock.Unlock()

	if p.images == nil {
		p.images = map[imageTuple]bool{}
	}

	p.images[tuple(builder, runImage)] = true
}

func tuple(builder, runImage string) imageTuple {
	return imageTuple{
		builder: builder,
		runner:  runImage,
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Philipp Meinen. All rights reserved.
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package pubsub

import (
	"sync"

	"fmt"
	"github.com/phicode/go/path"
)

// A TopicTree is a hierarchical structure of topics.
// Subscriber can subscribe on:
// - the root level to receive all messages
// - a 'leaf' level to receive a subset of messages, filtered for the specific path
// Published messages delivered to the root and all leafs of the publishing path.
type TopicTree interface {
	// Publish publishes a new message to the tree.
	// Publish panics if the supplied path is not valid.
	Publish(path string, msg interface{})

	// PublishPath publishes a new message to the tree.
	PublishPath(path *path.Path, msg interface{})

	// Subscribe creates a new Subscription on the tree.
	// Subscribe panics if the supplied path is not valid.
	Subscribe(path string) Subscription

	// SubscribePath creates a new Subscription on the tree.
	SubscribePath(path *path.Path) Subscription

	//TODO: per level/subscriber/global? scrap altogether ?
	//NumSubscribers(path string) int

	//TODO: per level/subscriber/global? scrap altogether ?
	//SetHWM(path string, hwm int, recursive bool)

	//TODO: subscriptions where the path of the received message is communicated
}

type tt struct {
	topic Topic

	mu    sync.Mutex
	refs  int64 // the number of subscriptions on this level or on child trees
	leafs map[string]*tt
}

var _ (TopicTree) = (*tt)(nil)

func NewTopicTree() TopicTree { return newtt() }

func (t *tt) Publish(treePath string, msg interface{}) {
	path, ok := path.Parse(treePath)
	if !ok {
		panic(fmt.Errorf("invalid path: %q", treePath))
	}
	t.PublishPath(&path, msg)
}
func (t *tt) PublishPath(path *path.Path, msg interface{}) {
	t.publish(path, 0, msg)
}

// publishing:
// - walk the tree until the final leaf is reached
// - publish the message on the topics of all leafs on the way
func (t *tt) publish(p *path.Path, depth int, msg interface{}) {
	t.topic.Publish(msg)

	t.mu.Lock()
	defer t.mu.Unlock()

	if next, ok := p.Elem(depth); ok {
		if leaf, ok := t.leafs[next]; ok {
			leaf.publish(p, depth+1, msg)
		}
	}
}

func (t *tt) Subscribe(treePath string) Subscription {
	path, ok := path.Parse(treePath)
	if !ok {
		panic(fmt.Errorf("invalid path: %q", treePath))
	}
	return t.SubscribePath(&path)
}
func (t *tt) SubscribePath(path *path.Path) Subscription {
	return t.subscribe(t, path, 0)
}

// subscription:
// - walk the tree until the requested leaf is reached
// - create non-existent leafs on the way
// - register on the leafs topic
func (t *tt) subscribe(root *tt, p *path.Path, depth int) Subscription {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.refs++

	if next, ok := p.Elem(depth); ok {
		// subscription is not for this level, propagate to the next level in the hierarchy
		leaf, ok := t.leafs[next]
		if !ok {
			leaf = newtt()
			t.leafs[next] = leaf
		}
		return leaf.subscribe(root, p, depth+1)
	}

	s := t.topic.Subscribe()
	return &ttsub{
		root: root,
		p:    p,
		s:    s,
	}
}

// unsubscribe:
// - walk the tree until the final leaf is reached
// - unsubscribe from the leaf
// - remove all unused leafs
func (t *tt) unsubscribe(p *path.Path, depth int) int64 {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.refs--

	if next, ok := p.Elem(depth); ok {
		// subscription is not for this level, propagate to the next level in the hierarchy
		leaf, ok := t.leafs[next]
		if !ok {
			panic("invalid unsubscribe")
		}
		if leaf.unsubscribe(p, depth+1) == 0 {
			delete(t.leafs, next)
		}
	}

	return t.refs
}

type ttsub struct {
	root *tt
	p    *path.Path
	s    Subscription
}

var _ Subscription = (*ttsub)(nil)

func (s *ttsub) C() <-chan interface{} {
	return s.s.C()
}

func (s *ttsub) Unsubscribe() bool {
	if !s.s.Unsubscribe() {
		return false
	}
	s.root.unsubscribe(s.p, 0)
	return true
}

func newtt() *tt {
	return &tt{
		topic: NewTopic(),
		leafs: make(map[string]*tt),
	}
}
<commit_msg>pubsub: goimports<commit_after>// Copyright 2015 Philipp Meinen. All rights reserved.
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package pubsub

import (
	"fmt"
	"sync"

	"github.com/phicode/go/path"
)

// A TopicTree is a hierarchical structure of topics.
// Subscriber can subscribe on:
// - the root level to receive all messages
// - a 'leaf' level to receive a subset of messages, filtered for the specific path
// Published messages delivered to the root and all leafs of the publishing path.
type TopicTree interface {
	// Publish publishes a new message to the tree.
	// Publish panics if the supplied path is not valid.
	Publish(path string, msg interface{})

	// PublishPath publishes a new message to the tree.
	PublishPath(path *path.Path, msg interface{})

	// Subscribe creates a new Subscription on the tree.
	// Subscribe panics if the supplied path is not valid.
	Subscribe(path string) Subscription

	// SubscribePath creates a new Subscription on the tree.
	SubscribePath(path *path.Path) Subscription

	//TODO: per level/subscriber/global? scrap altogether ?
	//NumSubscribers(path string) int

	//TODO: per level/subscriber/global? scrap altogether ?
	//SetHWM(path string, hwm int, recursive bool)

	//TODO: subscriptions where the path of the received message is communicated
}

type tt struct {
	topic Topic

	mu    sync.Mutex
	refs  int64 // the number of subscriptions on this level or on child trees
	leafs map[string]*tt
}

var _ (TopicTree) = (*tt)(nil)

func NewTopicTree() TopicTree { return newtt() }

func (t *tt) Publish(treePath string, msg interface{}) {
	path, ok := path.Parse(treePath)
	if !ok {
		panic(fmt.Errorf("invalid path: %q", treePath))
	}
	t.PublishPath(&path, msg)
}
func (t *tt) PublishPath(path *path.Path, msg interface{}) {
	t.publish(path, 0, msg)
}

// publishing:
// - walk the tree until the final leaf is reached
// - publish the message on the topics of all leafs on the way
func (t *tt) publish(p *path.Path, depth int, msg interface{}) {
	t.topic.Publish(msg)

	t.mu.Lock()
	defer t.mu.Unlock()

	if next, ok := p.Elem(depth); ok {
		if leaf, ok := t.leafs[next]; ok {
			leaf.publish(p, depth+1, msg)
		}
	}
}

func (t *tt) Subscribe(treePath string) Subscription {
	path, ok := path.Parse(treePath)
	if !ok {
		panic(fmt.Errorf("invalid path: %q", treePath))
	}
	return t.SubscribePath(&path)
}
func (t *tt) SubscribePath(path *path.Path) Subscription {
	return t.subscribe(t, path, 0)
}

// subscription:
// - walk the tree until the requested leaf is reached
// - create non-existent leafs on the way
// - register on the leafs topic
func (t *tt) subscribe(root *tt, p *path.Path, depth int) Subscription {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.refs++

	if next, ok := p.Elem(depth); ok {
		// subscription is not for this level, propagate to the next level in the hierarchy
		leaf, ok := t.leafs[next]
		if !ok {
			leaf = newtt()
			t.leafs[next] = leaf
		}
		return leaf.subscribe(root, p, depth+1)
	}

	s := t.topic.Subscribe()
	return &ttsub{
		root: root,
		p:    p,
		s:    s,
	}
}

// unsubscribe:
// - walk the tree until the final leaf is reached
// - unsubscribe from the leaf
// - remove all unused leafs
func (t *tt) unsubscribe(p *path.Path, depth int) int64 {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.refs--

	if next, ok := p.Elem(depth); ok {
		// subscription is not for this level, propagate to the next level in the hierarchy
		leaf, ok := t.leafs[next]
		if !ok {
			panic("invalid unsubscribe")
		}
		if leaf.unsubscribe(p, depth+1) == 0 {
			delete(t.leafs, next)
		}
	}

	return t.refs
}

type ttsub struct {
	root *tt
	p    *path.Path
	s    Subscription
}

var _ Subscription = (*ttsub)(nil)

func (s *ttsub) C() <-chan interface{} {
	return s.s.C()
}

func (s *ttsub) Unsubscribe() bool {
	if !s.s.Unsubscribe() {
		return false
	}
	s.root.unsubscribe(s.p, 0)
	return true
}

func newtt() *tt {
	return &tt{
		topic: NewTopic(),
		leafs: make(map[string]*tt),
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package identicon

import (
	"crypto/md5"
	"fmt"
	"image"
	"image/color"
	"math"
)

const (
	minSize       = 16 // 图片的最小尺寸
	maxForeColors = 32 // 在New()函数中可以指定的最大颜色数量
)

// Identicon 用于产生统一尺寸的头像。
// 可以根据用户提供的数据，经过一定的算法，自动产生相应的图案和颜色。
type Identicon struct {
	foreColors []color.Color
	backColor  color.Color
	size       int
}

// 声明一个Identicon实例。
// size表示整个头像的大小。
// back表示前景色。
// fore表示所有可能的前景色，会为每个图像随机挑选一个作为其前景色。不要与背景色太相近。
func New(size int, back color.Color, fore ...color.Color) (*Identicon, error) {
	if len(fore) == 0 || len(fore) > maxForeColors {
		return nil, fmt.Errorf("前景色数量必须介于[1]~[%v]之间，当前为[%v]", maxForeColors, len(fore))
	}

	if size < minSize {
		return nil, fmt.Errorf("参数size的值(%v)不能小于%v", size, minSize)
	}

	return &Identicon{
		foreColors: fore,
		backColor:  back,
		size:       size,
	}, nil
}

// 根据data数据产生一张唯一性的头像图片。
func (i *Identicon) Make(data []byte) image.Image {
	h := md5.New()
	h.Write(data)
	sum := h.Sum(nil)

	// 第一个方块
	index := int(math.Abs(float64(sum[0]+sum[1]+sum[2]+sum[3]))) % len(blocks)
	b1 := blocks[index]

	// 第二个方块
	index = int(math.Abs(float64(sum[4]+sum[5]+sum[6]+sum[7]))) % len(blocks)
	b2 := blocks[index]

	// 中间方块
	index = int(math.Abs(float64(sum[8]+sum[9]+sum[10]+sum[11]))) % len(centerBlocks)
	c := centerBlocks[index]

	// 旋转角度
	angle := int(math.Abs(float64(sum[12]+sum[13]+sum[14]))) % 4

	// 根据最后一个字段，获取前景颜色
	colorIndex := int(math.Abs(float64(sum[15]))) % len(i.foreColors)

	p := image.NewPaletted(image.Rect(0, 0, i.size, i.size), []color.Color{i.backColor, i.foreColors[colorIndex]})

	drawBlocks(p, i.size, c, b1, b2, angle)
	return p
}

// 根据data数据产生一张唯一性的头像图片。
// size 头像的大小。
// back, fore头像的背景和前景色。
func Make(size int, back, fore color.Color, data []byte) (image.Image, error) {
	if size < minSize {
		return nil, fmt.Errorf("New:产生的图片尺寸(%v)不能小于%v", size, minSize)
	}

	h := md5.New()
	h.Write(data)
	sum := h.Sum(nil)

	// 第一个方块
	index := int(math.Abs(float64(sum[0]+sum[1]+sum[2]+sum[3]))) % len(blocks)
	b1 := blocks[index]

	// 第二个方块
	index = int(math.Abs(float64(sum[4]+sum[5]+sum[6]+sum[7]))) % len(blocks)
	b2 := blocks[index]

	// 中间方块
	index = int(math.Abs(float64(sum[8]+sum[9]+sum[10]+sum[11]))) % len(centerBlocks)
	c := centerBlocks[index]

	// 旋转角度
	angle := int(math.Abs(float64(sum[12]+sum[13]+sum[14]+sum[15]))) % 4

	p := image.NewPaletted(image.Rect(0, 0, size, size), []color.Color{back, fore})
	drawBlocks(p, size, c, b1, b2, angle)
	return p, nil
}

// 将九个方格都填上内容。
// p为画板。
// c为中间方格的填充函数。
// b1,b2为边上8格的填充函数。
// angle为b1,b2的起始旋转角度。
func drawBlocks(p *image.Paletted, size int, c, b1, b2 blockFunc, angle int) {
	blockSize := float64(size / 3) // 每个格子的长宽
	twoBlockSize := 2 * blockSize

	incr := func() { // 增加angle的值，但不会大于3
		if angle > 2 {
			angle = 0
		} else {
			angle++
		}
	}

	c(p, blockSize, blockSize, blockSize, 0)

	b1(p, 0, 0, blockSize, angle)
	b2(p, blockSize, 0, blockSize, angle)

	incr()
	b1(p, twoBlockSize, 0, blockSize, angle)
	b2(p, twoBlockSize, blockSize, blockSize, angle)

	incr()
	b1(p, twoBlockSize, twoBlockSize, blockSize, angle)
	b2(p, blockSize, twoBlockSize, blockSize, angle)

	incr()
	b1(p, 0, twoBlockSize, blockSize, angle)
	b2(p, 0, blockSize, blockSize, angle)
}
<commit_msg>改进错误提示<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package identicon

import (
	"crypto/md5"
	"fmt"
	"image"
	"image/color"
	"math"
)

const (
	minSize       = 16 // 图片的最小尺寸
	maxForeColors = 32 // 在New()函数中可以指定的最大颜色数量
)

// Identicon 用于产生统一尺寸的头像。
// 可以根据用户提供的数据，经过一定的算法，自动产生相应的图案和颜色。
type Identicon struct {
	foreColors []color.Color
	backColor  color.Color
	size       int
}

// 声明一个Identicon实例。
// size表示整个头像的大小。
// back表示前景色。
// fore表示所有可能的前景色，会为每个图像随机挑选一个作为其前景色。不要与背景色太相近。
func New(size int, back color.Color, fore ...color.Color) (*Identicon, error) {
	if len(fore) == 0 || len(fore) > maxForeColors {
		return nil, fmt.Errorf("前景色数量必须介于[1]~[%v]之间，当前为[%v]", maxForeColors, len(fore))
	}

	if size < minSize {
		return nil, fmt.Errorf("参数size的值(%v)不能小于%v", size, minSize)
	}

	return &Identicon{
		foreColors: fore,
		backColor:  back,
		size:       size,
	}, nil
}

// 根据data数据产生一张唯一性的头像图片。
func (i *Identicon) Make(data []byte) image.Image {
	h := md5.New()
	h.Write(data)
	sum := h.Sum(nil)

	// 第一个方块
	index := int(math.Abs(float64(sum[0]+sum[1]+sum[2]+sum[3]))) % len(blocks)
	b1 := blocks[index]

	// 第二个方块
	index = int(math.Abs(float64(sum[4]+sum[5]+sum[6]+sum[7]))) % len(blocks)
	b2 := blocks[index]

	// 中间方块
	index = int(math.Abs(float64(sum[8]+sum[9]+sum[10]+sum[11]))) % len(centerBlocks)
	c := centerBlocks[index]

	// 旋转角度
	angle := int(math.Abs(float64(sum[12]+sum[13]+sum[14]))) % 4

	// 根据最后一个字段，获取前景颜色
	colorIndex := int(math.Abs(float64(sum[15]))) % len(i.foreColors)

	p := image.NewPaletted(image.Rect(0, 0, i.size, i.size), []color.Color{i.backColor, i.foreColors[colorIndex]})
	drawBlocks(p, i.size, c, b1, b2, angle)
	return p
}

// 根据data数据产生一张唯一性的头像图片。
// size 头像的大小。
// back, fore头像的背景和前景色。
func Make(size int, back, fore color.Color, data []byte) (image.Image, error) {
	if size < minSize {
		return nil, fmt.Errorf("参数size的值(%v)不能小于%v", size, minSize)
	}

	h := md5.New()
	h.Write(data)
	sum := h.Sum(nil)

	// 第一个方块
	index := int(math.Abs(float64(sum[0]+sum[1]+sum[2]+sum[3]))) % len(blocks)
	b1 := blocks[index]

	// 第二个方块
	index = int(math.Abs(float64(sum[4]+sum[5]+sum[6]+sum[7]))) % len(blocks)
	b2 := blocks[index]

	// 中间方块
	index = int(math.Abs(float64(sum[8]+sum[9]+sum[10]+sum[11]))) % len(centerBlocks)
	c := centerBlocks[index]

	// 旋转角度
	angle := int(math.Abs(float64(sum[12]+sum[13]+sum[14]+sum[15]))) % 4

	p := image.NewPaletted(image.Rect(0, 0, size, size), []color.Color{back, fore})
	drawBlocks(p, size, c, b1, b2, angle)
	return p, nil
}

// 将九个方格都填上内容。
// p为画板。
// c为中间方格的填充函数。
// b1,b2为边上8格的填充函数。
// angle为b1,b2的起始旋转角度。
func drawBlocks(p *image.Paletted, size int, c, b1, b2 blockFunc, angle int) {
	blockSize := float64(size / 3) // 每个格子的长宽
	twoBlockSize := 2 * blockSize

	incr := func() { // 增加angle的值，但不会大于3
		if angle > 2 {
			angle = 0
		} else {
			angle++
		}
	}

	c(p, blockSize, blockSize, blockSize, 0)

	b1(p, 0, 0, blockSize, angle)
	b2(p, blockSize, 0, blockSize, angle)

	incr()
	b1(p, twoBlockSize, 0, blockSize, angle)
	b2(p, twoBlockSize, blockSize, blockSize, angle)

	incr()
	b1(p, twoBlockSize, twoBlockSize, blockSize, angle)
	b2(p, blockSize, twoBlockSize, blockSize, angle)

	incr()
	b1(p, 0, twoBlockSize, blockSize, angle)
	b2(p, 0, blockSize, blockSize, angle)
}
<|endoftext|>