<commit_before>package commands

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/github/hub/git"
	"github.com/github/hub/github"
	"github.com/github/hub/utils"
)

var cmdRemote = &Command{
	Run:          remote,
	GitExtension: true,
	Usage: `
remote add [-p] [<OPTIONS>] <USER>[/<REPOSITORY>]
remote set-url [-p] [<OPTIONS>] <NAME> <USER>[/<REPOSITORY>]
`,
	Long: `Add a git remote for a GitHub repository.

## Options:
	-p
		(Deprecated) Use the 'ssh:' protocol for the remote URL.

	<USER>[/<REPOSITORY>]
		If <USER> is "origin", that value will be substituted for your GitHub
		username. <REPOSITORY> defaults to the name of the current working directory.

## Examples:
		$ hub remote add jingweno
		> git remote add jingweno git://github.com/jingweno/REPO.git

		$ hub remote add origin
		> git remote add origin git://github.com/USER/REPO.git

## See also:

hub-fork(1), hub(1), git-remote(1)
`,
}

func init() {
	CmdRunner.Use(cmdRemote)
}

/*
 */
func remote(command *Command, args *Args) {
	if !args.IsParamsEmpty() && (args.FirstParam() == "add" || args.FirstParam() == "set-url") {
		transformRemoteArgs(args)
	}
}

func transformRemoteArgs(args *Args) {
	ownerWithName := args.LastParam()
	owner, name := parseRepoNameOwner(ownerWithName)
	if owner == "" {
		return
	}

	localRepo, err := github.LocalRepo()
	utils.Check(err)

	var host string
	mainProject, err := localRepo.MainProject()
	if err == nil {
		host = mainProject.Host
	}

	if name == "" {
		if mainProject != nil {
			name = mainProject.Name
		} else {
			dirName, err := git.WorkdirName()
			utils.Check(err)
			name = github.SanitizeProjectName(dirName)
		}
	}

	var hostConfig *github.Host
	if host == "" {
		hostConfig, err = github.CurrentConfig().DefaultHost()
	} else {
		hostConfig, err = github.CurrentConfig().PromptForHost(host)
	}
	if err != nil {
		utils.Check(github.FormatError("adding remote", err))
	}
	host = hostConfig.Host

	isPrivate := parseRemotePrivateFlag(args)
	numWord := 0
	for i, p := range args.Params {
		if !looksLikeFlag(p) && (i < 1 || args.Params[i-1] != "-t") {
			numWord += 1
			if numWord == 2 && strings.Contains(p, "/") {
				args.ReplaceParam(i, owner)
			} else if numWord == 3 {
				args.RemoveParam(i)
			}
		}
	}
	if numWord == 2 && owner == "origin" {
		owner = hostConfig.User
	}

	if strings.ToLower(owner) == strings.ToLower(hostConfig.User) {
		owner = hostConfig.User
		isPrivate = true
	}

	project := github.NewProject(owner, name, host)
	url := project.GitURL("", "", isPrivate || project.Host != github.GitHubHost)
	args.AppendParams(url)
}

func parseRemotePrivateFlag(args *Args) bool {
	if i := args.IndexOfParam("-p"); i != -1 {
		args.RemoveParam(i)
		return true
	}

	return false
}

func parseRepoNameOwner(nameWithOwner string) (owner, name string) {
	nameWithOwnerRe := fmt.Sprintf("^(%s)(?:\\/(%s))?$", OwnerRe, NameRe)
	nameWithOwnerRegexp := regexp.MustCompile(nameWithOwnerRe)
	if nameWithOwnerRegexp.MatchString(nameWithOwner) {
		result := nameWithOwnerRegexp.FindStringSubmatch(nameWithOwner)
		owner, name = result[1], result[2]
	}
	return
}
<commit_msg>Add repo visibility and permission check for remote add command<commit_after>package commands

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/github/hub/git"
	"github.com/github/hub/github"
	"github.com/github/hub/utils"
)

var cmdRemote = &Command{
	Run:          remote,
	GitExtension: true,
	Usage: `
remote add [-p] [<OPTIONS>] <USER>[/<REPOSITORY>]
remote set-url [-p] [<OPTIONS>] <NAME> <USER>[/<REPOSITORY>]
`,
	Long: `Add a git remote for a GitHub repository.

## Options:
	-p
		(Deprecated) Use the 'ssh:' protocol for the remote URL.

	<USER>[/<REPOSITORY>]
		If <USER> is "origin", that value will be substituted for your GitHub
		username. <REPOSITORY> defaults to the name of the current working directory.

## Examples:
		$ hub remote add jingweno
		> git remote add jingweno git://github.com/jingweno/REPO.git

		$ hub remote add origin
		> git remote add origin git://github.com/USER/REPO.git

## See also:

hub-fork(1), hub(1), git-remote(1)
`,
}

func init() {
	CmdRunner.Use(cmdRemote)
}

/*
 */
func remote(command *Command, args *Args) {
	if !args.IsParamsEmpty() && (args.FirstParam() == "add" || args.FirstParam() == "set-url") {
		transformRemoteArgs(args)
	}
}

func transformRemoteArgs(args *Args) {
	ownerWithName := args.LastParam()
	owner, name := parseRepoNameOwner(ownerWithName)
	if owner == "" {
		return
	}

	localRepo, err := github.LocalRepo()
	utils.Check(err)

	var host string
	mainProject, err := localRepo.MainProject()
	if err == nil {
		host = mainProject.Host
	}

	if name == "" {
		if mainProject != nil {
			name = mainProject.Name
		} else {
			dirName, err := git.WorkdirName()
			utils.Check(err)
			name = github.SanitizeProjectName(dirName)
		}
	}

	var hostConfig *github.Host
	if host == "" {
		hostConfig, err = github.CurrentConfig().DefaultHost()
	} else {
		hostConfig, err = github.CurrentConfig().PromptForHost(host)
	}
	if err != nil {
		utils.Check(github.FormatError("adding remote", err))
	}
	host = hostConfig.Host

	numWord := 0
	for i, p := range args.Params {
		if !looksLikeFlag(p) && (i < 1 || args.Params[i-1] != "-t") {
			numWord += 1
			if numWord == 2 && strings.Contains(p, "/") {
				args.ReplaceParam(i, owner)
			} else if numWord == 3 {
				args.RemoveParam(i)
			}
		}
	}
	if numWord == 2 && owner == "origin" {
		owner = hostConfig.User
	}

	if strings.ToLower(owner) == strings.ToLower(hostConfig.User) {
		owner = hostConfig.User
	}

	project := github.NewProject(owner, name, host)
	gh := github.NewClient(project.Host)
	repo, err := gh.Repository(project)
	if err != nil {
		if strings.Contains(err.Error(), "HTTP 404") {
			err = fmt.Errorf("Error: repository %s/%s doesn't exist", project.Owner, project.Name)
		}
		utils.Check(err)
	}

	isPrivate := repo.Private || repo.Permissions.Push || parseRemotePrivateFlag(args)
	url := project.GitURL("", "", isPrivate || project.Host != github.GitHubHost)
	args.AppendParams(url)
}

func parseRemotePrivateFlag(args *Args) bool {
	if i := args.IndexOfParam("-p"); i != -1 {
		args.RemoveParam(i)
		return true
	}

	return false
}

func parseRepoNameOwner(nameWithOwner string) (owner, name string) {
	nameWithOwnerRe := fmt.Sprintf("^(%s)(?:\\/(%s))?$", OwnerRe, NameRe)
	nameWithOwnerRegexp := regexp.MustCompile(nameWithOwnerRe)
	if nameWithOwnerRegexp.MatchString(nameWithOwner) {
		result := nameWithOwnerRegexp.FindStringSubmatch(nameWithOwner)
		owner, name = result[1], result[2]
	}
	return
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	"github.com/ambientsound/pms/api"
	"github.com/ambientsound/pms/input/lexer"
)

// Remove removes songs from songlists.
type Remove struct {
	api api.API
}

func NewRemove(api api.API) Command {
	return &Remove{
		api: api,
	}
}

func (cmd *Remove) Execute(class int, s string) error {
	var err error

	switch class {
	case lexer.TokenEnd:
		list := cmd.api.Songlist()
		selection := list.SelectionIndices()

		if len(selection) == 0 {
			return fmt.Errorf("No song selected, cannot remove without any parameters.")
		}

		index := selection[0]
		err = list.RemoveIndices(selection)
		if err == nil {
			list.ClearSelection()
			list.SetCursor(index)
		}

		cmd.api.ListChanged()

	default:
		return fmt.Errorf("Unknown input '%s', expected END", s)
	}

	return err
}
<commit_msg>More informative message when using 'remove'<commit_after>package commands

import (
	"fmt"

	"github.com/ambientsound/pms/api"
	"github.com/ambientsound/pms/input/lexer"
)

// Remove removes songs from songlists.
type Remove struct {
	api api.API
}

func NewRemove(api api.API) Command {
	return &Remove{
		api: api,
	}
}

func (cmd *Remove) Execute(class int, s string) error {
	var err error

	switch class {
	case lexer.TokenEnd:
		list := cmd.api.Songlist()
		selection := list.Selection()
		indices := list.SelectionIndices()
		len := len(indices)

		if len == 0 {
			return fmt.Errorf("No song selected, cannot remove without any parameters.")
		}

		index := indices[0]
		err = list.RemoveIndices(indices)
		if err == nil {
			if len == 1 {
				cmd.api.Message("Removed '%s'", selection.Song(0).StringTags["file"])
			} else {
				cmd.api.Message("%d fewer songs", len)
			}
			list.ClearSelection()
			list.SetCursor(index)
		}

		cmd.api.ListChanged()

	default:
		return fmt.Errorf("Unknown input '%s', expected END", s)
	}

	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"strings"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// SysctlBuilder set up our sysctls
type SysctlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &SysctlBuilder{}

func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error {
	var sysctls []string

	// Common settings
	{
		sysctls = append(sysctls,
			"# Kubernetes Settings",
			"")

		// A higher vm.max_map_count is great for elasticsearch, mongo, or other mmap users
		// See https://github.com/kubernetes/kops/issues/1340
		sysctls = append(sysctls, "vm.max_map_count = 262144",
			"")

		// See https://github.com/kubernetes/kubernetes/pull/38001
		sysctls = append(sysctls,
			"kernel.softlockup_panic = 1",
			"kernel.softlockup_all_cpu_backtrace = 1",
			"")

		// See https://github.com/kubernetes/kube-deploy/issues/261
		sysctls = append(sysctls,
			"# Increase the number of connections",
			"net.core.somaxconn = 32768",
			"",

			"# Maximum Socket Receive Buffer",
			"net.core.rmem_max = 16777216",
			"",

			"# Default Socket Send Buffer",
			"net.core.wmem_max = 16777216",
			"",

			"# Increase the maximum total buffer-space allocatable",
			"net.ipv4.tcp_wmem = 4096 12582912 16777216",
			"net.ipv4.tcp_rmem = 4096 12582912 16777216",
			"",

			"# Increase the number of outstanding syn requests allowed",
			"net.ipv4.tcp_max_syn_backlog = 8096",
			"",

			"# For persistent HTTP connections",
			"net.ipv4.tcp_slow_start_after_idle = 0",
			"",

			"# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks",
			"net.ipv4.tcp_tw_reuse = 1",
			"",

			// We can't change the local_port_range without changing the NodePort range
			//"# Allowed local port range",
			//"net.ipv4.ip_local_port_range = 10240 65535",
			//"",

			"# Max number of packets that can be queued on interface input",
			"# If kernel is receiving packets faster than can be processed",
			"# this queue increases",
			"net.core.netdev_max_backlog = 16384",
			"",

			"# Increase size of file handles and inode cache",
			"fs.file-max = 2097152",
			"",
		)
	}

	if b.Cluster.Spec.CloudProvider == string(kops.CloudProviderAWS) {
		sysctls = append(sysctls,
			"# AWS settings",
			"",
			"# Issue #23395",
			"net.ipv4.neigh.default.gc_thresh1=0",
			"")
	}

	if b.Cluster.Spec.CloudProvider == string(kops.CloudProviderGCE) {
		sysctls = append(sysctls,
			"# GCE settings",
			"",
			"net.ipv4.ip_forward=1",
			"")
	}

	t := &nodetasks.File{
		Path:            "/etc/sysctl.d/99-k8s-general.conf",
		Contents:        fi.NewStringResource(strings.Join(sysctls, "\n")),
		Type:            nodetasks.FileType_File,
		OnChangeExecute: [][]string{{"sysctl", "--system"}},
	}
	c.AddTask(t)

	return nil
}
<commit_msg>Increase fs.inotify.max_user_instances limit. Fixes #2912<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"strings"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// SysctlBuilder set up our sysctls
type SysctlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &SysctlBuilder{}

func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error {
	var sysctls []string

	// Common settings
	{
		sysctls = append(sysctls,
			"# Kubernetes Settings",
			"")

		// A higher vm.max_map_count is great for elasticsearch, mongo, or other mmap users
		// See https://github.com/kubernetes/kops/issues/1340
		sysctls = append(sysctls, "vm.max_map_count = 262144",
			"")

		// See https://github.com/kubernetes/kubernetes/pull/38001
		sysctls = append(sysctls,
			"kernel.softlockup_panic = 1",
			"kernel.softlockup_all_cpu_backtrace = 1",
			"")

		// See https://github.com/kubernetes/kube-deploy/issues/261
		sysctls = append(sysctls,
			"# Increase the number of connections",
			"net.core.somaxconn = 32768",
			"",

			"# Maximum Socket Receive Buffer",
			"net.core.rmem_max = 16777216",
			"",

			"# Default Socket Send Buffer",
			"net.core.wmem_max = 16777216",
			"",

			"# Increase the maximum total buffer-space allocatable",
			"net.ipv4.tcp_wmem = 4096 12582912 16777216",
			"net.ipv4.tcp_rmem = 4096 12582912 16777216",
			"",

			"# Increase the number of outstanding syn requests allowed",
			"net.ipv4.tcp_max_syn_backlog = 8096",
			"",

			"# For persistent HTTP connections",
			"net.ipv4.tcp_slow_start_after_idle = 0",
			"",

			"# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks",
			"net.ipv4.tcp_tw_reuse = 1",
			"",

			// We can't change the local_port_range without changing the NodePort range
			//"# Allowed local port range",
			//"net.ipv4.ip_local_port_range = 10240 65535",
			//"",

			"# Max number of packets that can be queued on interface input",
			"# If kernel is receiving packets faster than can be processed",
			"# this queue increases",
			"net.core.netdev_max_backlog = 16384",
			"",

			"# Increase size of file handles and inode cache",
			"fs.file-max = 2097152",
			"",

			"# Max number of inotify watches for a user",
			"# Since dockerd runs as a single user, the default value of 128 per user is too low",
			"# e.g. uses of inotify: nginx ingress controller, kubectl logs -f",
			"fs.inotify.max_user_instances = 8192",
			"",
		)
	}

	if b.Cluster.Spec.CloudProvider == string(kops.CloudProviderAWS) {
		sysctls = append(sysctls,
			"# AWS settings",
			"",
			"# Issue #23395",
			"net.ipv4.neigh.default.gc_thresh1=0",
			"")
	}

	if b.Cluster.Spec.CloudProvider == string(kops.CloudProviderGCE) {
		sysctls = append(sysctls,
			"# GCE settings",
			"",
			"net.ipv4.ip_forward=1",
			"")
	}

	t := &nodetasks.File{
		Path:            "/etc/sysctl.d/99-k8s-general.conf",
		Contents:        fi.NewStringResource(strings.Join(sysctls, "\n")),
		Type:            nodetasks.FileType_File,
		OnChangeExecute: [][]string{{"sysctl", "--system"}},
	}
	c.AddTask(t)

	return nil
}
<|endoftext|>


<commit_before>package tdb

import (
	"bytes"
	"fmt"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/oxtoacart/tdb/values"
	"github.com/spaolacci/murmur3"
	"github.com/tecbot/gorocksdb"
	"gopkg.in/vmihailenco/msgpack.v2"
)

type Point struct {
	Ts   time.Time
	Dims map[string]interface{}
	Vals map[string]float64
}

type Value interface {
	Val() float64

	Add(addend float64) Value
}

type FloatValue float64

func (v FloatValue) Val() float64 {
	return float64(v)
}

func (v FloatValue) Add(addend float64) Value {
	return FloatValue(v.Val() + addend)
}

type bucket struct {
	start time.Time
	val   values.Value
	prev  *bucket
}

type partition struct {
	t       *table
	inserts chan *insert
	tail    map[string]*bucket
}

type insert struct {
	ts  time.Time
	key []byte
	val values.Value
}

type archiveRequest struct {
	key string
	b   *bucket
}

func (db *DB) Insert(table string, point *Point) error {
	t := db.getTable(table)
	if t == nil {
		return fmt.Errorf("Unknown table %v", table)
	}

	return t.insert(point)
}

func (t *table) insert(point *Point) error {
	t.clock.Advance(point.Ts)
	vals := floatsToValues(point.Vals)
	// TODO: deal with situation where name of inserted field conflicts with
	// derived field
	for _, field := range t.derivedFields {
		vals[field.Name] = field.Expr(vals)
	}

	for field, val := range vals {
		key, err := point.keyFor(field)
		if err != nil {
			return err
		}
		h := int(murmur3.Sum32(key))
		p := h % len(t.partitions)
		select {
		case t.partitions[p].inserts <- &insert{point.Ts, key, val}:
			// ok
		default:
			log.Error("Partition full, dropping point")
		}
	}

	return nil
}

func (p *partition) processInserts() {
	// TODO: base this on the passage of fake time
	archivePeriod := p.t.hotPeriod / 10
	log.Debugf("Archiving every %v", archivePeriod)
	archiveTicker := p.t.clock.NewTicker(archivePeriod)
	for {
		select {
		case insert := <-p.inserts:
			p.insert(insert)
		case <-archiveTicker.C:
			p.requestArchiving()
		}
	}
}

func (p *partition) insert(insert *insert) {
	key := string(insert.key)
	now := p.t.clock.Now()
	start := roundTime(insert.ts, p.t.resolution)
	if now.Sub(start) > p.t.hotPeriod {
		log.Trace("Discarding insert outside of hot period")
		return
	}
	b := p.tail[key]
	if b == nil || b.start.Before(start) {
		p.t.statsMutex.Lock()
		if b == nil {
			p.t.stats.HotKeys++
		}
		p.t.statsMutex.Unlock()
		b = &bucket{start, insert.val, b}
		p.tail[key] = b
		return
	}
	for {
		if b.start == start {
			// Update existing bucket
			b.val = b.val.Plus(insert.val)
			return
		}
		if b.prev == nil || b.prev.start.Before(start) {
			// Insert new bucket
			p.t.statsMutex.Lock()
			p.t.statsMutex.Unlock()
			b.prev = &bucket{start, insert.val, b.prev}
			return
		}
		// Continue looking
		b = b.prev
	}
}

func (p *partition) requestArchiving() {
	now := p.t.clock.Now()
	log.Tracef("Requested archiving at %v", now)
	for key, b := range p.tail {
		if now.Sub(b.start) > p.t.hotPeriod {
			log.Tracef("Archiving full. %v / %v %v", b.start, now, b.prev != nil)
			delete(p.tail, key)
			p.t.statsMutex.Lock()
			p.t.stats.HotKeys--
			p.t.statsMutex.Unlock()
			p.t.toArchive <- &archiveRequest{key, b}
			continue
		}
		next := b
		for {
			b = b.prev
			if b == nil {
				break
			}
			log.Tracef("Checking %v", b.start)
			if now.Sub(b.start) > p.t.hotPeriod {
				log.Trace("Archiving partial")
				p.t.toArchive <- &archiveRequest{key, b}
				next.prev = nil
				break
			}
		}
	}
}

func (t *table) archive() {
	batch := gorocksdb.NewWriteBatch()
	wo := gorocksdb.NewDefaultWriteOptions()
	for req := range t.toArchive {
		batch = t.doArchive(batch, wo, req)
	}
}

func (t *table) doArchive(batch *gorocksdb.WriteBatch, wo *gorocksdb.WriteOptions, req *archiveRequest) *gorocksdb.WriteBatch {
	key := []byte(req.key)
	seq := req.b.toSequence(t.resolution)
	numBuckets := int64(seq.numBuckets())
	if log.IsTraceEnabled() {
		log.Tracef("Archiving %d buckets starting at %v", numBuckets, seq.start().In(time.UTC))
	}
	t.statsMutex.Lock()
	t.stats.ArchivedBuckets += numBuckets
	t.statsMutex.Unlock()
	batch.Merge(key, seq)
	count := int64(batch.Count())
	if count >= t.batchSize {
		err := t.archiveByKey.Write(wo, batch)
		if err != nil {
			log.Errorf("Unable to write batch: %v", err)
		}
		batch = gorocksdb.NewWriteBatch()
	}
	return batch
}

func (t *table) retain() {
	retentionTicker := t.clock.NewTicker(t.retentionPeriod)
	wo := gorocksdb.NewDefaultWriteOptions()
	for range retentionTicker.C {
		t.doRetain(wo)
	}
}

func (t *table) doRetain(wo *gorocksdb.WriteOptions) {
	log.Debug("Removing expired keys")
	start := time.Now()
	batch := gorocksdb.NewWriteBatch()
	var keysToFree []*gorocksdb.Slice
	defer func() {
		for _, k := range keysToFree {
			k.Free()
		}
	}()

	retainUntil := t.clock.Now().Add(-1 * t.retentionPeriod)
	ro := gorocksdb.NewDefaultReadOptions()
	ro.SetFillCache(false)
	it := t.archiveByKey.NewIterator(ro)
	defer it.Close()
	for it.SeekToFirst(); it.Valid(); it.Next() {
		v := it.Value()
		vd := v.Data()
		if vd == nil || len(vd) < size64bits || sequence(vd).start().Before(retainUntil) {
			k := it.Key()
			keysToFree = append(keysToFree, k)
			batch.Delete(k.Data())
		}
		v.Free()
	}

	if batch.Count() > 0 {
		err := t.archiveByKey.Write(wo, batch)
		if err != nil {
			log.Errorf("Unable to remove expired keys: %v", err)
		} else {
			delta := time.Now().Sub(start)
			log.Debugf("Removed %v expired keys in %v", humanize(batch.Count()), delta)
		}
	} else {
		log.Debug("No expired keys to remove")
	}
}

func (p *Point) keyFor(field string) ([]byte, error) {
	buf := &bytes.Buffer{}
	enc := msgpack.NewEncoder(buf)
	enc.SortMapKeys(true)
	err := enc.Encode(field)
	if err != nil {
		return nil, fmt.Errorf("Unable to encode field: %v", err)
	}
	err = enc.Encode(p.Dims)
	if err != nil {
		return nil, fmt.Errorf("Unable to encode dims: %v", err)
	}
	return buf.Bytes(), nil
}

func floatsToValues(in map[string]float64) map[string]values.Value {
	out := make(map[string]values.Value, len(in))
	for key, value := range in {
		out[key] = values.Float(value)
	}
	return out
}
<commit_msg>Fixed compile issue<commit_after>package tdb

import (
	"bytes"
	"fmt"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/oxtoacart/tdb/values"
	"github.com/spaolacci/murmur3"
	"github.com/tecbot/gorocksdb"
	"gopkg.in/vmihailenco/msgpack.v2"
)

type Point struct {
	Ts   time.Time
	Dims map[string]interface{}
	Vals map[string]float64
}

type Value interface {
	Val() float64

	Add(addend float64) Value
}

type FloatValue float64

func (v FloatValue) Val() float64 {
	return float64(v)
}

func (v FloatValue) Add(addend float64) Value {
	return FloatValue(v.Val() + addend)
}

type bucket struct {
	start time.Time
	val   values.Value
	prev  *bucket
}

type partition struct {
	t       *table
	inserts chan *insert
	tail    map[string]*bucket
}

type insert struct {
	ts  time.Time
	key []byte
	val values.Value
}

type archiveRequest struct {
	key string
	b   *bucket
}

func (db *DB) Insert(table string, point *Point) error {
	t := db.getTable(table)
	if t == nil {
		return fmt.Errorf("Unknown table %v", table)
	}

	return t.insert(point)
}

func (t *table) insert(point *Point) error {
	t.clock.Advance(point.Ts)
	vals := floatsToValues(point.Vals)
	// TODO: deal with situation where name of inserted field conflicts with
	// derived field
	for _, field := range t.derivedFields {
		vals[field.Name] = field.Expr(vals)
	}

	for field, val := range vals {
		key, err := point.keyFor(field)
		if err != nil {
			return err
		}
		h := int(murmur3.Sum32(key))
		p := h % len(t.partitions)
		select {
		case t.partitions[p].inserts <- &insert{point.Ts, key, val}:
			// ok
		default:
			log.Error("Partition full, dropping point")
		}
	}

	return nil
}

func (p *partition) processInserts() {
	// TODO: base this on the passage of fake time
	archivePeriod := p.t.hotPeriod / 10
	log.Debugf("Archiving every %v", archivePeriod)
	archiveTicker := p.t.clock.NewTicker(archivePeriod)
	for {
		select {
		case insert := <-p.inserts:
			p.insert(insert)
		case <-archiveTicker.C:
			p.requestArchiving()
		}
	}
}

func (p *partition) insert(insert *insert) {
	key := string(insert.key)
	now := p.t.clock.Now()
	start := roundTime(insert.ts, p.t.resolution)
	if now.Sub(start) > p.t.hotPeriod {
		log.Trace("Discarding insert outside of hot period")
		return
	}
	b := p.tail[key]
	if b == nil || b.start.Before(start) {
		p.t.statsMutex.Lock()
		if b == nil {
			p.t.stats.HotKeys++
		}
		p.t.statsMutex.Unlock()
		b = &bucket{start, insert.val, b}
		p.tail[key] = b
		return
	}
	for {
		if b.start == start {
			// Update existing bucket
			b.val = b.val.Plus(insert.val)
			return
		}
		if b.prev == nil || b.prev.start.Before(start) {
			// Insert new bucket
			p.t.statsMutex.Lock()
			p.t.statsMutex.Unlock()
			b.prev = &bucket{start, insert.val, b.prev}
			return
		}
		// Continue looking
		b = b.prev
	}
}

func (p *partition) requestArchiving() {
	now := p.t.clock.Now()
	log.Tracef("Requested archiving at %v", now)
	for key, b := range p.tail {
		if now.Sub(b.start) > p.t.hotPeriod {
			log.Tracef("Archiving full. %v / %v %v", b.start, now, b.prev != nil)
			delete(p.tail, key)
			p.t.statsMutex.Lock()
			p.t.stats.HotKeys--
			p.t.statsMutex.Unlock()
			p.t.toArchive <- &archiveRequest{key, b}
			continue
		}
		next := b
		for {
			b = b.prev
			if b == nil {
				break
			}
			log.Tracef("Checking %v", b.start)
			if now.Sub(b.start) > p.t.hotPeriod {
				log.Trace("Archiving partial")
				p.t.toArchive <- &archiveRequest{key, b}
				next.prev = nil
				break
			}
		}
	}
}

func (t *table) archive() {
	batch := gorocksdb.NewWriteBatch()
	wo := gorocksdb.NewDefaultWriteOptions()
	for req := range t.toArchive {
		batch = t.doArchive(batch, wo, req)
	}
}

func (t *table) doArchive(batch *gorocksdb.WriteBatch, wo *gorocksdb.WriteOptions, req *archiveRequest) *gorocksdb.WriteBatch {
	key := []byte(req.key)
	seq := req.b.toSequence(t.resolution)
	numBuckets := int64(seq.numBuckets())
	if log.IsTraceEnabled() {
		log.Tracef("Archiving %d buckets starting at %v", numBuckets, seq.start().In(time.UTC))
	}
	t.statsMutex.Lock()
	t.stats.ArchivedBuckets += numBuckets
	t.statsMutex.Unlock()
	batch.Merge(key, seq)
	count := int64(batch.Count())
	if count >= t.batchSize {
		err := t.archiveByKey.Write(wo, batch)
		if err != nil {
			log.Errorf("Unable to write batch: %v", err)
		}
		batch = gorocksdb.NewWriteBatch()
	}
	return batch
}

func (t *table) retain() {
	retentionTicker := t.clock.NewTicker(t.retentionPeriod)
	wo := gorocksdb.NewDefaultWriteOptions()
	for range retentionTicker.C {
		t.doRetain(wo)
	}
}

func (t *table) doRetain(wo *gorocksdb.WriteOptions) {
	log.Debug("Removing expired keys")
	start := time.Now()
	batch := gorocksdb.NewWriteBatch()
	var keysToFree []*gorocksdb.Slice
	defer func() {
		for _, k := range keysToFree {
			k.Free()
		}
	}()

	retainUntil := t.clock.Now().Add(-1 * t.retentionPeriod)
	ro := gorocksdb.NewDefaultReadOptions()
	ro.SetFillCache(false)
	it := t.archiveByKey.NewIterator(ro)
	defer it.Close()
	for it.SeekToFirst(); it.Valid(); it.Next() {
		v := it.Value()
		vd := v.Data()
		if vd == nil || len(vd) < size64bits || sequence(vd).start().Before(retainUntil) {
			k := it.Key()
			keysToFree = append(keysToFree, k)
			batch.Delete(k.Data())
		}
		v.Free()
	}

	if batch.Count() > 0 {
		err := t.archiveByKey.Write(wo, batch)
		if err != nil {
			log.Errorf("Unable to remove expired keys: %v", err)
		} else {
			delta := time.Now().Sub(start)
			log.Debugf("Removed %v expired keys in %v", humanize.Comma(int64(batch.Count())), delta)
		}
	} else {
		log.Debug("No expired keys to remove")
	}
}

func (p *Point) keyFor(field string) ([]byte, error) {
	buf := &bytes.Buffer{}
	enc := msgpack.NewEncoder(buf)
	enc.SortMapKeys(true)
	err := enc.Encode(field)
	if err != nil {
		return nil, fmt.Errorf("Unable to encode field: %v", err)
	}
	err = enc.Encode(p.Dims)
	if err != nil {
		return nil, fmt.Errorf("Unable to encode dims: %v", err)
	}
	return buf.Bytes(), nil
}

func floatsToValues(in map[string]float64) map[string]values.Value {
	out := make(map[string]values.Value, len(in))
	for key, value := range in {
		out[key] = values.Float(value)
	}
	return out
}
<|endoftext|>


<commit_before>package match

import (
	"fmt"

	"github.com/tisp-lang/tisp/src/lib/ast"
)

type desugarer struct {
	lets []interface{}
}

func newDesugarer() *desugarer {
	return &desugarer{nil}
}

func (d *desugarer) desugar(x interface{}) interface{} {
	switch x := x.(type) {
	case ast.App:
		return ast.NewApp(
			d.desugar(x.Function()),
			d.desugar(x.Arguments()).(ast.Arguments),
			x.DebugInfo())
	case ast.Arguments:
		ps := make([]ast.PositionalArgument, 0, len(x.Positionals()))

		for _, p := range x.Positionals() {
			ps = append(ps, d.desugar(p).(ast.PositionalArgument))
		}

		ks := make([]ast.KeywordArgument, 0, len(x.Keywords()))

		for _, k := range x.Keywords() {
			ks = append(ks, d.desugar(k).(ast.KeywordArgument))
		}

		dicts := make([]interface{}, 0, len(x.ExpandedDicts()))

		for _, dict := range x.ExpandedDicts() {
			dicts = append(dicts, d.desugar(dict))
		}

		return ast.NewArguments(ps, ks, dicts)
	case ast.KeywordArgument:
		return ast.NewKeywordArgument(x.Name(), d.desugar(x.Value()))
	case ast.LetFunction:
		ls := make([]interface{}, 0, len(x.Lets()))

		for _, l := range x.Lets() {
			l := d.desugar(l)
			ls = append(ls, append(d.takeLets(), l)...)
		}

		b := d.desugar(x.Body())
		ls = append(ls, d.takeLets()...)

		return ast.NewLetFunction(
			x.Name(),
			x.Signature(),
			ls,
			b,
			x.DebugInfo())
	case ast.LetVar:
		return ast.NewLetVar(x.Name(), d.desugar(x.Expr()))
	case ast.Match:
		cs := make([]ast.MatchCase, 0, len(x.Cases()))

		for _, c := range x.Cases() {
			cs = append(cs, ast.NewMatchCase(c.Pattern(), d.desugar(c.Value())))
		}

		return d.desugarMatchExpression(ast.NewMatch(d.desugar(x.Value()), cs))
	case ast.Output:
		return ast.NewOutput(d.desugar(x.Expr()), x.Expanded())
	case ast.PositionalArgument:
		return ast.NewPositionalArgument(d.desugar(x.Value()), x.Expanded())
	default:
		return x
	}
}

func (d *desugarer) takeLets() []interface{} {
	ls := d.lets
	d.lets = nil
	return ls
}

func (d *desugarer) letVar(s string, v interface{}) {
	d.lets = append(d.lets, ast.NewLetVar(s, v))
}

func (d *desugarer) desugarMatchExpression(m ast.Match) interface{} {
	css := map[patternType][]ast.MatchCase{}

	for i, c := range m.Cases() {
		t := getPatternType(c.Pattern())

		if t == namePattern && i < len(m.Cases())-1 {
			panic(fmt.Errorf("A wildcard pattern is found, but some cases are left"))
		}

		css[t] = append(css[t], c)
	}

	ks := []ast.SwitchCase{}

	if cs, ok := css[listPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("list", d.desugarListCases(cs)))
	}

	if cs, ok := css[dictPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("dict", d.desugarDictCases(cs)))
	}

	dc := interface{}(nil)

	if cs, ok := css[namePattern]; ok {
		dc = d.desugarNameCases(cs)
	}

	return ast.NewSwitch(app("typeOf", m.Value()), ks, dc)
}

func getPatternType(p interface{}) patternType {
	switch x := p.(type) {
	case string:
		return namePattern
	case ast.App:
		switch x.Function().(string) {
		case "$list":
			return listPattern
		case "$dict":
			return dictPattern
		}
	}

	panic(fmt.Errorf("Invalid pattern: %#v", p))
}

func (d *desugarer) desugarListCases(cs []ast.MatchCase) interface{} {
	panic("Not implemented")
}

func (d *desugarer) desugarDictCases(cs []ast.MatchCase) interface{} {
	panic("Not implemented")
}

func (d *desugarer) desugarNameCases(cs []ast.MatchCase) interface{} {
	panic("Not implemented")
}

// func (d *desugarer) casesToBody(arg string, cs []ast.MatchCase) interface{} {
//	cs = renameBoundNamesInCases(cs)
//	body := app("error", "\"MatchError\"", "\"Failed to match a value with patterns.\"")

//	for _, cs := range groupCases(cs) {
//		result, ok := d.matchCasesOfSamePatterns(arg, cs)
//		body = app("if", ok, result, body)
//	}

//	return body
// }

// func renameBoundNamesInCases(cs []ast.MatchCase) []ast.MatchCase {
//	new := make([]ast.MatchCase, 0, len(cs))

//	for _, c := range cs {
//		new = append(new, renameBoundNamesInCase(c))
//	}

//	return new
// }

// func renameBoundNamesInCase(c ast.MatchCase) ast.MatchCase {
//	p, ns := newPatternRenamer().rename(c.Pattern())
//	return ast.NewMatchCase(p, newValueRenamer(ns).rename(c.Value()))
// }

// func (d *desugarer) matchCasesOfSamePatterns(v interface{}, cs []ast.MatchCase) (interface{}, interface{}) {
//	switch getPatternType(cs[0].Pattern()) {
//	case listPattern:
//		panic("Not implemented")
//	case dictPattern:
//		panic("Not implemented")
//	case scalarPattern:
//		ss := make([]interface{}, 0, 2*len(cs))

//		for _, c := range cs {
//			ss = append(ss, c.Pattern(), c.Value())
//		}

//		dict := gensym.GenSym("match", "scalar", "dict")
//		d.letVar(dict, app("dict", ss...))

//		return app(dict, v), app("include", dict, v)
//	case namePattern:
//		if len(cs) != 1 {
//			panic(fmt.Errorf("Duplicate name patterns: %v", len(cs)))
//		}

//		c := cs[0]
//		d.letVar(c.Pattern().(string), v)
//		return c.Value(), "true"
//	}

//	panic(fmt.Errorf("Invalid cases: %#v", cs))
// }

// func matchType(v string, typ string) interface{} {
//	return app("=", app("typeOf", v), typ)
// }
<commit_msg>Rename bound names in match cases<commit_after>package match

import (
	"fmt"

	"github.com/tisp-lang/tisp/src/lib/ast"
)

type desugarer struct {
	lets []interface{}
}

func newDesugarer() *desugarer {
	return &desugarer{nil}
}

func (d *desugarer) desugar(x interface{}) interface{} {
	switch x := x.(type) {
	case ast.App:
		return ast.NewApp(
			d.desugar(x.Function()),
			d.desugar(x.Arguments()).(ast.Arguments),
			x.DebugInfo())
	case ast.Arguments:
		ps := make([]ast.PositionalArgument, 0, len(x.Positionals()))

		for _, p := range x.Positionals() {
			ps = append(ps, d.desugar(p).(ast.PositionalArgument))
		}

		ks := make([]ast.KeywordArgument, 0, len(x.Keywords()))

		for _, k := range x.Keywords() {
			ks = append(ks, d.desugar(k).(ast.KeywordArgument))
		}

		dicts := make([]interface{}, 0, len(x.ExpandedDicts()))

		for _, dict := range x.ExpandedDicts() {
			dicts = append(dicts, d.desugar(dict))
		}

		return ast.NewArguments(ps, ks, dicts)
	case ast.KeywordArgument:
		return ast.NewKeywordArgument(x.Name(), d.desugar(x.Value()))
	case ast.LetFunction:
		ls := make([]interface{}, 0, len(x.Lets()))

		for _, l := range x.Lets() {
			l := d.desugar(l)
			ls = append(ls, append(d.takeLets(), l)...)
		}

		b := d.desugar(x.Body())
		ls = append(ls, d.takeLets()...)

		return ast.NewLetFunction(
			x.Name(),
			x.Signature(),
			ls,
			b,
			x.DebugInfo())
	case ast.LetVar:
		return ast.NewLetVar(x.Name(), d.desugar(x.Expr()))
	case ast.Match:
		cs := make([]ast.MatchCase, 0, len(x.Cases()))

		for _, c := range x.Cases() {
			cs = append(cs, renameBoundNamesInCase(ast.NewMatchCase(c.Pattern(), d.desugar(c.Value()))))
		}

		return d.desugarMatchExpression(ast.NewMatch(d.desugar(x.Value()), cs))
	case ast.Output:
		return ast.NewOutput(d.desugar(x.Expr()), x.Expanded())
	case ast.PositionalArgument:
		return ast.NewPositionalArgument(d.desugar(x.Value()), x.Expanded())
	default:
		return x
	}
}

func (d *desugarer) takeLets() []interface{} {
	ls := d.lets
	d.lets = nil
	return ls
}

func (d *desugarer) letVar(s string, v interface{}) {
	d.lets = append(d.lets, ast.NewLetVar(s, v))
}

func (d *desugarer) desugarMatchExpression(m ast.Match) interface{} {
	css := map[patternType][]ast.MatchCase{}

	for i, c := range m.Cases() {
		t := getPatternType(c.Pattern())

		if t == namePattern && i < len(m.Cases())-1 {
			panic(fmt.Errorf("A wildcard pattern is found, but some cases are left"))
		}

		css[t] = append(css[t], c)
	}

	ks := []ast.SwitchCase{}

	if cs, ok := css[listPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("list", d.desugarListCases(cs)))
	}

	if cs, ok := css[dictPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("dict", d.desugarDictCases(cs)))
	}

	dc := interface{}(nil)

	if cs, ok := css[namePattern]; ok {
		dc = d.desugarNameCases(cs)
	}

	return ast.NewSwitch(app("typeOf", m.Value()), ks, dc)
}

func getPatternType(p interface{}) patternType {
	switch x := p.(type) {
	case string:
		return namePattern
	case ast.App:
		switch x.Function().(string) {
		case "$list":
			return listPattern
		case "$dict":
			return dictPattern
		}
	}

	panic(fmt.Errorf("Invalid pattern: %#v", p))
}

func (d *desugarer) desugarListCases(cs []ast.MatchCase) interface{} {
	panic("Not implemented")
}

func (d *desugarer) desugarDictCases(cs []ast.MatchCase) interface{} {
	panic("Not implemented")
}

func (d *desugarer) desugarNameCases(cs []ast.MatchCase) interface{} {
	panic("Not implemented")
}

func renameBoundNamesInCase(c ast.MatchCase) ast.MatchCase {
	p, ns := newPatternRenamer().rename(c.Pattern())
	return ast.NewMatchCase(p, newValueRenamer(ns).rename(c.Value()))
}

// func (d *desugarer) casesToBody(arg string, cs []ast.MatchCase) interface{} {
//	cs = renameBoundNamesInCases(cs)
//	body := app("error", "\"MatchError\"", "\"Failed to match a value with patterns.\"")

//	for _, cs := range groupCases(cs) {
//		result, ok := d.matchCasesOfSamePatterns(arg, cs)
//		body = app("if", ok, result, body)
//	}

//	return body
// }

// func (d *desugarer) matchCasesOfSamePatterns(v interface{}, cs []ast.MatchCase) (interface{}, interface{}) {
//	switch getPatternType(cs[0].Pattern()) {
//	case listPattern:
//		panic("Not implemented")
//	case dictPattern:
//		panic("Not implemented")
//	case scalarPattern:
//		ss := make([]interface{}, 0, 2*len(cs))

//		for _, c := range cs {
//			ss = append(ss, c.Pattern(), c.Value())
//		}

//		dict := gensym.GenSym("match", "scalar", "dict")
//		d.letVar(dict, app("dict", ss...))

//		return app(dict, v), app("include", dict, v)
//	case namePattern:
//		if len(cs) != 1 {
//			panic(fmt.Errorf("Duplicate name patterns: %v", len(cs)))
//		}

//		c := cs[0]
//		d.letVar(c.Pattern().(string), v)
//		return c.Value(), "true"
//	}

//	panic(fmt.Errorf("Invalid cases: %#v", cs))
// }

// func matchType(v string, typ string) interface{} {
//	return app("=", app("typeOf", v), typ)
// }
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package cgo contains runtime support for code generated
by the cgo tool.  See the documentation for the cgo command
for details on using cgo.
*/
package cgo

/*

#cgo darwin LDFLAGS: -lpthread
#cgo dragonfly LDFLAGS: -lpthread
#cgo freebsd LDFLAGS: -lpthread
#cgo android LDFLAGS: -llog
#cgo !android,linux LDFLAGS: -lpthread
#cgo netbsd LDFLAGS: -lpthread
#cgo openbsd LDFLAGS: -lpthread
#cgo windows LDFLAGS: -lm -mthreads

#cgo CFLAGS: -Wall -Werror

*/
import "C"
<commit_msg>runtime/cgo: on darwin/arm, -lpthread is not needed.<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package cgo contains runtime support for code generated
by the cgo tool.  See the documentation for the cgo command
for details on using cgo.
*/
package cgo

/*

#cgo darwin,!arm LDFLAGS: -lpthread
#cgo dragonfly LDFLAGS: -lpthread
#cgo freebsd LDFLAGS: -lpthread
#cgo android LDFLAGS: -llog
#cgo !android,linux LDFLAGS: -lpthread
#cgo netbsd LDFLAGS: -lpthread
#cgo openbsd LDFLAGS: -lpthread
#cgo windows LDFLAGS: -lm -mthreads

#cgo CFLAGS: -Wall -Werror

*/
import "C"
<|endoftext|>


<commit_before>// +build windows

package winterm

import (
	"bytes"
	"io/ioutil"
	"os"
	"strconv"

	. "github.com/Azure/go-ansiterm"
	"github.com/Sirupsen/logrus"
)

var logger *logrus.Logger

type WindowsAnsiEventHandler struct {
	fd         uintptr
	file       *os.File
	infoReset  *CONSOLE_SCREEN_BUFFER_INFO
	sr         scrollRegion
	buffer     bytes.Buffer
	attributes WORD
	inverted   bool
}

func CreateWinEventHandler(fd uintptr, file *os.File) AnsiEventHandler {
	logFile := ioutil.Discard

	if isDebugEnv := os.Getenv(LogEnv); isDebugEnv == "1" {
		logFile, _ = os.Create("winEventHandler.log")
	}

	logger = &logrus.Logger{
		Out:       logFile,
		Formatter: new(logrus.TextFormatter),
		Level:     logrus.DebugLevel,
	}

	infoReset, err := GetConsoleScreenBufferInfo(fd)
	if err != nil {
		return nil
	}

	sr := scrollRegion{int(infoReset.Window.Top), int(infoReset.Window.Bottom)}

	return &WindowsAnsiEventHandler{
		fd:         fd,
		file:       file,
		infoReset:  infoReset,
		sr:         sr,
		attributes: infoReset.Attributes,
	}
}

type scrollRegion struct {
	top    int
	bottom int
}

func (h *WindowsAnsiEventHandler) Print(b byte) error {
	return h.buffer.WriteByte(b)
}

func (h *WindowsAnsiEventHandler) Execute(b byte) error {
	if ANSI_LINE_FEED == b {
		info, err := GetConsoleScreenBufferInfo(h.fd)
		if err != nil {
			return err
		}

		if int(info.CursorPosition.Y) == h.sr.bottom {
			if err := h.Flush(); err != nil {
				return err
			}

			logger.Infof("Scrolling due to LF at bottom of scroll region")

			// Scroll up one row if we attempt to line feed at the bottom
			// of the scroll region
			if err := h.scrollUp(1); err != nil {
				return err
			}

			// Clear line
			// if err := h.CUD(1); err != nil {
			// 	return err
			// }
			if err := h.EL(0); err != nil {
				return err
			}
		}
	}

	if ANSI_BEL <= b && b <= ANSI_CARRIAGE_RETURN {
		return h.buffer.WriteByte(b)
	}

	return nil
}

func (h *WindowsAnsiEventHandler) CUU(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUU: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorVertical(-param)
}

func (h *WindowsAnsiEventHandler) CUD(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUD: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorVertical(param)
}

func (h *WindowsAnsiEventHandler) CUF(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUF: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorHorizontal(param)
}

func (h *WindowsAnsiEventHandler) CUB(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUB: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorHorizontal(-param)
}

func (h *WindowsAnsiEventHandler) CNL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CNL: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorLine(param)
}

func (h *WindowsAnsiEventHandler) CPL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CPL: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorLine(-param)
}

func (h *WindowsAnsiEventHandler) CHA(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CHA: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorColumn(param)
}

func (h *WindowsAnsiEventHandler) VPA(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("VPA: [[%d]]", param)
	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}
	pos := info.CursorPosition
	pos.Y = ensureInRange(SHORT(param-1), info.Window.Top, info.Window.Bottom)
	return SetConsoleCursorPosition(h.fd, pos)
}

func (h *WindowsAnsiEventHandler) CUP(row int, col int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	rowStr, colStr := strconv.Itoa(row), strconv.Itoa(col)
	logger.Infof("CUP: [%v]", []string{rowStr, colStr})
	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	rect := info.Window
	rowS := AddInRange(SHORT(row-1), rect.Top, rect.Top, rect.Bottom)
	colS := AddInRange(SHORT(col-1), rect.Left, rect.Left, rect.Right)
	position := COORD{colS, rowS}

	return h.setCursorPosition(position, info.Size)
}

func (h *WindowsAnsiEventHandler) HVP(row int, col int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	rowS, colS := strconv.Itoa(row), strconv.Itoa(row)
	logger.Infof("HVP: [%v]", []string{rowS, colS})
	return h.CUP(row, col)
}

func (h *WindowsAnsiEventHandler) DECTCEM(visible bool) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("DECTCEM: [%v]", []string{strconv.FormatBool(visible)})

	return nil
}

func (h *WindowsAnsiEventHandler) ED(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("ED: [%v]", []string{strconv.Itoa(param)})

	// [J  -- Erases from the cursor to the end of the screen, including the cursor position.
	// [1J -- Erases from the beginning of the screen to the cursor, including the cursor position.
	// [2J -- Erases the complete display. The cursor does not move.
	// [3J -- Erases the complete display and backing buffer, cursor moves to (0,0)
	// Notes:
	// -- ANSI.SYS always moved the cursor to (0,0) for both [2J and [3J
	// -- Clearing the entire buffer, versus just the Window, works best for Windows Consoles

	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	var start COORD
	var end COORD

	switch param {
	case 0:
		start = info.CursorPosition
		end = COORD{info.Size.X - 1, info.Size.Y - 1}

	case 1:
		start = COORD{0, 0}
		end = info.CursorPosition

	case 2:
		start = COORD{0, 0}
		end = COORD{info.Size.X - 1, info.Size.Y - 1}

	case 3:
		start = COORD{0, 0}
		end = COORD{info.Size.X - 1, info.Size.Y - 1}
	}

	err = h.clearRange(h.attributes, start, end)
	if err != nil {
		return err
	}

	if param == 2 || param == 3 {
		err = h.setCursorPosition(COORD{0, 0}, info.Size)
		if err != nil {
			return err
		}
	}

	return nil
}

func (h *WindowsAnsiEventHandler) EL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("EL: [%v]", strconv.Itoa(param))

	// [K  -- Erases from the cursor to the end of the line, including the cursor position.
	// [1K -- Erases from the beginning of the line to the cursor, including the cursor position.
	// [2K -- Erases the complete line.

	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	var start COORD
	var end COORD

	switch param {
	case 0:
		start = info.CursorPosition
		end = COORD{info.Size.X, info.CursorPosition.Y}

	case 1:
		start = COORD{0, info.CursorPosition.Y}
		end = info.CursorPosition

	case 2:
		start = COORD{0, info.CursorPosition.Y}
		end = COORD{info.Size.X, info.CursorPosition.Y}
	}

	err = h.clearRange(h.attributes, start, end)
	if err != nil {
		return err
	}

	return nil
}

func (h *WindowsAnsiEventHandler) IL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("IL: [%v]", strconv.Itoa(param))
	if err := h.scrollDown(param); err != nil {
		return err
	}

	return h.EL(2)
}

func (h *WindowsAnsiEventHandler) DL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("DL: [%v]", strconv.Itoa(param))
	return h.scrollUp(param)
}

func (h *WindowsAnsiEventHandler) SGR(params []int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	strings := []string{}
	for _, v := range params {
		logger.Infof("SGR: [%v]", strings)
		strings = append(strings, strconv.Itoa(v))
	}

	logger.Infof("SGR: [%v]", strings)

	if len(params) <= 0 {
		h.attributes = h.infoReset.Attributes
		h.inverted = false
	} else {
		for _, attr := range params {

			if attr == ANSI_SGR_RESET {
				h.attributes = h.infoReset.Attributes
				continue
			}

			h.attributes, h.inverted = collectAnsiIntoWindowsAttributes(h.attributes, h.inverted, h.infoReset.Attributes, SHORT(attr))
		}
	}

	attributes := h.attributes
	if h.inverted {
		attributes = invertAttributes(attributes)
	}
	err := SetConsoleTextAttribute(h.fd, attributes)
	if err != nil {
		return err
	}

	return nil
}

func (h *WindowsAnsiEventHandler) SU(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("SU: [%v]", []string{strconv.Itoa(param)})
	return h.scrollPageUp()
}

func (h *WindowsAnsiEventHandler) SD(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("SD: [%v]", []string{strconv.Itoa(param)})
	return h.scrollPageDown()
}

func (h *WindowsAnsiEventHandler) DA(params []string) error {
	logger.Infof("DA: [%v]", params)

	// See the site below for details of the device attributes command
	// http://vt100.net/docs/vt220-rm/chapter4.html

	// First character of first parameter string is '>'
	if params[0][0] == '>' {
		// Secondary device attribute request:
		// Respond with:
		// "I am a VT220 version 1.0, no options.
		//                    CSI     >     1     ;     1     0     ;     0     c    CR    LF
		h.buffer.Write([]byte{CSI_ENTRY, 0x3E, 0x31, 0x3B, 0x31, 0x30, 0x3B, 0x30, 0x63, 0x0D, 0x0A})

	} else {
		// Primary device attribute request:
		// Respond with:
		// "I am a service class 2 terminal (62) with 132 columns (1),
		// printer port (2), selective erase (6), DRCS (7), UDK (8),
		// and I support 7-bit national replacement character sets (9)."
		//                    CSI     ?     6     2     ;     1     ;     2     ;     6     ;     7     ;     8     ;     9     c    CR    LF
		h.buffer.Write([]byte{CSI_ENTRY, 0x3F, 0x36, 0x32, 0x3B, 0x31, 0x3B, 0x32, 0x3B, 0x36, 0x3B, 0x37, 0x3B, 0x38, 0x3B, 0x39, 0x63, 0x0D, 0x0A})
	}

	return nil
}

func (h *WindowsAnsiEventHandler) DECSTBM(top int, bottom int) error {
	logger.Infof("DECSTBM: [%d, %d]", top, bottom)

	// Windows is 0 indexed, Linux is 1 indexed
	h.sr.top = top - 1
	h.sr.bottom = bottom - 1

	return nil
}

func (h *WindowsAnsiEventHandler) RI() error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Info("RI: []")

	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	if info.Window.Top == info.CursorPosition.Y {
		if err := h.scrollPageDown(); err != nil {
			return err
		}

		return h.EL(2)
	} else {
		return h.CUU(1)
	}
}

func (h *WindowsAnsiEventHandler) Flush() error {
	if h.buffer.Len() > 0 {
		logger.Infof("Flush: [%s]", h.buffer.Bytes())
		if _, err := h.buffer.WriteTo(h.file); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Clear inverse mode on SGR reset<commit_after>// +build windows

package winterm

import (
	"bytes"
	"io/ioutil"
	"os"
	"strconv"

	. "github.com/Azure/go-ansiterm"
	"github.com/Sirupsen/logrus"
)

var logger *logrus.Logger

type WindowsAnsiEventHandler struct {
	fd         uintptr
	file       *os.File
	infoReset  *CONSOLE_SCREEN_BUFFER_INFO
	sr         scrollRegion
	buffer     bytes.Buffer
	attributes WORD
	inverted   bool
}

func CreateWinEventHandler(fd uintptr, file *os.File) AnsiEventHandler {
	logFile := ioutil.Discard

	if isDebugEnv := os.Getenv(LogEnv); isDebugEnv == "1" {
		logFile, _ = os.Create("winEventHandler.log")
	}

	logger = &logrus.Logger{
		Out:       logFile,
		Formatter: new(logrus.TextFormatter),
		Level:     logrus.DebugLevel,
	}

	infoReset, err := GetConsoleScreenBufferInfo(fd)
	if err != nil {
		return nil
	}

	sr := scrollRegion{int(infoReset.Window.Top), int(infoReset.Window.Bottom)}

	return &WindowsAnsiEventHandler{
		fd:         fd,
		file:       file,
		infoReset:  infoReset,
		sr:         sr,
		attributes: infoReset.Attributes,
	}
}

type scrollRegion struct {
	top    int
	bottom int
}

func (h *WindowsAnsiEventHandler) Print(b byte) error {
	return h.buffer.WriteByte(b)
}

func (h *WindowsAnsiEventHandler) Execute(b byte) error {
	if ANSI_LINE_FEED == b {
		info, err := GetConsoleScreenBufferInfo(h.fd)
		if err != nil {
			return err
		}

		if int(info.CursorPosition.Y) == h.sr.bottom {
			if err := h.Flush(); err != nil {
				return err
			}

			logger.Infof("Scrolling due to LF at bottom of scroll region")

			// Scroll up one row if we attempt to line feed at the bottom
			// of the scroll region
			if err := h.scrollUp(1); err != nil {
				return err
			}

			// Clear line
			// if err := h.CUD(1); err != nil {
			// 	return err
			// }
			if err := h.EL(0); err != nil {
				return err
			}
		}
	}

	if ANSI_BEL <= b && b <= ANSI_CARRIAGE_RETURN {
		return h.buffer.WriteByte(b)
	}

	return nil
}

func (h *WindowsAnsiEventHandler) CUU(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUU: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorVertical(-param)
}

func (h *WindowsAnsiEventHandler) CUD(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUD: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorVertical(param)
}

func (h *WindowsAnsiEventHandler) CUF(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUF: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorHorizontal(param)
}

func (h *WindowsAnsiEventHandler) CUB(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CUB: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorHorizontal(-param)
}

func (h *WindowsAnsiEventHandler) CNL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CNL: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorLine(param)
}

func (h *WindowsAnsiEventHandler) CPL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CPL: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorLine(-param)
}

func (h *WindowsAnsiEventHandler) CHA(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("CHA: [%v]", []string{strconv.Itoa(param)})
	return h.moveCursorColumn(param)
}

func (h *WindowsAnsiEventHandler) VPA(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("VPA: [[%d]]", param)
	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}
	pos := info.CursorPosition
	pos.Y = ensureInRange(SHORT(param-1), info.Window.Top, info.Window.Bottom)
	return SetConsoleCursorPosition(h.fd, pos)
}

func (h *WindowsAnsiEventHandler) CUP(row int, col int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	rowStr, colStr := strconv.Itoa(row), strconv.Itoa(col)
	logger.Infof("CUP: [%v]", []string{rowStr, colStr})
	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	rect := info.Window
	rowS := AddInRange(SHORT(row-1), rect.Top, rect.Top, rect.Bottom)
	colS := AddInRange(SHORT(col-1), rect.Left, rect.Left, rect.Right)
	position := COORD{colS, rowS}

	return h.setCursorPosition(position, info.Size)
}

func (h *WindowsAnsiEventHandler) HVP(row int, col int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	rowS, colS := strconv.Itoa(row), strconv.Itoa(row)
	logger.Infof("HVP: [%v]", []string{rowS, colS})
	return h.CUP(row, col)
}

func (h *WindowsAnsiEventHandler) DECTCEM(visible bool) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("DECTCEM: [%v]", []string{strconv.FormatBool(visible)})

	return nil
}

func (h *WindowsAnsiEventHandler) ED(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("ED: [%v]", []string{strconv.Itoa(param)})

	// [J  -- Erases from the cursor to the end of the screen, including the cursor position.
	// [1J -- Erases from the beginning of the screen to the cursor, including the cursor position.
	// [2J -- Erases the complete display. The cursor does not move.
	// [3J -- Erases the complete display and backing buffer, cursor moves to (0,0)
	// Notes:
	// -- ANSI.SYS always moved the cursor to (0,0) for both [2J and [3J
	// -- Clearing the entire buffer, versus just the Window, works best for Windows Consoles

	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	var start COORD
	var end COORD

	switch param {
	case 0:
		start = info.CursorPosition
		end = COORD{info.Size.X - 1, info.Size.Y - 1}

	case 1:
		start = COORD{0, 0}
		end = info.CursorPosition

	case 2:
		start = COORD{0, 0}
		end = COORD{info.Size.X - 1, info.Size.Y - 1}

	case 3:
		start = COORD{0, 0}
		end = COORD{info.Size.X - 1, info.Size.Y - 1}
	}

	err = h.clearRange(h.attributes, start, end)
	if err != nil {
		return err
	}

	if param == 2 || param == 3 {
		err = h.setCursorPosition(COORD{0, 0}, info.Size)
		if err != nil {
			return err
		}
	}

	return nil
}

func (h *WindowsAnsiEventHandler) EL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("EL: [%v]", strconv.Itoa(param))

	// [K  -- Erases from the cursor to the end of the line, including the cursor position.
	// [1K -- Erases from the beginning of the line to the cursor, including the cursor position.
	// [2K -- Erases the complete line.

	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	var start COORD
	var end COORD

	switch param {
	case 0:
		start = info.CursorPosition
		end = COORD{info.Size.X, info.CursorPosition.Y}

	case 1:
		start = COORD{0, info.CursorPosition.Y}
		end = info.CursorPosition

	case 2:
		start = COORD{0, info.CursorPosition.Y}
		end = COORD{info.Size.X, info.CursorPosition.Y}
	}

	err = h.clearRange(h.attributes, start, end)
	if err != nil {
		return err
	}

	return nil
}

func (h *WindowsAnsiEventHandler) IL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("IL: [%v]", strconv.Itoa(param))
	if err := h.scrollDown(param); err != nil {
		return err
	}

	return h.EL(2)
}

func (h *WindowsAnsiEventHandler) DL(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("DL: [%v]", strconv.Itoa(param))
	return h.scrollUp(param)
}

func (h *WindowsAnsiEventHandler) SGR(params []int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	strings := []string{}
	for _, v := range params {
		logger.Infof("SGR: [%v]", strings)
		strings = append(strings, strconv.Itoa(v))
	}

	logger.Infof("SGR: [%v]", strings)

	if len(params) <= 0 {
		h.attributes = h.infoReset.Attributes
		h.inverted = false
	} else {
		for _, attr := range params {

			if attr == ANSI_SGR_RESET {
				h.attributes = h.infoReset.Attributes
				h.inverted = false
				continue
			}

			h.attributes, h.inverted = collectAnsiIntoWindowsAttributes(h.attributes, h.inverted, h.infoReset.Attributes, SHORT(attr))
		}
	}

	attributes := h.attributes
	if h.inverted {
		attributes = invertAttributes(attributes)
	}
	err := SetConsoleTextAttribute(h.fd, attributes)
	if err != nil {
		return err
	}

	return nil
}

func (h *WindowsAnsiEventHandler) SU(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("SU: [%v]", []string{strconv.Itoa(param)})
	return h.scrollPageUp()
}

func (h *WindowsAnsiEventHandler) SD(param int) error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Infof("SD: [%v]", []string{strconv.Itoa(param)})
	return h.scrollPageDown()
}

func (h *WindowsAnsiEventHandler) DA(params []string) error {
	logger.Infof("DA: [%v]", params)

	// See the site below for details of the device attributes command
	// http://vt100.net/docs/vt220-rm/chapter4.html

	// First character of first parameter string is '>'
	if params[0][0] == '>' {
		// Secondary device attribute request:
		// Respond with:
		// "I am a VT220 version 1.0, no options.
		//                    CSI     >     1     ;     1     0     ;     0     c    CR    LF
		h.buffer.Write([]byte{CSI_ENTRY, 0x3E, 0x31, 0x3B, 0x31, 0x30, 0x3B, 0x30, 0x63, 0x0D, 0x0A})

	} else {
		// Primary device attribute request:
		// Respond with:
		// "I am a service class 2 terminal (62) with 132 columns (1),
		// printer port (2), selective erase (6), DRCS (7), UDK (8),
		// and I support 7-bit national replacement character sets (9)."
		//                    CSI     ?     6     2     ;     1     ;     2     ;     6     ;     7     ;     8     ;     9     c    CR    LF
		h.buffer.Write([]byte{CSI_ENTRY, 0x3F, 0x36, 0x32, 0x3B, 0x31, 0x3B, 0x32, 0x3B, 0x36, 0x3B, 0x37, 0x3B, 0x38, 0x3B, 0x39, 0x63, 0x0D, 0x0A})
	}

	return nil
}

func (h *WindowsAnsiEventHandler) DECSTBM(top int, bottom int) error {
	logger.Infof("DECSTBM: [%d, %d]", top, bottom)

	// Windows is 0 indexed, Linux is 1 indexed
	h.sr.top = top - 1
	h.sr.bottom = bottom - 1

	return nil
}

func (h *WindowsAnsiEventHandler) RI() error {
	if err := h.Flush(); err != nil {
		return err
	}
	logger.Info("RI: []")

	info, err := GetConsoleScreenBufferInfo(h.fd)
	if err != nil {
		return err
	}

	if info.Window.Top == info.CursorPosition.Y {
		if err := h.scrollPageDown(); err != nil {
			return err
		}

		return h.EL(2)
	} else {
		return h.CUU(1)
	}
}

func (h *WindowsAnsiEventHandler) Flush() error {
	if h.buffer.Len() > 0 {
		logger.Infof("Flush: [%s]", h.buffer.Bytes())
		if _, err := h.buffer.WriteTo(h.file); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package repo_manager

/*
   This file contains canned pre-upload steps for RepoManagers to use.
*/

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"

	"go.skia.org/infra/go/cipd"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/go_install"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/go/metrics2"
)

var cipdRoot = path.Join(os.TempDir(), "cipd")

// PreUploadStep is a function to be run after the roll is performed but before
// a CL is uploaded. The string slice parameter is the set of environment
// variables which should be used by the pre-upload step. The http.Client should
// be authenticated. The string parameter is the absolute path to the directory
// of the parent repo.
type PreUploadStep func(context.Context, []string, *http.Client, string) error

// Return the PreUploadStep with the given name.
func GetPreUploadStep(s string) (PreUploadStep, error) {
	rv, ok := map[string]PreUploadStep{
		"ANGLECodeGeneration":             ANGLECodeGeneration,
		"GoGenerateCipd":                  GoGenerateCipd,
		"TrainInfra":                      TrainInfra,
		"FlutterLicenseScripts":           FlutterLicenseScripts,
		"FlutterLicenseScriptsForDart":    FlutterLicenseScriptsForDart,
		"FlutterLicenseScriptsForFuchsia": FlutterLicenseScriptsForFuchsia,
		"UpdateFlutterDepsForDart":        UpdateFlutterDepsForDart,
	}[s]
	if !ok {
		return nil, fmt.Errorf("No such pre-upload step: %s", s)
	}
	return rv, nil
}

// Return the PreUploadSteps with the given names.
func GetPreUploadSteps(steps []string) ([]PreUploadStep, error) {
	rv := make([]PreUploadStep, 0, len(steps))
	for _, s := range steps {
		step, err := GetPreUploadStep(s)
		if err != nil {
			return nil, err
		}
		rv = append(rv, step)
	}
	return rv, nil
}

// Train the infra expectations.
func TrainInfra(ctx context.Context, env []string, client *http.Client, parentRepoDir string) error {
	// TODO(borenet): Should we plumb through --local and --workdir?
	sklog.Info("Installing Go...")
	_, goEnv, err := go_install.EnsureGo(ctx, client, cipdRoot)
	if err != nil {
		return err
	}
	envMap := make(map[string]string, len(goEnv)+len(env))
	for _, v := range env {
		split := strings.SplitN(v, "=", 2)
		if len(split) != 2 {
			return fmt.Errorf("Invalid environment variable: %q", v)
		}
		envMap[split[0]] = split[1]
	}
	for k, v := range goEnv {
		if k == "PATH" {
			oldPath := envMap["PATH"]
			if oldPath != "" {
				v += ":" + oldPath
			}
		}
		envMap[k] = v
	}
	envSlice := make([]string, 0, len(envMap))
	for k, v := range envMap {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v))
	}
	sklog.Info("Training infra expectations...")
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Name: "make",
		Args: []string{"train"},
		Dir:  path.Join(parentRepoDir, "infra", "bots"),
		Env:  envSlice,
	}); err != nil {
		return err
	}
	return nil
}

// Run the create_updated_flutter_deps.py for Dart in Flutter.
// See https://bugs.chromium.org/p/skia/issues/detail?id=8437#c18 and
// https://bugs.chromium.org/p/skia/issues/detail?id=8437#c21 for context.
func UpdateFlutterDepsForDart(ctx context.Context, env []string, _ *http.Client, parentRepoDir string) error {
	sklog.Info("Running create_updated_flutter_deps.py and then \"gclient sync\"")

	scriptDir := filepath.Join(parentRepoDir, "..", "tools", "dart")
	scriptName := "create_updated_flutter_deps.py"
	if _, err := exec.RunCwd(ctx, scriptDir, "python", scriptName); err != nil {
		return fmt.Errorf("Error when running python %s: %s", scriptName, err)
	}

	// Do "gclient sync" after the script runs.
	if _, err := exec.RunCwd(ctx, parentRepoDir, filepath.Join(parentRepoDir, "..", "..", "depot_tools", "gclient"), "sync"); err != nil {
		return fmt.Errorf("Error when running \"gclient sync\" in %s: %s", parentRepoDir, err)
	}

	return nil
}

// Run the flutter license scripts as described in
// https://bugs.chromium.org/p/skia/issues/detail?id=7730#c6 and in
// https://github.com/flutter/engine/blob/master/tools/licenses/README.md
func FlutterLicenseScripts(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
	return flutterLicenseScripts(ctx, parentRepoDir, "licenses_skia")
}

// Run the flutter license scripts for fuchsia.
func FlutterLicenseScriptsForFuchsia(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
	return flutterLicenseScripts(ctx, parentRepoDir, "licenses_fuchsia")
}

// Run the flutter license scripts for dart.
func FlutterLicenseScriptsForDart(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
	return flutterLicenseScripts(ctx, parentRepoDir, "licenses_third_party")
}

func flutterLicenseScripts(ctx context.Context, parentRepoDir, licenseFileName string) error {
	licenseScriptFailure := int64(1)
	defer func() {
		metrics2.GetInt64Metric("flutter_license_script_failure", nil).Update(licenseScriptFailure)
	}()

	sklog.Info("Running flutter license scripts.")
	binariesPath := filepath.Join(parentRepoDir, "..", "third_party", "dart", "tools", "sdks", "dart-sdk", "bin")

	// Step1: Run pub get.
	licenseToolsDir := filepath.Join(parentRepoDir, "tools", "licenses")
	pubBinary := filepath.Join(binariesPath, "pub")
	sklog.Info("Running put get.")
	if _, err := exec.RunCwd(ctx, licenseToolsDir, pubBinary, "get"); err != nil {
		return fmt.Errorf("Error when running pub get: %s", err)
	}

	// Step2: Clean out out/licenses. Things fail without this step.
	licensesOutDir := filepath.Join(parentRepoDir, "..", "out", "licenses")
	if err := os.RemoveAll(licensesOutDir); err != nil {
		return fmt.Errorf("Error when cleaning out/licenses: %s", err)
	}

	// Step3: Run dart license script to create new licenses.
	dartBinary := filepath.Join(binariesPath, "dart")
	licensesGoldenDir := filepath.Join(parentRepoDir, "ci", "licenses_golden")
	licenseCmd := []string{dartBinary, "lib/main.dart", "--src", "../../..", "--out", licensesOutDir, "--golden", licensesGoldenDir}
	sklog.Infof("Running %s", licenseCmd)
	if err := exec.Run(ctx, &exec.Command{
		Dir:            licenseToolsDir,
		Name:           licenseCmd[0],
		Args:           licenseCmd[1:],
		CombinedOutput: os.Stdout,
	}); err != nil {
		return fmt.Errorf("Error when running dart license script: %s", err)
	}

	// Step4: Check to see if the target license file was created in the out dir.
	//        It will be created if the hash changes.
	if _, err := os.Stat(filepath.Join(licensesOutDir, licenseFileName)); err == nil {
		sklog.Infof("Found %s", licenseFileName)

		// Step5: Copy from out dir to goldens dir. This is required for updating
		//        the release file in sky_engine/LICENSE.
		if _, err := exec.RunCwd(ctx, licenseToolsDir, "cp", filepath.Join(licensesOutDir, licenseFileName), filepath.Join(licensesGoldenDir, licenseFileName)); err != nil {
			return fmt.Errorf("Error when copying %s from out to golden dir: %s", licenseFileName, err)
		}
		// Step6: Capture diff of licenses_golden/${licenseFileName}.
		licensesDiffOutput, err := git.GitDir(licenseToolsDir).Git(ctx, "diff", "--no-ext-diff", filepath.Join(licensesGoldenDir, licenseFileName))
		if err != nil {
			return fmt.Errorf("Error when seeing diff of golden %s: %s", licenseFileName, err)
		}
		sklog.Infof("The licenses diff output is:\n%s", licensesDiffOutput)

		// Step7: Update sky_engine/LICENSE. It should always be run
		// according to https://github.com/flutter/engine/pull/4959#issuecomment-380222322
		updateLicenseCmd := []string{dartBinary, "lib/main.dart", "--release", "--src", "../../..", "--out", licensesOutDir}
		sklog.Infof("Running %s", updateLicenseCmd)
		releasesLicensePath := filepath.Join(parentRepoDir, "sky", "packages", "sky_engine", "LICENSE")
		outFile, err := os.Create(releasesLicensePath)
		if err != nil {
			return fmt.Errorf("Could not open %s: %s", releasesLicensePath, err)
		}
		if err := exec.Run(ctx, &exec.Command{
			Dir:    licenseToolsDir,
			Name:   updateLicenseCmd[0],
			Args:   updateLicenseCmd[1:],
			Stdout: outFile,
			Stderr: os.Stderr,
		}); err != nil {
			return fmt.Errorf("Error when running dart license script: %s", err)
		}
	}

	sklog.Info("Done running flutter license scripts.")
	licenseScriptFailure = 0
	return nil
}

// Run "go generate" in go/cipd.
func GoGenerateCipd(ctx context.Context, _ []string, client *http.Client, parentRepoDir string) error {
	// TODO(borenet): Should we plumb through --local and --workdir?
	sklog.Info("Installing Go...")
	goExc, goEnv, err := go_install.EnsureGo(ctx, client, cipdRoot)
	if err != nil {
		return err
	}

	// Also install the protoc asset. Use a different CIPD root dir to
	// prevent conflicts with the Go packages.
	protocRoot := path.Join(os.TempDir(), "cipd_protoc")
	if err := cipd.Ensure(ctx, client, protocRoot, cipd.PkgProtoc); err != nil {
		return err
	}

	envSlice := make([]string, 0, len(goEnv))
	for k, v := range goEnv {
		if k == "PATH" {
			// Construct PATH by adding protoc and the required PATH
			// entries for Go on to the existing PATH.
			v = path.Join(protocRoot, cipd.PkgProtoc.Path, "bin") + ":" + v + ":" + os.Getenv("PATH")
		}
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v))
	}

	// Run go generate.
	generateDir := filepath.Join(parentRepoDir, "go", "cipd")
	sklog.Infof("Running 'go generate' in %s", generateDir)
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Name: goExc,
		Args: []string{"generate"},
		Dir:  generateDir,
		Env:  envSlice,
	}); err != nil {
		return err
	}
	return nil
}

// Run the ANGLE code generation script.
func ANGLECodeGeneration(ctx context.Context, env []string, client *http.Client, parentRepoDir string) error {
	sklog.Info("Running code generation script...")
	out, err := exec.RunCommand(ctx, &exec.Command{
		Name: "python",
		Args: []string{filepath.Join("scripts", "run_code_generation.py")},
		Dir:  parentRepoDir,
		Env:  env,
	})
	sklog.Infof("Output from run_code_generation:\n%s", out)
	return skerr.Wrap(err)
}
<commit_msg>[Autoroller] Delete unversioned trees during gclient sync in dart pre_upload_step<commit_after>package repo_manager

/*
   This file contains canned pre-upload steps for RepoManagers to use.
*/

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"

	"go.skia.org/infra/go/cipd"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/go_install"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/go/metrics2"
)

var cipdRoot = path.Join(os.TempDir(), "cipd")

// PreUploadStep is a function to be run after the roll is performed but before
// a CL is uploaded. The string slice parameter is the set of environment
// variables which should be used by the pre-upload step. The http.Client should
// be authenticated. The string parameter is the absolute path to the directory
// of the parent repo.
type PreUploadStep func(context.Context, []string, *http.Client, string) error

// Return the PreUploadStep with the given name.
func GetPreUploadStep(s string) (PreUploadStep, error) {
	rv, ok := map[string]PreUploadStep{
		"ANGLECodeGeneration":             ANGLECodeGeneration,
		"GoGenerateCipd":                  GoGenerateCipd,
		"TrainInfra":                      TrainInfra,
		"FlutterLicenseScripts":           FlutterLicenseScripts,
		"FlutterLicenseScriptsForDart":    FlutterLicenseScriptsForDart,
		"FlutterLicenseScriptsForFuchsia": FlutterLicenseScriptsForFuchsia,
		"UpdateFlutterDepsForDart":        UpdateFlutterDepsForDart,
	}[s]
	if !ok {
		return nil, fmt.Errorf("No such pre-upload step: %s", s)
	}
	return rv, nil
}

// Return the PreUploadSteps with the given names.
func GetPreUploadSteps(steps []string) ([]PreUploadStep, error) {
	rv := make([]PreUploadStep, 0, len(steps))
	for _, s := range steps {
		step, err := GetPreUploadStep(s)
		if err != nil {
			return nil, err
		}
		rv = append(rv, step)
	}
	return rv, nil
}

// Train the infra expectations.
func TrainInfra(ctx context.Context, env []string, client *http.Client, parentRepoDir string) error {
	// TODO(borenet): Should we plumb through --local and --workdir?
	sklog.Info("Installing Go...")
	_, goEnv, err := go_install.EnsureGo(ctx, client, cipdRoot)
	if err != nil {
		return err
	}
	envMap := make(map[string]string, len(goEnv)+len(env))
	for _, v := range env {
		split := strings.SplitN(v, "=", 2)
		if len(split) != 2 {
			return fmt.Errorf("Invalid environment variable: %q", v)
		}
		envMap[split[0]] = split[1]
	}
	for k, v := range goEnv {
		if k == "PATH" {
			oldPath := envMap["PATH"]
			if oldPath != "" {
				v += ":" + oldPath
			}
		}
		envMap[k] = v
	}
	envSlice := make([]string, 0, len(envMap))
	for k, v := range envMap {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v))
	}
	sklog.Info("Training infra expectations...")
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Name: "make",
		Args: []string{"train"},
		Dir:  path.Join(parentRepoDir, "infra", "bots"),
		Env:  envSlice,
	}); err != nil {
		return err
	}
	return nil
}

// Run the create_updated_flutter_deps.py for Dart in Flutter.
// See https://bugs.chromium.org/p/skia/issues/detail?id=8437#c18 and
// https://bugs.chromium.org/p/skia/issues/detail?id=8437#c21 for context.
func UpdateFlutterDepsForDart(ctx context.Context, env []string, _ *http.Client, parentRepoDir string) error {
	sklog.Info("Running create_updated_flutter_deps.py and then \"gclient sync\"")

	scriptDir := filepath.Join(parentRepoDir, "..", "tools", "dart")
	scriptName := "create_updated_flutter_deps.py"
	if _, err := exec.RunCwd(ctx, scriptDir, "python", scriptName); err != nil {
		return fmt.Errorf("Error when running python %s: %s", scriptName, err)
	}

	// Do "gclient sync" after the script runs.
	if _, err := exec.RunCwd(ctx, parentRepoDir, filepath.Join(parentRepoDir, "..", "..", "depot_tools", "gclient"), "sync", "--delete_unversioned_trees", "--force"); err != nil {
		return fmt.Errorf("Error when running \"gclient sync\" in %s: %s", parentRepoDir, err)
	}

	return nil
}

// Run the flutter license scripts as described in
// https://bugs.chromium.org/p/skia/issues/detail?id=7730#c6 and in
// https://github.com/flutter/engine/blob/master/tools/licenses/README.md
func FlutterLicenseScripts(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
	return flutterLicenseScripts(ctx, parentRepoDir, "licenses_skia")
}

// Run the flutter license scripts for fuchsia.
func FlutterLicenseScriptsForFuchsia(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
	return flutterLicenseScripts(ctx, parentRepoDir, "licenses_fuchsia")
}

// Run the flutter license scripts for dart.
func FlutterLicenseScriptsForDart(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
	return flutterLicenseScripts(ctx, parentRepoDir, "licenses_third_party")
}

func flutterLicenseScripts(ctx context.Context, parentRepoDir, licenseFileName string) error {
	licenseScriptFailure := int64(1)
	defer func() {
		metrics2.GetInt64Metric("flutter_license_script_failure", nil).Update(licenseScriptFailure)
	}()

	sklog.Info("Running flutter license scripts.")
	binariesPath := filepath.Join(parentRepoDir, "..", "third_party", "dart", "tools", "sdks", "dart-sdk", "bin")

	// Step1: Run pub get.
	licenseToolsDir := filepath.Join(parentRepoDir, "tools", "licenses")
	pubBinary := filepath.Join(binariesPath, "pub")
	sklog.Info("Running put get.")
	if _, err := exec.RunCwd(ctx, licenseToolsDir, pubBinary, "get"); err != nil {
		return fmt.Errorf("Error when running pub get: %s", err)
	}

	// Step2: Clean out out/licenses. Things fail without this step.
	licensesOutDir := filepath.Join(parentRepoDir, "..", "out", "licenses")
	if err := os.RemoveAll(licensesOutDir); err != nil {
		return fmt.Errorf("Error when cleaning out/licenses: %s", err)
	}

	// Step3: Run dart license script to create new licenses.
	dartBinary := filepath.Join(binariesPath, "dart")
	licensesGoldenDir := filepath.Join(parentRepoDir, "ci", "licenses_golden")
	licenseCmd := []string{dartBinary, "lib/main.dart", "--src", "../../..", "--out", licensesOutDir, "--golden", licensesGoldenDir}
	sklog.Infof("Running %s", licenseCmd)
	if err := exec.Run(ctx, &exec.Command{
		Dir:            licenseToolsDir,
		Name:           licenseCmd[0],
		Args:           licenseCmd[1:],
		CombinedOutput: os.Stdout,
	}); err != nil {
		return fmt.Errorf("Error when running dart license script: %s", err)
	}

	// Step4: Check to see if the target license file was created in the out dir.
	//        It will be created if the hash changes.
	if _, err := os.Stat(filepath.Join(licensesOutDir, licenseFileName)); err == nil {
		sklog.Infof("Found %s", licenseFileName)

		// Step5: Copy from out dir to goldens dir. This is required for updating
		//        the release file in sky_engine/LICENSE.
		if _, err := exec.RunCwd(ctx, licenseToolsDir, "cp", filepath.Join(licensesOutDir, licenseFileName), filepath.Join(licensesGoldenDir, licenseFileName)); err != nil {
			return fmt.Errorf("Error when copying %s from out to golden dir: %s", licenseFileName, err)
		}
		// Step6: Capture diff of licenses_golden/${licenseFileName}.
		licensesDiffOutput, err := git.GitDir(licenseToolsDir).Git(ctx, "diff", "--no-ext-diff", filepath.Join(licensesGoldenDir, licenseFileName))
		if err != nil {
			return fmt.Errorf("Error when seeing diff of golden %s: %s", licenseFileName, err)
		}
		sklog.Infof("The licenses diff output is:\n%s", licensesDiffOutput)

		// Step7: Update sky_engine/LICENSE. It should always be run
		// according to https://github.com/flutter/engine/pull/4959#issuecomment-380222322
		updateLicenseCmd := []string{dartBinary, "lib/main.dart", "--release", "--src", "../../..", "--out", licensesOutDir}
		sklog.Infof("Running %s", updateLicenseCmd)
		releasesLicensePath := filepath.Join(parentRepoDir, "sky", "packages", "sky_engine", "LICENSE")
		outFile, err := os.Create(releasesLicensePath)
		if err != nil {
			return fmt.Errorf("Could not open %s: %s", releasesLicensePath, err)
		}
		if err := exec.Run(ctx, &exec.Command{
			Dir:    licenseToolsDir,
			Name:   updateLicenseCmd[0],
			Args:   updateLicenseCmd[1:],
			Stdout: outFile,
			Stderr: os.Stderr,
		}); err != nil {
			return fmt.Errorf("Error when running dart license script: %s", err)
		}
	}

	sklog.Info("Done running flutter license scripts.")
	licenseScriptFailure = 0
	return nil
}

// Run "go generate" in go/cipd.
func GoGenerateCipd(ctx context.Context, _ []string, client *http.Client, parentRepoDir string) error {
	// TODO(borenet): Should we plumb through --local and --workdir?
	sklog.Info("Installing Go...")
	goExc, goEnv, err := go_install.EnsureGo(ctx, client, cipdRoot)
	if err != nil {
		return err
	}

	// Also install the protoc asset. Use a different CIPD root dir to
	// prevent conflicts with the Go packages.
	protocRoot := path.Join(os.TempDir(), "cipd_protoc")
	if err := cipd.Ensure(ctx, client, protocRoot, cipd.PkgProtoc); err != nil {
		return err
	}

	envSlice := make([]string, 0, len(goEnv))
	for k, v := range goEnv {
		if k == "PATH" {
			// Construct PATH by adding protoc and the required PATH
			// entries for Go on to the existing PATH.
			v = path.Join(protocRoot, cipd.PkgProtoc.Path, "bin") + ":" + v + ":" + os.Getenv("PATH")
		}
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v))
	}

	// Run go generate.
	generateDir := filepath.Join(parentRepoDir, "go", "cipd")
	sklog.Infof("Running 'go generate' in %s", generateDir)
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Name: goExc,
		Args: []string{"generate"},
		Dir:  generateDir,
		Env:  envSlice,
	}); err != nil {
		return err
	}
	return nil
}

// Run the ANGLE code generation script.
func ANGLECodeGeneration(ctx context.Context, env []string, client *http.Client, parentRepoDir string) error {
	sklog.Info("Running code generation script...")
	out, err := exec.RunCommand(ctx, &exec.Command{
		Name: "python",
		Args: []string{filepath.Join("scripts", "run_code_generation.py")},
		Dir:  parentRepoDir,
		Env:  env,
	})
	sklog.Infof("Output from run_code_generation:\n%s", out)
	return skerr.Wrap(err)
}
<|endoftext|>


<commit_before>/*
 * Simple recommendation engine
 *     Copyright (c) 2014, Christian Muehlhaeuser <muesli@gmail.com>
 *
 *   For license see LICENSE.txt
 */

package regommend

import (
	"errors"
	"log"
	"fmt"
	_ "sort"
	"math"
	"sort"
	"sync"
	_ "time"
)

// Structure of a table with items in the engine.
type RegommendTable struct {
	sync.RWMutex

	// The table's name.
	name string
	// All items in the table.
	items map[interface{}]*RegommendItem

	// The logger used for this table.
	logger *log.Logger

	// Callback method triggered when trying to load a non-existing key.
	loadData func(key interface{}) *RegommendItem
	// Callback method triggered when adding a new item to the engine.
	addedItem func(item *RegommendItem)
	// Callback method triggered before deleting an item from the engine.
	aboutToDeleteItem func(item *RegommendItem)
}

// Returns how many items are currently stored in the engine.
func (table *RegommendTable) Count() int {
	table.RLock()
	defer table.RUnlock()
	return len(table.items)
}

// Configures a data-loader callback, which will be called when trying
// to use access a non-existing key.
func (table *RegommendTable) SetDataLoader(f func(interface{}) *RegommendItem) {
	table.Lock()
	defer table.Unlock()
	table.loadData = f
}

// Configures a callback, which will be called every time a new item
// is added to the engine.
func (table *RegommendTable) SetAddedItemCallback(f func(*RegommendItem)) {
	table.Lock()
	defer table.Unlock()
	table.addedItem = f
}

// Configures a callback, which will be called every time an item
// is about to be removed from the engine.
func (table *RegommendTable) SetAboutToDeleteItemCallback(f func(*RegommendItem)) {
	table.Lock()
	defer table.Unlock()
	table.aboutToDeleteItem = f
}

// Sets the logger to be used by this engine table.
func (table *RegommendTable) SetLogger(logger *log.Logger) {
	table.Lock()
	defer table.Unlock()
	table.logger = logger
}

// Adds a key/value pair to the engine.
// Parameter key is the item's engine-key.
// Parameter data is the item's value.
func (table *RegommendTable) Add(key interface{}, data map[interface{}]float64) *RegommendItem {
	item := CreateRegommendItem(key, data)

	// Add item to engine.
	table.Lock()
	table.items[key] = &item

	// engine values so we don't keep blocking the mutex.
	addedItem := table.addedItem
	table.Unlock()

	// Trigger callback after adding an item to engine.
	if addedItem != nil {
		addedItem(&item)
	}

	return &item
}

// Delete an item from the engine.
func (table *RegommendTable) Delete(key interface{}) (*RegommendItem, error) {
	table.RLock()
	r, ok := table.items[key]
	if !ok {
		table.RUnlock()
		return nil, errors.New("Key not found in engine")
	}

	// engine value so we don't keep blocking the mutex.
	aboutToDeleteItem := table.aboutToDeleteItem
	table.RUnlock()

	// Trigger callbacks before deleting an item from engine.
	if aboutToDeleteItem != nil {
		aboutToDeleteItem(r)
	}

	r.RLock()
	defer r.RUnlock()

	table.Lock()
	defer table.Unlock()
	delete(table.items, key)

	return r, nil
}

// Test whether an item exists in the engine. Unlike the Value method
// Exists neither tries to fetch data via the loadData callback nor
// does it keep the item alive in the engine.
func (table *RegommendTable) Exists(key interface{}) bool {
	table.RLock()
	defer table.RUnlock()
	_, ok := table.items[key]

	return ok
}

// Get an item from the engine and mark it to be kept alive.
func (table *RegommendTable) Value(key interface{}) (*RegommendItem, error) {
	table.RLock()
	r, ok := table.items[key]
	loadData := table.loadData
	table.RUnlock()

	if ok {
		return r, nil
	}

	// Item doesn't exist in engine. Try and fetch it with a data-loader.
	if loadData != nil {
		item := loadData(key)
		if item != nil {
			table.Add(key, item.data)
			return item, nil
		}

		return nil, errors.New("Key not found and could not be loaded into engine")
	}

	return nil, errors.New("Key not found in engine")
}

// Delete all items from engine.
func (table *RegommendTable) Flush() {
	table.Lock()
	defer table.Unlock()

	table.log("Flushing table", table.name)

	table.items = make(map[interface{}]*RegommendItem)
}

// Internal logging method for convenience.
func (table *RegommendTable) log(v ...interface{}) {
	if table.logger == nil {
		return
	}

	table.logger.Println(v)
}
<commit_msg>* Cosine similarity comparison implementation.<commit_after>/*
 * Simple recommendation engine
 *     Copyright (c) 2014, Christian Muehlhaeuser <muesli@gmail.com>
 *
 *   For license see LICENSE.txt
 */

package regommend

import (
	"errors"
	"log"
	"fmt"
	_ "sort"
	"math"
	"sort"
	"sync"
	_ "time"
)

// Structure of a table with items in the engine.
type RegommendTable struct {
	sync.RWMutex

	// The table's name.
	name string
	// All items in the table.
	items map[interface{}]*RegommendItem

	// The logger used for this table.
	logger *log.Logger

	// Callback method triggered when trying to load a non-existing key.
	loadData func(key interface{}) *RegommendItem
	// Callback method triggered when adding a new item to the engine.
	addedItem func(item *RegommendItem)
	// Callback method triggered before deleting an item from the engine.
	aboutToDeleteItem func(item *RegommendItem)
}

// Returns how many items are currently stored in the engine.
func (table *RegommendTable) Count() int {
	table.RLock()
	defer table.RUnlock()
	return len(table.items)
}

// Configures a data-loader callback, which will be called when trying
// to use access a non-existing key.
func (table *RegommendTable) SetDataLoader(f func(interface{}) *RegommendItem) {
	table.Lock()
	defer table.Unlock()
	table.loadData = f
}

// Configures a callback, which will be called every time a new item
// is added to the engine.
func (table *RegommendTable) SetAddedItemCallback(f func(*RegommendItem)) {
	table.Lock()
	defer table.Unlock()
	table.addedItem = f
}

// Configures a callback, which will be called every time an item
// is about to be removed from the engine.
func (table *RegommendTable) SetAboutToDeleteItemCallback(f func(*RegommendItem)) {
	table.Lock()
	defer table.Unlock()
	table.aboutToDeleteItem = f
}

// Sets the logger to be used by this engine table.
func (table *RegommendTable) SetLogger(logger *log.Logger) {
	table.Lock()
	defer table.Unlock()
	table.logger = logger
}

// Adds a key/value pair to the engine.
// Parameter key is the item's engine-key.
// Parameter data is the item's value.
func (table *RegommendTable) Add(key interface{}, data map[interface{}]float64) *RegommendItem {
	item := CreateRegommendItem(key, data)

	// Add item to engine.
	table.Lock()
	table.items[key] = &item

	// engine values so we don't keep blocking the mutex.
	addedItem := table.addedItem
	table.Unlock()

	// Trigger callback after adding an item to engine.
	if addedItem != nil {
		addedItem(&item)
	}

	return &item
}

// Delete an item from the engine.
func (table *RegommendTable) Delete(key interface{}) (*RegommendItem, error) {
	table.RLock()
	r, ok := table.items[key]
	if !ok {
		table.RUnlock()
		return nil, errors.New("Key not found in engine")
	}

	// engine value so we don't keep blocking the mutex.
	aboutToDeleteItem := table.aboutToDeleteItem
	table.RUnlock()

	// Trigger callbacks before deleting an item from engine.
	if aboutToDeleteItem != nil {
		aboutToDeleteItem(r)
	}

	r.RLock()
	defer r.RUnlock()

	table.Lock()
	defer table.Unlock()
	delete(table.items, key)

	return r, nil
}

// Test whether an item exists in the engine. Unlike the Value method
// Exists neither tries to fetch data via the loadData callback nor
// does it keep the item alive in the engine.
func (table *RegommendTable) Exists(key interface{}) bool {
	table.RLock()
	defer table.RUnlock()
	_, ok := table.items[key]

	return ok
}

// Get an item from the engine and mark it to be kept alive.
func (table *RegommendTable) Value(key interface{}) (*RegommendItem, error) {
	table.RLock()
	r, ok := table.items[key]
	loadData := table.loadData
	table.RUnlock()

	if ok {
		return r, nil
	}

	// Item doesn't exist in engine. Try and fetch it with a data-loader.
	if loadData != nil {
		item := loadData(key)
		if item != nil {
			table.Add(key, item.data)
			return item, nil
		}

		return nil, errors.New("Key not found and could not be loaded into engine")
	}

	return nil, errors.New("Key not found in engine")
}

// Delete all items from engine.
func (table *RegommendTable) Flush() {
	table.Lock()
	defer table.Unlock()

	table.log("Flushing table", table.name)

	table.items = make(map[interface{}]*RegommendItem)
}

func cosineSim(t1, t2 map[interface{}]float64) float64 {
	sum_xy := 0.0
	sum_x2 := 0.0
	sum_y2 := 0.0

	for key, x := range t1 {
		y, ok := t2[key]
		if ok {
			fmt.Println("Found shared:", key, x, y)

			sum_xy += x * y
			sum_x2 += math.Pow(x, 2)
			sum_y2 += math.Pow(y, 2)
		}
	}

	denominator := math.Sqrt(sum_x2) * math.Sqrt(sum_y2)
	if denominator == 0 {
		return 0
	}

	return sum_xy / denominator
}

// Internal logging method for convenience.
func (table *RegommendTable) log(v ...interface{}) {
	if table.logger == nil {
		return
	}

	table.logger.Println(v)
}
<|endoftext|>