<commit_before>package printer

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/hashicorp/hcl/hcl/parser"
)

var update = flag.Bool("update", false, "update golden files")

const (
	dataDir = "testdata"
)

type entry struct {
	source, golden string
}

// Use go test -update to create/update the respective golden files.
var data = []entry{
	{"complexhcl.input", "complexhcl.golden"},
	{"list.input", "list.golden"},
	{"list_comment.input", "list_comment.golden"},
	{"comment.input", "comment.golden"},
	{"comment_crlf.input", "comment.golden"},
	{"comment_aligned.input", "comment_aligned.golden"},
	{"comment_array.input", "comment_array.golden"},
	{"comment_end_file.input", "comment_end_file.golden"},
	{"comment_multiline_indent.input", "comment_multiline_indent.golden"},
	{"comment_multiline_no_stanza.input", "comment_multiline_no_stanza.golden"},
	{"comment_multiline_stanza.input", "comment_multiline_stanza.golden"},
	{"comment_newline.input", "comment_newline.golden"},
	{"comment_object_multi.input", "comment_object_multi.golden"},
	{"comment_standalone.input", "comment_standalone.golden"},
	{"empty_block.input", "empty_block.golden"},
	{"list_of_objects.input", "list_of_objects.golden"},
	{"multiline_string.input", "multiline_string.golden"},
	{"object_singleline.input", "object_singleline.golden"},
	{"object_with_heredoc.input", "object_with_heredoc.golden"},
}

func TestFiles(t *testing.T) {
	for _, e := range data {
		source := filepath.Join(dataDir, e.source)
		golden := filepath.Join(dataDir, e.golden)
		t.Run(e.source, func(t *testing.T) {
			check(t, source, golden)
		})
	}
}

func check(t *testing.T, source, golden string) {
	src, err := ioutil.ReadFile(source)
	if err != nil {
		t.Error(err)
		return
	}

	res, err := format(src)
	if err != nil {
		t.Error(err)
		return
	}

	// update golden files if necessary
	if *update {
		if err := ioutil.WriteFile(golden, res, 0644); err != nil {
			t.Error(err)
		}
		return
	}

	// get golden
	gld, err := ioutil.ReadFile(golden)
	if err != nil {
		t.Error(err)
		return
	}

	// formatted source and golden must be the same
	if err := diff(source, golden, res, gld); err != nil {
		t.Error(err)
		return
	}
}

// diff compares a and b.
func diff(aname, bname string, a, b []byte) error {
	var buf bytes.Buffer // holding long error message

	// compare lengths
	if len(a) != len(b) {
		fmt.Fprintf(&buf, "\nlength changed: len(%s) = %d, len(%s) = %d", aname, len(a), bname, len(b))
	}

	// compare contents
	line := 1
	offs := 1
	for i := 0; i < len(a) && i < len(b); i++ {
		ch := a[i]
		if ch != b[i] {
			fmt.Fprintf(&buf, "\n%s:%d:%d: %q", aname, line, i-offs+1, lineAt(a, offs))
			fmt.Fprintf(&buf, "\n%s:%d:%d: %q", bname, line, i-offs+1, lineAt(b, offs))
			fmt.Fprintf(&buf, "\n\n")
			break
		}
		if ch == '\n' {
			line++
			offs = i + 1
		}
	}

	if buf.Len() > 0 {
		return errors.New(buf.String())
	}
	return nil
}

// format parses src, prints the corresponding AST, verifies the resulting
// src is syntactically correct, and returns the resulting src or an error
// if any.
func format(src []byte) ([]byte, error) {
	formatted, err := Format(src)
	if err != nil {
		return nil, err
	}

	// make sure formatted output is syntactically correct
	if _, err := parser.Parse(formatted); err != nil {
		return nil, fmt.Errorf("parse: %s\n%s", err, formatted)
	}

	return formatted, nil
}

// lineAt returns the line in text starting at offset offs.
func lineAt(text []byte, offs int) []byte {
	i := offs
	for i < len(text) && text[i] != '\n' {
		i++
	}
	return text[offs:i]
}

// TestFormatParsable ensures that the output of Format() is can be parsed again.
func TestFormatValidOutput(t *testing.T) {
	cases := []string{
		"#\x00",
		"#\ue123t",
		"Y=<<4\n4/\n\n\n/4/@=4/\n\n\n/4000000004\r\r\n00004\n",
		"x=<<_\n_\r\r\n_\n",
	}

	for _, c := range cases {
		f, err := Format([]byte(c))
		if err != nil {
			// ignore these failures, not all inputs are valid HCL.
			t.Logf("Format(%q) = %v", c, err)
			continue
		}

		if _, err := parser.Parse(f); err != nil {
			t.Errorf("Format(%q) = %q; Parse(%q) = %v", c, f, f, err)
			continue
		}
	}
}
<commit_msg>printer: Add another failing input to TestFormatParsable.<commit_after>package printer

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/hashicorp/hcl/hcl/parser"
)

var update = flag.Bool("update", false, "update golden files")

const (
	dataDir = "testdata"
)

type entry struct {
	source, golden string
}

// Use go test -update to create/update the respective golden files.
var data = []entry{
	{"complexhcl.input", "complexhcl.golden"},
	{"list.input", "list.golden"},
	{"list_comment.input", "list_comment.golden"},
	{"comment.input", "comment.golden"},
	{"comment_crlf.input", "comment.golden"},
	{"comment_aligned.input", "comment_aligned.golden"},
	{"comment_array.input", "comment_array.golden"},
	{"comment_end_file.input", "comment_end_file.golden"},
	{"comment_multiline_indent.input", "comment_multiline_indent.golden"},
	{"comment_multiline_no_stanza.input", "comment_multiline_no_stanza.golden"},
	{"comment_multiline_stanza.input", "comment_multiline_stanza.golden"},
	{"comment_newline.input", "comment_newline.golden"},
	{"comment_object_multi.input", "comment_object_multi.golden"},
	{"comment_standalone.input", "comment_standalone.golden"},
	{"empty_block.input", "empty_block.golden"},
	{"list_of_objects.input", "list_of_objects.golden"},
	{"multiline_string.input", "multiline_string.golden"},
	{"object_singleline.input", "object_singleline.golden"},
	{"object_with_heredoc.input", "object_with_heredoc.golden"},
}

func TestFiles(t *testing.T) {
	for _, e := range data {
		source := filepath.Join(dataDir, e.source)
		golden := filepath.Join(dataDir, e.golden)
		t.Run(e.source, func(t *testing.T) {
			check(t, source, golden)
		})
	}
}

func check(t *testing.T, source, golden string) {
	src, err := ioutil.ReadFile(source)
	if err != nil {
		t.Error(err)
		return
	}

	res, err := format(src)
	if err != nil {
		t.Error(err)
		return
	}

	// update golden files if necessary
	if *update {
		if err := ioutil.WriteFile(golden, res, 0644); err != nil {
			t.Error(err)
		}
		return
	}

	// get golden
	gld, err := ioutil.ReadFile(golden)
	if err != nil {
		t.Error(err)
		return
	}

	// formatted source and golden must be the same
	if err := diff(source, golden, res, gld); err != nil {
		t.Error(err)
		return
	}
}

// diff compares a and b.
func diff(aname, bname string, a, b []byte) error {
	var buf bytes.Buffer // holding long error message

	// compare lengths
	if len(a) != len(b) {
		fmt.Fprintf(&buf, "\nlength changed: len(%s) = %d, len(%s) = %d", aname, len(a), bname, len(b))
	}

	// compare contents
	line := 1
	offs := 1
	for i := 0; i < len(a) && i < len(b); i++ {
		ch := a[i]
		if ch != b[i] {
			fmt.Fprintf(&buf, "\n%s:%d:%d: %q", aname, line, i-offs+1, lineAt(a, offs))
			fmt.Fprintf(&buf, "\n%s:%d:%d: %q", bname, line, i-offs+1, lineAt(b, offs))
			fmt.Fprintf(&buf, "\n\n")
			break
		}
		if ch == '\n' {
			line++
			offs = i + 1
		}
	}

	if buf.Len() > 0 {
		return errors.New(buf.String())
	}
	return nil
}

// format parses src, prints the corresponding AST, verifies the resulting
// src is syntactically correct, and returns the resulting src or an error
// if any.
func format(src []byte) ([]byte, error) {
	formatted, err := Format(src)
	if err != nil {
		return nil, err
	}

	// make sure formatted output is syntactically correct
	if _, err := parser.Parse(formatted); err != nil {
		return nil, fmt.Errorf("parse: %s\n%s", err, formatted)
	}

	return formatted, nil
}

// lineAt returns the line in text starting at offset offs.
func lineAt(text []byte, offs int) []byte {
	i := offs
	for i < len(text) && text[i] != '\n' {
		i++
	}
	return text[offs:i]
}

// TestFormatParsable ensures that the output of Format() is can be parsed again.
func TestFormatValidOutput(t *testing.T) {
	cases := []string{
		"#\x00",
		"#\ue123t",
		"x=//\n0y=<<_\n_\n",
		"Y=<<4\n4/\n\n\n/4/@=4/\n\n\n/4000000004\r\r\n00004\n",
		"x=<<_\n_\r\r\n_\n",
	}

	for _, c := range cases {
		f, err := Format([]byte(c))
		if err != nil {
			// ignore these failures, not all inputs are valid HCL.
			t.Logf("Format(%q) = %v", c, err)
			continue
		}

		if _, err := parser.Parse(f); err != nil {
			t.Errorf("Format(%q) = %q; Parse(%q) = %v", c, f, f, err)
			continue
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package endtoend

import (
	"fmt"
	"sync"
	"testing"

	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/mysql"
	"github.com/youtube/vitess/go/vt/vttest"
)

// This file contains various long-running tests for mysql.

// BenchmarkWithRealDatabase runs a real MySQL database, and runs all kinds
// of benchmarks on it. To minimize overhead, we only run one database, and
// run all the benchmarks on it.
func BenchmarkWithRealDatabase(b *testing.B) {
	// Common setup code.
	hdl, err := vttest.LaunchVitess(
		vttest.MySQLOnly("vttest"),
		vttest.Schema("create table a(id int, name varchar(128), primary key(id))"),
		vttest.NoStderr())
	if err != nil {
		b.Fatal(err)
	}
	defer func() {
		err = hdl.TearDown()
		if err != nil {
			b.Error(err)
		}
	}()
	params, err := hdl.MySQLConnParams()
	if err != nil {
		b.Error(err)
	}

	b.Run("Inserts", func(b *testing.B) {
		benchmarkInserts(b, &params)
	})
	b.Run("ParallelReads", func(b *testing.B) {
		benchmarkParallelReads(b, &params, 10)
	})
}

func benchmarkInserts(b *testing.B, params *mysql.ConnParams) {
	// Connect.
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, params)
	if err != nil {
		b.Fatal(err)
	}
	defer conn.Close()

	// Delete what we may already have in the database.
	if _, err := conn.ExecuteFetch("delete from a", 0, false); err != nil {
		b.Fatalf("delete failed: %v", err)
	}

	// Now reset timer.
	b.ResetTimer()

	// Do the insert.
	for i := 0; i < b.N; i++ {
		_, err := conn.ExecuteFetch(fmt.Sprintf("insert into a(id, name) values(%v, 'nice name %v')", i, i), 0, false)
		if err != nil {
			b.Fatalf("ExecuteFetch(%v) failed: %v", i, err)
		}
	}
}

func benchmarkParallelReads(b *testing.B, params *mysql.ConnParams, parallelCount int) {
	ctx := context.Background()
	wg := sync.WaitGroup{}
	for i := 0; i < parallelCount; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()

			conn, err := mysql.Connect(ctx, params)
			if err != nil {
				b.Fatal(err)
			}

			for j := 0; j < b.N; j++ {
				if _, err := conn.ExecuteFetch("select * from a", 10000, true); err != nil {
					b.Fatalf("ExecuteFetch(%v, %v) failed: %v", i, j, err)
				}
			}
			conn.Close()
		}(i)
	}
	wg.Wait()
}
<commit_msg>Converting mysql/query_benchmark to new vttest.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package endtoend

import (
	"fmt"
	"os"
	"sync"
	"testing"

	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/mysql"
	vttestpb "github.com/youtube/vitess/go/vt/proto/vttest"
	"github.com/youtube/vitess/go/vt/vttest"
)

// This file contains various long-running tests for mysql.

// BenchmarkWithRealDatabase runs a real MySQL database, and runs all kinds
// of benchmarks on it. To minimize overhead, we only run one database, and
// run all the benchmarks on it.
func BenchmarkWithRealDatabase(b *testing.B) {
	// Launch MySQL.
	// We need a Keyspace in the topology, so the DbName is set.
	// We need a Shard too, so the database 'vttest' is created.
	cfg := vttest.Config{
		Topology: &vttestpb.VTTestTopology{
			Keyspaces: []*vttestpb.Keyspace{
				{
					Name: "vttest",
					Shards: []*vttestpb.Shard{
						{
							Name:           "0",
							DbNameOverride: "vttest",
						},
					},
				},
			},
		},
		OnlyMySQL: true,
	}
	if err := cfg.InitSchemas("vttest", "create table a(id int, name varchar(128), primary key(id))", nil); err != nil {
		b.Fatalf("InitSchemas failed: %v\n", err)
	}
	defer os.RemoveAll(cfg.SchemaDir)
	cluster := vttest.LocalCluster{
		Config: cfg,
	}
	if err := cluster.Setup(); err != nil {
		b.Fatalf("could not launch mysql: %v\n", err)
	}
	defer cluster.TearDown()
	params := cluster.MySQLConnParams()

	b.Run("Inserts", func(b *testing.B) {
		benchmarkInserts(b, &params)
	})
	b.Run("ParallelReads", func(b *testing.B) {
		benchmarkParallelReads(b, &params, 10)
	})
}

func benchmarkInserts(b *testing.B, params *mysql.ConnParams) {
	// Connect.
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, params)
	if err != nil {
		b.Fatal(err)
	}
	defer conn.Close()

	// Delete what we may already have in the database.
	if _, err := conn.ExecuteFetch("delete from a", 0, false); err != nil {
		b.Fatalf("delete failed: %v", err)
	}

	// Now reset timer.
	b.ResetTimer()

	// Do the insert.
	for i := 0; i < b.N; i++ {
		_, err := conn.ExecuteFetch(fmt.Sprintf("insert into a(id, name) values(%v, 'nice name %v')", i, i), 0, false)
		if err != nil {
			b.Fatalf("ExecuteFetch(%v) failed: %v", i, err)
		}
	}
}

func benchmarkParallelReads(b *testing.B, params *mysql.ConnParams, parallelCount int) {
	ctx := context.Background()
	wg := sync.WaitGroup{}
	for i := 0; i < parallelCount; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()

			conn, err := mysql.Connect(ctx, params)
			if err != nil {
				b.Fatal(err)
			}

			for j := 0; j < b.N; j++ {
				if _, err := conn.ExecuteFetch("select * from a", 10000, true); err != nil {
					b.Fatalf("ExecuteFetch(%v, %v) failed: %v", i, j, err)
				}
			}
			conn.Close()
		}(i)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package uploader

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"time"

	"koding/kites/keygen"
	"koding/klient/storage"
	"koding/logrotate"

	"github.com/boltdb/bolt"
	"github.com/koding/kite"
	"github.com/koding/logging"
)

var defaultLog = logging.NewCustom("uploader", false)

// Options represents arguments required to create a Uploader value.
type Options struct {
	KeygenURL string      // required
	Kite      *kite.Kite  // required
	Bucket    string      // required
	Region    string      // required
	DB        *bolt.DB    // optional; in-memory store if nil
	Log       kite.Logger // optional; defaultLog if nil
}

// Uploader is a kite handler for "log.upload" method.
type Uploader struct {
	cfg    *Options
	rotate *logrotate.Uploader
	req    chan *UploadRequest
	close  chan struct{}
}

// New gives new uploader built from the given options.
func New(cfg *Options) *Uploader {
	log := defaultLog
	if l, ok := cfg.Log.(logging.Logger); ok {
		log = l
	}

	up := &Uploader{
		cfg: cfg,
		rotate: &logrotate.Uploader{
			UserBucket: keygen.NewUserBucket(&keygen.Config{
				ServerURL: cfg.KeygenURL,
				Kite:      cfg.Kite,
				Bucket:    cfg.Bucket,
				Region:    cfg.Region,
				Log:       log,
			}),
			MetaStore: storage.NewEncodingStorage(cfg.DB, []byte("uploader.metadata")),
		},
		req:   make(chan *UploadRequest),
		close: make(chan struct{}),
	}

	go up.process()

	return up
}

// UploadRequest represents a request of the "log.upload" kite method.
type UploadRequest struct {
	// File is a path to a file to upload.
	// File path is also used as a S3 key.
	File string `json:"file"`

	// Interval makes the uploader stream the File at the given interval.
	// By default File is not streamed.
	//
	// Ignored when File is empty and Content is used instead.
	// Minimal interval is 15m.
	Interval time.Duration `json:"interval"`

	// Content is a one-time log content to upload, requires key to be non-empty.
	// Superseded by File, if non-empty.
	Content []byte `json:"content"`

	// Key is required when Content is set.
	Key string `json:"key"`
}

// Valid validates the request.
func (req *UploadRequest) Valid() error {
	if req.File == "" && len(req.Content) != 0 && req.Key == "" {
		return errors.New("missing key")
	}

	if req.File != "" {
		if _, err := os.Stat(req.File); err != nil {
			return err
		}

		req.File = filepath.ToSlash(filepath.Clean(req.File))
	}

	return nil
}

func (req *UploadRequest) key() string {
	if len(req.Content) != 0 && req.File == "" {
		return req.Key
	}

	return req.File
}

// UploadResponse represents a response of the "log.upload" kite method.
type UploadResponse struct {
	URL string `json:"url"`
}

// UploadFile uploads the given file to the S3.
//
// If interval is > 0 then the given file is uploaded to S3 at the given interval.
func (up *Uploader) UploadFile(file string, interval time.Duration) (string, error) {
	req := &UploadRequest{
		File:     file,
		Interval: interval,
	}

	return up.upload(req)
}

// Upload is a kite handler for the "log.upload" method.
func (up *Uploader) Upload(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, errors.New("missing argument")
	}

	var req UploadRequest
	if err := r.Args.One().Unmarshal(&req); err != nil {
		return nil, err
	}

	s, err := up.upload(&req)
	if err != nil {
		return nil, err
	}

	return &UploadResponse{
		URL: s,
	}, nil
}

// Close stops uploading logs to the S3.
func (up *Uploader) Close() error {
	if up.close != nil {
		close(up.close)
		up.close = nil
	}

	return nil
}

func (up *Uploader) upload(req *UploadRequest) (string, error) {
	if err := req.Valid(); err != nil {
		return "", err
	}

	up.req <- req

	return up.url(req), nil
}

func (up *Uploader) url(req *UploadRequest) string {
	return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", up.cfg.Bucket, path.Clean(up.cfg.Kite.Config.Id+"/"+req.key()))
}

func (up *Uploader) log() kite.Logger {
	if up.cfg.Log != nil {
		return up.cfg.Log
	}

	return defaultLog
}

func (up *Uploader) process() {
	var (
		files   = make(map[int]string)
		tickers []*time.Ticker
		cases   = []reflect.SelectCase{
			0: {Dir: reflect.SelectRecv, Chan: reflect.ValueOf(up.close)},
			1: {Dir: reflect.SelectRecv, Chan: reflect.ValueOf(up.req)},
		}
		prefix = up.cfg.Kite.Config.Id
	)

	for {
		var req *UploadRequest

		switch n, v, _ := reflect.Select(cases); n {
		case 0:
			for _, t := range tickers {
				t.Stop()
			}
			return
		case 1:
			req = v.Interface().(*UploadRequest)

			if req.Interval > 0 && req.File != "" {
				if req.Interval < 15*time.Minute {
					req.Interval = 15 * time.Minute
				}

				t := time.NewTicker(req.Interval)

				tickers = append(tickers, t)
				files[len(cases)] = req.File
				cases = append(cases, reflect.SelectCase{
					Dir:  reflect.SelectRecv,
					Chan: reflect.ValueOf(t.C),
				})
			}

			fallthrough

		default:
			var err error
			var key string

			if req == nil {
				err = up.rotate.UploadFile(prefix, files[n])
				key = prefix + "/" + files[n]
			} else if req.File != "" {
				err = up.rotate.UploadFile(prefix, req.File)
				key = prefix + "/" + req.File
			} else {
				err = up.rotate.Upload(key, bytes.NewReader(req.Content))
				key = prefix + "/" + req.Key
			}

			switch key = path.Clean(key); {
			case err == nil:
				up.log().Debug("%s: uploaded successfully", key)
			case logrotate.IsNop(err):
				up.log().Debug("%s: nothing to upload", key)
			case os.IsNotExist(err):
				up.log().Debug("%s: file does not exist", key)
			default:
				up.log().Error("%s: failed to upload: %s", key, err)
			}
		}
	}
}
<commit_msg>klient/uploader: do not watch the same file multiple times<commit_after>package uploader

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"time"

	"koding/kites/keygen"
	"koding/klient/storage"
	"koding/logrotate"

	"github.com/boltdb/bolt"
	"github.com/koding/kite"
	"github.com/koding/logging"
)

var defaultLog = logging.NewCustom("uploader", false)

// Options represents arguments required to create a Uploader value.
type Options struct {
	KeygenURL string      // required
	Kite      *kite.Kite  // required
	Bucket    string      // required
	Region    string      // required
	DB        *bolt.DB    // optional; in-memory store if nil
	Log       kite.Logger // optional; defaultLog if nil
}

// Uploader is a kite handler for "log.upload" method.
type Uploader struct {
	cfg    *Options
	rotate *logrotate.Uploader
	req    chan *UploadRequest
	close  chan struct{}
}

// New gives new uploader built from the given options.
func New(cfg *Options) *Uploader {
	log := defaultLog
	if l, ok := cfg.Log.(logging.Logger); ok {
		log = l
	}

	up := &Uploader{
		cfg: cfg,
		rotate: &logrotate.Uploader{
			UserBucket: keygen.NewUserBucket(&keygen.Config{
				ServerURL: cfg.KeygenURL,
				Kite:      cfg.Kite,
				Bucket:    cfg.Bucket,
				Region:    cfg.Region,
				Log:       log,
			}),
			MetaStore: storage.NewEncodingStorage(cfg.DB, []byte("uploader.metadata")),
		},
		req:   make(chan *UploadRequest),
		close: make(chan struct{}),
	}

	go up.process()

	return up
}

// UploadRequest represents a request of the "log.upload" kite method.
type UploadRequest struct {
	// File is a path to a file to upload.
	// File path is also used as a S3 key.
	File string `json:"file"`

	// Interval makes the uploader stream the File at the given interval.
	// By default File is not streamed.
	//
	// Ignored when File is empty and Content is used instead.
	// Minimal interval is 15m.
	Interval time.Duration `json:"interval"`

	// Content is a one-time log content to upload, requires key to be non-empty.
	// Superseded by File, if non-empty.
	Content []byte `json:"content"`

	// Key is required when Content is set.
	Key string `json:"key"`
}

// Valid validates the request.
func (req *UploadRequest) Valid() error {
	if req.File == "" && len(req.Content) != 0 && req.Key == "" {
		return errors.New("missing key")
	}

	if req.File != "" {
		if _, err := os.Stat(req.File); err != nil {
			return err
		}

		req.File = filepath.ToSlash(filepath.Clean(req.File))
	}

	return nil
}

func (req *UploadRequest) key() string {
	if len(req.Content) != 0 && req.File == "" {
		return req.Key
	}

	return req.File
}

// UploadResponse represents a response of the "log.upload" kite method.
type UploadResponse struct {
	URL string `json:"url"`
}

// UploadFile uploads the given file to the S3.
//
// If interval is > 0 then the given file is uploaded to S3 at the given interval.
func (up *Uploader) UploadFile(file string, interval time.Duration) (string, error) {
	req := &UploadRequest{
		File:     file,
		Interval: interval,
	}

	return up.upload(req)
}

// Upload is a kite handler for the "log.upload" method.
func (up *Uploader) Upload(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, errors.New("missing argument")
	}

	var req UploadRequest
	if err := r.Args.One().Unmarshal(&req); err != nil {
		return nil, err
	}

	s, err := up.upload(&req)
	if err != nil {
		return nil, err
	}

	return &UploadResponse{
		URL: s,
	}, nil
}

// Close stops uploading logs to the S3.
func (up *Uploader) Close() error {
	if up.close != nil {
		close(up.close)
		up.close = nil
	}

	return nil
}

func (up *Uploader) upload(req *UploadRequest) (string, error) {
	if err := req.Valid(); err != nil {
		return "", err
	}

	up.req <- req

	return up.url(req), nil
}

func (up *Uploader) url(req *UploadRequest) string {
	return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", up.cfg.Bucket, path.Clean(up.cfg.Kite.Config.Id+"/"+req.key()))
}

func (up *Uploader) log() kite.Logger {
	if up.cfg.Log != nil {
		return up.cfg.Log
	}

	return defaultLog
}

func (up *Uploader) process() {
	var (
		files   = make(map[int]string)
		tickers []*time.Ticker
		cases   = []reflect.SelectCase{
			0: {Dir: reflect.SelectRecv, Chan: reflect.ValueOf(up.close)},
			1: {Dir: reflect.SelectRecv, Chan: reflect.ValueOf(up.req)},
		}
		prefix  = up.cfg.Kite.Config.Id
		watched = func(file string) bool {
			for _, f := range files {
				if f == file {
					return true
				}
			}

			return false
		}
	)

	for {
		var req *UploadRequest

		switch n, v, _ := reflect.Select(cases); n {
		case 0:
			for _, t := range tickers {
				t.Stop()
			}
			return
		case 1:
			req = v.Interface().(*UploadRequest)

			if req.Interval > 0 && req.File != "" {
				if req.Interval < 15*time.Minute {
					req.Interval = 15 * time.Minute
				}

				if !watched(req.File) {
					t := time.NewTicker(req.Interval)

					tickers = append(tickers, t)
					files[len(cases)] = req.File
					cases = append(cases, reflect.SelectCase{
						Dir:  reflect.SelectRecv,
						Chan: reflect.ValueOf(t.C),
					})
				}
			}

			fallthrough

		default:
			var err error
			var key string

			if req == nil {
				err = up.rotate.UploadFile(prefix, files[n])
				key = prefix + "/" + files[n]
			} else if req.File != "" {
				err = up.rotate.UploadFile(prefix, req.File)
				key = prefix + "/" + req.File
			} else {
				err = up.rotate.Upload(key, bytes.NewReader(req.Content))
				key = prefix + "/" + req.Key
			}

			switch key = path.Clean(key); {
			case err == nil:
				up.log().Debug("%s: uploaded successfully", key)
			case logrotate.IsNop(err):
				up.log().Debug("%s: nothing to upload", key)
			case os.IsNotExist(err):
				up.log().Debug("%s: file does not exist", key)
			default:
				up.log().Error("%s: failed to upload: %s", key, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Package main runs the server for team worker
package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	"socialapi/workers/team"

	"github.com/koding/runner"
)

var (
	// Name holds the worker name
	Name = "Team"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}
	// init mongo connection
	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	r.SetContext(team.NewController(r.Log, appConfig))
	r.Register(models.ChannelParticipant{}).OnCreate().Handle((*team.Controller).HandleParticipant)
	r.Register(models.ChannelParticipant{}).OnUpdate().Handle((*team.Controller).HandleParticipant)
	r.Register(models.ChannelParticipant{}).OnDelete().Handle((*team.Controller).HandleParticipant)
	r.Register(models.Channel{}).OnDelete().Handle((*team.Controller).HandleChannel)
	r.Listen()
	r.Wait()
}
<commit_msg>social/team: add listener after team creation<commit_after>// Package main runs the server for team worker
package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	"socialapi/workers/team"

	"github.com/koding/runner"
)

var (
	// Name holds the worker name
	Name = "Team"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}
	// init mongo connection
	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	r.SetContext(team.NewController(r.Log, appConfig))
	r.Register(models.ChannelParticipant{}).OnCreate().Handle((*team.Controller).HandleParticipant)
	r.Register(models.ChannelParticipant{}).OnUpdate().Handle((*team.Controller).HandleParticipant)
	r.Register(models.ChannelParticipant{}).OnDelete().Handle((*team.Controller).HandleParticipant)
	r.Register(models.ChannelParticipant{}).OnDelete().Handle((*team.Controller).HandleParticipant)
	r.Register(models.Channel{}).OnCreate().Handle((*team.Controller).HandleCreator)
	r.Register(models.Channel{}).OnDelete().Handle((*team.Controller).HandleChannel)
	r.Listen()
	r.Wait()
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package abstract

import (
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vtgate/semantics"
)

// Join represents a join. If we have a predicate, this is an inner join. If no predicate exists, it is a cross join
type Join struct {
	LHS, RHS  Operator
	Predicate sqlparser.Expr
	LeftJoin  bool
}

var _ Operator = (*Join)(nil)

// PushPredicate implements the Operator interface
func (j *Join) PushPredicate(expr sqlparser.Expr, semTable *semantics.SemTable) error {
	deps := semTable.RecursiveDeps(expr)
	switch {
	case deps.IsSolvedBy(j.LHS.TableID()):
		return j.LHS.PushPredicate(expr, semTable)
	case deps.IsSolvedBy(j.RHS.TableID()):
		if !j.LeftJoin {
			return j.RHS.PushPredicate(expr, semTable)
		}
		// we are looking for predicates like `tbl.col = <>` or `<> = tbl.col`,
		// where tbl is on the rhs of the left outer join
		if cmp, isCmp := expr.(*sqlparser.ComparisonExpr); isCmp && cmp.Operator != sqlparser.NullSafeEqualOp &&
			sqlparser.IsColName(cmp.Left) && semTable.RecursiveDeps(cmp.Left).IsSolvedBy(j.RHS.TableID()) ||
			sqlparser.IsColName(cmp.Right) && semTable.RecursiveDeps(cmp.Right).IsSolvedBy(j.RHS.TableID()) {
			// When the predicate we are pushing is using information from an outer table, we can
			// check whether the predicate is "null-intolerant" or not. Null-intolerant in this context means that
			// the predicate will not return true if the table columns are null.
			// Since an outer join is an inner join with the addition of all the rows from the left-hand side that
			// matched no rows on the right-hand, if we are later going to remove all the rows where the right-hand
			// side did not match, we might as well turn the join into an inner join.

			// This is based on the paper "Canonical Abstraction for Outerjoin Optimization" by J Rao et al
			j.LeftJoin = false
			return j.RHS.PushPredicate(expr, semTable)
		}
		// TODO - we should do this on the vtgate level once we have a Filter primitive
		return vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "unsupported: cross-shard left join and where clause")
	case deps.IsSolvedBy(j.LHS.TableID().Merge(j.RHS.TableID())):
		j.Predicate = sqlparser.AndExpressions(j.Predicate, expr)
		return nil
	}

	return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "Cannot push predicate: %s", sqlparser.String(expr))
}

// TableID implements the Operator interface
func (j *Join) TableID() semantics.TableSet {
	return j.RHS.TableID().Merge(j.LHS.TableID())
}

// UnsolvedPredicates implements the Operator interface
func (j *Join) UnsolvedPredicates(semTable *semantics.SemTable) []sqlparser.Expr {
	ts := j.TableID()
	var result []sqlparser.Expr
	for _, expr := range j.LHS.UnsolvedPredicates(semTable) {
		deps := semTable.DirectDeps(expr)
		if !deps.IsSolvedBy(ts) {
			result = append(result, expr)
		}
	}
	for _, expr := range j.RHS.UnsolvedPredicates(semTable) {
		deps := semTable.DirectDeps(expr)
		if !deps.IsSolvedBy(ts) {
			result = append(result, expr)
		}
	}
	return result
}

// CheckValid implements the Operator interface
func (j *Join) CheckValid() error {
	err := j.LHS.CheckValid()
	if err != nil {
		return err
	}

	return j.RHS.CheckValid()
}

// Compact implements the Operator interface
func (j *Join) Compact(semTable *semantics.SemTable) (Operator, error) {
	if j.LeftJoin {
		// we can't merge outer joins into a single QG
		return j, nil
	}

	lqg, lok := j.LHS.(*QueryGraph)
	rqg, rok := j.RHS.(*QueryGraph)
	if !lok || !rok {
		return j, nil
	}

	op := &QueryGraph{
		Tables:     append(lqg.Tables, rqg.Tables...),
		innerJoins: append(lqg.innerJoins, rqg.innerJoins...),
		NoDeps:     sqlparser.AndExpressions(lqg.NoDeps, rqg.NoDeps),
	}
	err := op.collectPredicate(j.Predicate, semTable)
	if err != nil {
		return nil, err
	}
	return op, nil
}
<commit_msg>bug: added missing parens that could otherwise lead to nil pointer errors<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package abstract

import (
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vtgate/semantics"
)

// Join represents a join. If we have a predicate, this is an inner join. If no predicate exists, it is a cross join
type Join struct {
	LHS, RHS  Operator
	Predicate sqlparser.Expr
	LeftJoin  bool
}

var _ Operator = (*Join)(nil)

// PushPredicate implements the Operator interface
func (j *Join) PushPredicate(expr sqlparser.Expr, semTable *semantics.SemTable) error {
	deps := semTable.RecursiveDeps(expr)
	switch {
	case deps.IsSolvedBy(j.LHS.TableID()):
		return j.LHS.PushPredicate(expr, semTable)
	case deps.IsSolvedBy(j.RHS.TableID()):
		if !j.LeftJoin {
			return j.RHS.PushPredicate(expr, semTable)
		}
		// we are looking for predicates like `tbl.col = <>` or `<> = tbl.col`,
		// where tbl is on the rhs of the left outer join
		if cmp, isCmp := expr.(*sqlparser.ComparisonExpr); isCmp && cmp.Operator != sqlparser.NullSafeEqualOp &&
			(sqlparser.IsColName(cmp.Left) && semTable.RecursiveDeps(cmp.Left).IsSolvedBy(j.RHS.TableID()) ||
				sqlparser.IsColName(cmp.Right) && semTable.RecursiveDeps(cmp.Right).IsSolvedBy(j.RHS.TableID())) {
			// When the predicate we are pushing is using information from an outer table, we can
			// check whether the predicate is "null-intolerant" or not. Null-intolerant in this context means that
			// the predicate will not return true if the table columns are null.
			// Since an outer join is an inner join with the addition of all the rows from the left-hand side that
			// matched no rows on the right-hand, if we are later going to remove all the rows where the right-hand
			// side did not match, we might as well turn the join into an inner join.

			// This is based on the paper "Canonical Abstraction for Outerjoin Optimization" by J Rao et al
			j.LeftJoin = false
			return j.RHS.PushPredicate(expr, semTable)
		}
		// TODO - we should do this on the vtgate level once we have a Filter primitive
		return vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "unsupported: cross-shard left join and where clause")
	case deps.IsSolvedBy(j.LHS.TableID().Merge(j.RHS.TableID())):
		j.Predicate = sqlparser.AndExpressions(j.Predicate, expr)
		return nil
	}

	return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "Cannot push predicate: %s", sqlparser.String(expr))
}

// TableID implements the Operator interface
func (j *Join) TableID() semantics.TableSet {
	return j.RHS.TableID().Merge(j.LHS.TableID())
}

// UnsolvedPredicates implements the Operator interface
func (j *Join) UnsolvedPredicates(semTable *semantics.SemTable) []sqlparser.Expr {
	ts := j.TableID()
	var result []sqlparser.Expr
	for _, expr := range j.LHS.UnsolvedPredicates(semTable) {
		deps := semTable.DirectDeps(expr)
		if !deps.IsSolvedBy(ts) {
			result = append(result, expr)
		}
	}
	for _, expr := range j.RHS.UnsolvedPredicates(semTable) {
		deps := semTable.DirectDeps(expr)
		if !deps.IsSolvedBy(ts) {
			result = append(result, expr)
		}
	}
	return result
}

// CheckValid implements the Operator interface
func (j *Join) CheckValid() error {
	err := j.LHS.CheckValid()
	if err != nil {
		return err
	}

	return j.RHS.CheckValid()
}

// Compact implements the Operator interface
func (j *Join) Compact(semTable *semantics.SemTable) (Operator, error) {
	if j.LeftJoin {
		// we can't merge outer joins into a single QG
		return j, nil
	}

	lqg, lok := j.LHS.(*QueryGraph)
	rqg, rok := j.RHS.(*QueryGraph)
	if !lok || !rok {
		return j, nil
	}

	op := &QueryGraph{
		Tables:     append(lqg.Tables, rqg.Tables...),
		innerJoins: append(lqg.innerJoins, rqg.innerJoins...),
		NoDeps:     sqlparser.AndExpressions(lqg.NoDeps, rqg.NoDeps),
	}
	err := op.collectPredicate(j.Predicate, semTable)
	if err != nil {
		return nil, err
	}
	return op, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"fmt"
	"strings"
	"testing"

	"github.com/stretchr/testify/mock"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/stretchr/testify/assert"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
)

func TestBuildGenericLabels(t *testing.T) {
	labels := buildGenericLabels(&asgTemplate{
		InstanceType: &instanceType{
			InstanceType: "c4.large",
			VCPU:         2,
			MemoryMb:     3840,
		},
		Region: "us-east-1",
	}, "sillyname")
	assert.Equal(t, "us-east-1", labels[kubeletapis.LabelZoneRegion])
	assert.Equal(t, "sillyname", labels[kubeletapis.LabelHostname])
	assert.Equal(t, "c4.large", labels[kubeletapis.LabelInstanceType])
	assert.Equal(t, cloudprovider.DefaultArch, labels[kubeletapis.LabelArch])
	assert.Equal(t, cloudprovider.DefaultOS, labels[kubeletapis.LabelOS])
}

func TestExtractLabelsFromAsg(t *testing.T) {
	tags := []*autoscaling.TagDescription{
		{
			Key:   aws.String("k8s.io/cluster-autoscaler/node-template/label/foo"),
			Value: aws.String("bar"),
		},
		{
			Key:   aws.String("bar"),
			Value: aws.String("baz"),
		},
	}

	labels := extractLabelsFromAsg(tags)

	assert.Equal(t, 1, len(labels))
	assert.Equal(t, "bar", labels["foo"])
}

func TestExtractTaintsFromAsg(t *testing.T) {
	tags := []*autoscaling.TagDescription{
		{
			Key:   aws.String("k8s.io/cluster-autoscaler/node-template/taint/dedicated"),
			Value: aws.String("foo:NoSchedule"),
		},
		{
			Key:   aws.String("bar"),
			Value: aws.String("baz"),
		},
	}

	expectedTaints := []apiv1.Taint{
		{
			Key:    "dedicated",
			Value:  "foo",
			Effect: apiv1.TaintEffectNoSchedule,
		},
	}

	taints := extractTaintsFromAsg(tags)
	assert.Equal(t, 1, len(taints))
	assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints))
}

func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool {
	set := make(map[apiv1.Taint]bool)
	for _, taint := range taints {
		set[taint] = true
	}
	return set
}
func TestBuildAsg(t *testing.T) {
	do := cloudprovider.NodeGroupDiscoveryOptions{}
	m, err := createAWSManagerInternal(nil, do, &testService)
	assert.NoError(t, err)

	asg, err := m.buildAsgFromSpec("1:5:test-asg")
	assert.NoError(t, err)
	assert.Equal(t, asg.MinSize(), 1)
	assert.Equal(t, asg.MaxSize(), 5)
	assert.Equal(t, asg.Id(), "test-asg")
	assert.Equal(t, asg.Name, "test-asg")
	assert.Equal(t, asg.Debug(), "test-asg (1:5)")

	_, err = m.buildAsgFromSpec("a")
	assert.Error(t, err)
	_, err = m.buildAsgFromSpec("a:b:c")
	assert.Error(t, err)
	_, err = m.buildAsgFromSpec("1:")
	assert.Error(t, err)
	_, err = m.buildAsgFromSpec("1:2:")
	assert.Error(t, err)
}

func validateAsg(t *testing.T, asg *Asg, name string, minSize int, maxSize int) {
	assert.Equal(t, name, asg.Name)
	assert.Equal(t, minSize, asg.minSize)
	assert.Equal(t, maxSize, asg.maxSize)
}

func TestFetchExplicitAsgs(t *testing.T) {
	min, max, groupname := 1, 10, "coolasg"

	s := &AutoScalingMock{}
	s.On("DescribeAutoScalingGroups", &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{aws.String(groupname)},
		MaxRecords:            aws.Int64(1),
	}).Return(&autoscaling.DescribeAutoScalingGroupsOutput{
		AutoScalingGroups: []*autoscaling.Group{
			{AutoScalingGroupName: aws.String(groupname)},
		},
	})

	s.On("DescribeAutoScalingGroupsPages",
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: aws.StringSlice([]string{groupname}),
			MaxRecords:            aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool)
		fn(&autoscaling.DescribeAutoScalingGroupsOutput{
			AutoScalingGroups: []*autoscaling.Group{
				{AutoScalingGroupName: aws.String(groupname)},
			}}, false)
	}).Return(nil)

	do := cloudprovider.NodeGroupDiscoveryOptions{
		// Register the same node group twice with different max nodes.
		// The intention is to test that the asgs.Register method will update
		// the node group instead of registering it twice.
		NodeGroupSpecs: []string{
			fmt.Sprintf("%d:%d:%s", min, max-1, groupname),
			fmt.Sprintf("%d:%d:%s", min, max, groupname),
		},
	}
	// fetchExplicitASGs is called at manager creation time.
	m, err := createAWSManagerInternal(nil, do, &autoScalingWrapper{s})
	assert.NoError(t, err)

	asgs := m.asgCache.get()
	assert.Equal(t, 1, len(asgs))
	validateAsg(t, asgs[0].config, groupname, min, max)
}

func TestFetchAutoAsgs(t *testing.T) {
	min, max := 1, 10
	groupname, tags := "coolasg", []string{"tag", "anothertag"}

	s := &AutoScalingMock{}
	// Lookup groups associated with tags
	s.On("DescribeTagsPages",
		&autoscaling.DescribeTagsInput{
			Filters: []*autoscaling.Filter{
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[0]})},
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[1]})},
			},
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeTagsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeTagsOutput, bool) bool)
		fn(&autoscaling.DescribeTagsOutput{
			Tags: []*autoscaling.TagDescription{
				{ResourceId: aws.String(groupname)},
				{ResourceId: aws.String(groupname)},
			}}, false)
	}).Return(nil).Once()

	// Describe the group to register it, then again to generate the instance
	// cache.
	s.On("DescribeAutoScalingGroupsPages",
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: aws.StringSlice([]string{groupname}),
			MaxRecords:            aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool)
		fn(&autoscaling.DescribeAutoScalingGroupsOutput{
			AutoScalingGroups: []*autoscaling.Group{{
				AutoScalingGroupName: aws.String(groupname),
				MinSize:              aws.Int64(int64(min)),
				MaxSize:              aws.Int64(int64(max)),
			}}}, false)
	}).Return(nil).Twice()

	do := cloudprovider.NodeGroupDiscoveryOptions{
		NodeGroupAutoDiscoverySpecs: []string{fmt.Sprintf("asg:tag=%s", strings.Join(tags, ","))},
	}

	// fetchAutoASGs is called at manager creation time, via forceRefresh
	m, err := createAWSManagerInternal(nil, do, &autoScalingWrapper{s})
	assert.NoError(t, err)

	asgs := m.asgCache.get()
	assert.Equal(t, 1, len(asgs))
	validateAsg(t, asgs[0].config, groupname, min, max)

	// Simulate the previously discovered ASG disappearing
	s.On("DescribeTagsPages",
		&autoscaling.DescribeTagsInput{
			Filters: []*autoscaling.Filter{
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[0]})},
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[1]})},
			},
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeTagsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeTagsOutput, bool) bool)
		fn(&autoscaling.DescribeTagsOutput{Tags: []*autoscaling.TagDescription{}}, false)
	}).Return(nil).Once()

	err = m.fetchAutoAsgs()
	assert.NoError(t, err)
	assert.Empty(t, m.asgCache.get())
}
<commit_msg>Disable flaky AWS test<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/mock"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/stretchr/testify/assert"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
)

func TestBuildGenericLabels(t *testing.T) {
	labels := buildGenericLabels(&asgTemplate{
		InstanceType: &instanceType{
			InstanceType: "c4.large",
			VCPU:         2,
			MemoryMb:     3840,
		},
		Region: "us-east-1",
	}, "sillyname")
	assert.Equal(t, "us-east-1", labels[kubeletapis.LabelZoneRegion])
	assert.Equal(t, "sillyname", labels[kubeletapis.LabelHostname])
	assert.Equal(t, "c4.large", labels[kubeletapis.LabelInstanceType])
	assert.Equal(t, cloudprovider.DefaultArch, labels[kubeletapis.LabelArch])
	assert.Equal(t, cloudprovider.DefaultOS, labels[kubeletapis.LabelOS])
}

func TestExtractLabelsFromAsg(t *testing.T) {
	tags := []*autoscaling.TagDescription{
		{
			Key:   aws.String("k8s.io/cluster-autoscaler/node-template/label/foo"),
			Value: aws.String("bar"),
		},
		{
			Key:   aws.String("bar"),
			Value: aws.String("baz"),
		},
	}

	labels := extractLabelsFromAsg(tags)

	assert.Equal(t, 1, len(labels))
	assert.Equal(t, "bar", labels["foo"])
}

func TestExtractTaintsFromAsg(t *testing.T) {
	tags := []*autoscaling.TagDescription{
		{
			Key:   aws.String("k8s.io/cluster-autoscaler/node-template/taint/dedicated"),
			Value: aws.String("foo:NoSchedule"),
		},
		{
			Key:   aws.String("bar"),
			Value: aws.String("baz"),
		},
	}

	expectedTaints := []apiv1.Taint{
		{
			Key:    "dedicated",
			Value:  "foo",
			Effect: apiv1.TaintEffectNoSchedule,
		},
	}

	taints := extractTaintsFromAsg(tags)
	assert.Equal(t, 1, len(taints))
	assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints))
}

func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool {
	set := make(map[apiv1.Taint]bool)
	for _, taint := range taints {
		set[taint] = true
	}
	return set
}
func TestBuildAsg(t *testing.T) {
	do := cloudprovider.NodeGroupDiscoveryOptions{}
	m, err := createAWSManagerInternal(nil, do, &testService)
	assert.NoError(t, err)

	asg, err := m.buildAsgFromSpec("1:5:test-asg")
	assert.NoError(t, err)
	assert.Equal(t, asg.MinSize(), 1)
	assert.Equal(t, asg.MaxSize(), 5)
	assert.Equal(t, asg.Id(), "test-asg")
	assert.Equal(t, asg.Name, "test-asg")
	assert.Equal(t, asg.Debug(), "test-asg (1:5)")

	_, err = m.buildAsgFromSpec("a")
	assert.Error(t, err)
	_, err = m.buildAsgFromSpec("a:b:c")
	assert.Error(t, err)
	_, err = m.buildAsgFromSpec("1:")
	assert.Error(t, err)
	_, err = m.buildAsgFromSpec("1:2:")
	assert.Error(t, err)
}

func validateAsg(t *testing.T, asg *Asg, name string, minSize int, maxSize int) {
	assert.Equal(t, name, asg.Name)
	assert.Equal(t, minSize, asg.minSize)
	assert.Equal(t, maxSize, asg.maxSize)
}

func TestFetchExplicitAsgs(t *testing.T) {
	min, max, groupname := 1, 10, "coolasg"

	s := &AutoScalingMock{}
	s.On("DescribeAutoScalingGroups", &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{aws.String(groupname)},
		MaxRecords:            aws.Int64(1),
	}).Return(&autoscaling.DescribeAutoScalingGroupsOutput{
		AutoScalingGroups: []*autoscaling.Group{
			{AutoScalingGroupName: aws.String(groupname)},
		},
	})

	s.On("DescribeAutoScalingGroupsPages",
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: aws.StringSlice([]string{groupname}),
			MaxRecords:            aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool)
		fn(&autoscaling.DescribeAutoScalingGroupsOutput{
			AutoScalingGroups: []*autoscaling.Group{
				{AutoScalingGroupName: aws.String(groupname)},
			}}, false)
	}).Return(nil)

	do := cloudprovider.NodeGroupDiscoveryOptions{
		// Register the same node group twice with different max nodes.
		// The intention is to test that the asgs.Register method will update
		// the node group instead of registering it twice.
		NodeGroupSpecs: []string{
			fmt.Sprintf("%d:%d:%s", min, max-1, groupname),
			fmt.Sprintf("%d:%d:%s", min, max, groupname),
		},
	}
	// fetchExplicitASGs is called at manager creation time.
	m, err := createAWSManagerInternal(nil, do, &autoScalingWrapper{s})
	assert.NoError(t, err)

	asgs := m.asgCache.get()
	assert.Equal(t, 1, len(asgs))
	validateAsg(t, asgs[0].config, groupname, min, max)
}

/* Disabled due to flakiness. See https://github.com/kubernetes/autoscaler/issues/608
func TestFetchAutoAsgs(t *testing.T) {
	min, max := 1, 10
	groupname, tags := "coolasg", []string{"tag", "anothertag"}

	s := &AutoScalingMock{}
	// Lookup groups associated with tags
	s.On("DescribeTagsPages",
		&autoscaling.DescribeTagsInput{
			Filters: []*autoscaling.Filter{
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[0]})},
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[1]})},
			},
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeTagsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeTagsOutput, bool) bool)
		fn(&autoscaling.DescribeTagsOutput{
			Tags: []*autoscaling.TagDescription{
				{ResourceId: aws.String(groupname)},
				{ResourceId: aws.String(groupname)},
			}}, false)
	}).Return(nil).Once()

	// Describe the group to register it, then again to generate the instance
	// cache.
	s.On("DescribeAutoScalingGroupsPages",
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: aws.StringSlice([]string{groupname}),
			MaxRecords:            aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool)
		fn(&autoscaling.DescribeAutoScalingGroupsOutput{
			AutoScalingGroups: []*autoscaling.Group{{
				AutoScalingGroupName: aws.String(groupname),
				MinSize:              aws.Int64(int64(min)),
				MaxSize:              aws.Int64(int64(max)),
			}}}, false)
	}).Return(nil).Twice()

	do := cloudprovider.NodeGroupDiscoveryOptions{
		NodeGroupAutoDiscoverySpecs: []string{fmt.Sprintf("asg:tag=%s", strings.Join(tags, ","))},
	}

	// fetchAutoASGs is called at manager creation time, via forceRefresh
	m, err := createAWSManagerInternal(nil, do, &autoScalingWrapper{s})
	assert.NoError(t, err)

	asgs := m.asgCache.get()
	assert.Equal(t, 1, len(asgs))
	validateAsg(t, asgs[0].config, groupname, min, max)

	// Simulate the previously discovered ASG disappearing
	s.On("DescribeTagsPages",
		&autoscaling.DescribeTagsInput{
			Filters: []*autoscaling.Filter{
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[0]})},
				{Name: aws.String("key"), Values: aws.StringSlice([]string{tags[1]})},
			},
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		},
		mock.AnythingOfType("func(*autoscaling.DescribeTagsOutput, bool) bool"),
	).Run(func(args mock.Arguments) {
		fn := args.Get(1).(func(*autoscaling.DescribeTagsOutput, bool) bool)
		fn(&autoscaling.DescribeTagsOutput{Tags: []*autoscaling.TagDescription{}}, false)
	}).Return(nil).Once()

	err = m.fetchAutoAsgs()
	assert.NoError(t, err)
	assert.Empty(t, m.asgCache.get())
}
*/
<|endoftext|>


<commit_before>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vtgate

import (
	"flag"
	"fmt"
	"html/template"
	"sort"
	"sync"
	"time"

	log "github.com/golang/glog"

	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/topo/topoproto"
	"golang.org/x/net/context"

	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)

var (
	srvTopoCacheTTL = flag.Duration("srv_topo_cache_ttl", 1*time.Second, "how long to use cached entries for topology")
	srvTopoTimeout  = flag.Duration("srv_topo_timeout", 2*time.Second, "topo server timeout")
)

const (
	queryCategory  = "query"
	cachedCategory = "cached"
	errorCategory  = "error"

	// TopoTemplate is the HTML to use to display the
	// ResilientSrvTopoServerCacheStatus object
	TopoTemplate = `

    SrvKeyspace Names Cache

    Cell
    SrvKeyspace Names

  {{range $i, $skn := .SrvKeyspaceNames}}

    {{github_com_youtube_vitess_vtctld_srv_cell $skn.Cell}}
    {{if $skn.LastError}}{{$skn.LastError}}{{else}}{{range $j, $value := $skn.Value}}{{github_com_youtube_vitess_vtctld_srv_keyspace $skn.Cell $value}} {{end}}{{end}}

  {{end}}

    SrvKeyspace Cache

    Cell
    Keyspace
    SrvKeyspace

  {{range $i, $sk := .SrvKeyspaces}}

    {{github_com_youtube_vitess_vtctld_srv_cell $sk.Cell}}
    {{github_com_youtube_vitess_vtctld_srv_keyspace $sk.Cell $sk.Keyspace}}
    {{if $sk.LastError}}{{$sk.LastError}}{{else}}{{$sk.StatusAsHTML}}{{end}}

  {{end}}

`
)

// ResilientSrvTopoServer is an implementation of SrvTopoServer based
// on a topo.Server that uses a cache for two purposes:
// - limit the QPS to the underlying topo.Server
// - return the last known value of the data if there is an error
type ResilientSrvTopoServer struct {
	topoServer topo.Server
	cacheTTL   time.Duration
	counts     *stats.Counters

	// mutex protects the cache map itself, not the individual
	// values in the cache.
	mutex                 sync.RWMutex
	srvKeyspaceNamesCache map[string]*srvKeyspaceNamesEntry
	srvKeyspaceCache      map[string]*srvKeyspaceEntry
}

type srvKeyspaceNamesEntry struct {
	// unmutable values
	cell string

	// the mutex protects any access to this structure (read or write)
	mutex sync.Mutex

	insertionTime time.Time
	value         []string
	lastError     error
	lastErrorCtx  context.Context
}

type srvKeyspaceEntry struct {
	// unmutable values
	cell     string
	keyspace string

	// the mutex protects any access to this structure (read or write)
	mutex sync.RWMutex

	// watchRunning describes if the watch go routine is running.
	// It is easier to have an explicit field instead of guessing
	// based on value and lastError.
	//
	// if watchrunning is not set, the next time we try to access the
	// keyspace, we will start a watch.
	// if watchrunning is set, we are guaranteed to have exactly one of
	// value or lastError be nil, and the other non-nil.
	watchRunning bool
	value        *topodatapb.SrvKeyspace
	lastError    error

	// lastErrorCtx tries to remember the context of the query
	// that failed to get the SrvKeyspace, so we can display it in
	// the status UI. The background routine that refreshes the
	// keyspace will not populate this field.
	// The intent is to have the source of a query that for instance
	// has a bad keyspace or cell name.
	lastErrorCtx context.Context
}

// NewResilientSrvTopoServer creates a new ResilientSrvTopoServer
// based on the provided topo.Server.
func NewResilientSrvTopoServer(base topo.Server, counterPrefix string) *ResilientSrvTopoServer {
	return &ResilientSrvTopoServer{
		topoServer: base,
		cacheTTL:   *srvTopoCacheTTL,
		counts:     stats.NewCounters(counterPrefix + "Counts"),

		srvKeyspaceNamesCache: make(map[string]*srvKeyspaceNamesEntry),
		srvKeyspaceCache:      make(map[string]*srvKeyspaceEntry),
	}
}

// GetSrvKeyspaceNames returns all keyspace names for the given cell.
func (server *ResilientSrvTopoServer) GetSrvKeyspaceNames(ctx context.Context, cell string) ([]string, error) {
	server.counts.Add(queryCategory, 1)

	// find the entry in the cache, add it if not there
	key := cell
	server.mutex.Lock()
	entry, ok := server.srvKeyspaceNamesCache[key]
	if !ok {
		entry = &srvKeyspaceNamesEntry{
			cell: cell,
		}
		server.srvKeyspaceNamesCache[key] = entry
	}
	server.mutex.Unlock()

	// Lock the entry, and do everything holding the lock.  This
	// means two concurrent requests will only issue one
	// underlying query.
	entry.mutex.Lock()
	defer entry.mutex.Unlock()

	// If the entry is fresh enough, return it
	if time.Now().Sub(entry.insertionTime) < server.cacheTTL {
		return entry.value, entry.lastError
	}

	// not in cache or too old, get the real value
	newCtx, cancel := context.WithTimeout(context.Background(), *srvTopoTimeout)
	defer cancel()

	result, err := server.topoServer.GetSrvKeyspaceNames(newCtx, cell)
	if err != nil {
		if entry.insertionTime.IsZero() {
			server.counts.Add(errorCategory, 1)
			log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (no cached value, caching and returning error)", newCtx, cell, err)
		} else {
			server.counts.Add(cachedCategory, 1)
			log.Warningf("GetSrvKeyspaceNames(%v, %v) failed: %v (returning cached value: %v %v)", newCtx, cell, err, entry.value, entry.lastError)
			return entry.value, entry.lastError
		}
	}

	// save the value we got and the current time in the cache
	entry.insertionTime = time.Now()
	entry.value = result
	entry.lastError = err
	entry.lastErrorCtx = newCtx
	return result, err
}

// WatchSrvVSchema is part of the SrvTopoServer API
func (server *ResilientSrvTopoServer) WatchSrvVSchema(ctx context.Context, cell string) (*topo.WatchSrvVSchemaData, <-chan *topo.WatchSrvVSchemaData, topo.CancelFunc) {
	return server.topoServer.WatchSrvVSchema(ctx, cell)
}

func (server *ResilientSrvTopoServer) getSrvKeyspaceEntry(cell, keyspace string) *srvKeyspaceEntry {
	// find the entry in the cache, add it if not there
	key := cell + "." + keyspace
	server.mutex.RLock()
	entry, ok := server.srvKeyspaceCache[key]
	if ok {
		server.mutex.RUnlock()
		return entry
	}
	server.mutex.RUnlock()

	server.mutex.Lock()
	entry, ok = server.srvKeyspaceCache[key]
	if !ok {
		entry = &srvKeyspaceEntry{
			cell:     cell,
			keyspace: keyspace,
		}
		server.srvKeyspaceCache[key] = entry
	}
	server.mutex.Unlock()
	return entry
}

// GetSrvKeyspace returns SrvKeyspace object for the given cell and keyspace.
func (server *ResilientSrvTopoServer) GetSrvKeyspace(ctx context.Context, cell, keyspace string) (*topodatapb.SrvKeyspace, error) {
	entry := server.getSrvKeyspaceEntry(cell, keyspace)

	// If the watch is already running, return the value
	entry.mutex.RLock()
	if entry.watchRunning {
		v, e := entry.value, entry.lastError
		entry.mutex.RUnlock()
		return v, e
	}
	entry.mutex.RUnlock()

	// Lock the entry, and do everything holding the lock.  This
	// means two concurrent requests will only issue one
	// underlying query.
	entry.mutex.Lock()
	defer entry.mutex.Unlock()

	// If the watch is already running, return the value
	if entry.watchRunning {
		return entry.value, entry.lastError
	}

	// Watch is not running, let's try to start it.
	// We use a background context, as starting the watch should keep going
	// even if the current query context is short-lived.
	newCtx := context.Background()
	current, changes, _ := server.topoServer.WatchSrvKeyspace(newCtx, cell, keyspace)
	if current.Err != nil {
		// lastError and lastErrorCtx will be visible from the UI
		// until the next try
		entry.value = nil
		entry.lastError = current.Err
		entry.lastErrorCtx = ctx
		log.Errorf("WatchSrvKeyspace failed for %v/%v: %v", cell, keyspace, current.Err)
		return nil, current.Err
	}

	// we are now watching, cache the first notification
	entry.watchRunning = true
	entry.value = current.Value
	entry.lastError = nil
	entry.lastErrorCtx = nil

	go func() {
		for c := range changes {
			if c.Err != nil {
				// Watch errored out. We log it, clear
				// our record, and return.
				err := fmt.Errorf("watch for SrvKeyspace %v in cell %v failed: %v", keyspace, cell, c.Err)
				log.Errorf("%v", err)
				entry.mutex.Lock()
				entry.watchRunning = false
				entry.value = nil
				entry.lastError = err
				entry.lastErrorCtx = nil
				entry.mutex.Unlock()
				return
			}

			// We got a new value, save it.
			entry.mutex.Lock()
			entry.value = c.Value
			entry.lastError = nil
			entry.lastErrorCtx = nil
			entry.mutex.Unlock()
		}
	}()

	return entry.value, entry.lastError
}

// The next few structures and methods are used to get a displayable
// version of the cache in a status page.

// SrvKeyspaceNamesCacheStatus is the current value for SrvKeyspaceNames
type SrvKeyspaceNamesCacheStatus struct {
	Cell         string
	Value        []string
	LastError    error
	LastErrorCtx context.Context
}

// SrvKeyspaceNamesCacheStatusList is used for sorting
type SrvKeyspaceNamesCacheStatusList []*SrvKeyspaceNamesCacheStatus

// Len is part of sort.Interface
func (skncsl SrvKeyspaceNamesCacheStatusList) Len() int {
	return len(skncsl)
}

// Less is part of sort.Interface
func (skncsl SrvKeyspaceNamesCacheStatusList) Less(i, j int) bool {
	return skncsl[i].Cell < skncsl[j].Cell
}

// Swap is part of sort.Interface
func (skncsl SrvKeyspaceNamesCacheStatusList) Swap(i, j int) {
	skncsl[i], skncsl[j] = skncsl[j], skncsl[i]
}

// SrvKeyspaceCacheStatus is the current value for a SrvKeyspace object
type SrvKeyspaceCacheStatus struct {
	Cell         string
	Keyspace     string
	Value        *topodatapb.SrvKeyspace
	LastError    error
	LastErrorCtx context.Context
}

// StatusAsHTML returns an HTML version of our status.
// It works best if there is data in the cache.
func (st *SrvKeyspaceCacheStatus) StatusAsHTML() template.HTML {
	if st.Value == nil {
		return template.HTML("No Data")
	}

	result := "Partitions:
"
	for tabletType, keyspacePartition := range st.Value.Partitions {
		result += " " + string(tabletType) + ""
		for _, shard := range keyspacePartition.ShardReferences {
			result += " " + shard.Name
		}
		result += "
"
	}

	if st.Value.ShardingColumnName != "" {
		result += "ShardingColumnName: " + st.Value.ShardingColumnName + "
"
		result += "ShardingColumnType: " + string(st.Value.ShardingColumnType) + "
"
	}

	if len(st.Value.ServedFrom) > 0 {
		result += "ServedFrom:
"
		for _, sf := range st.Value.ServedFrom {
			result += " " + topoproto.TabletTypeLString(sf.TabletType) + " " + sf.Keyspace + "
"
		}
	}

	return template.HTML(result)
}

// SrvKeyspaceCacheStatusList is used for sorting
type SrvKeyspaceCacheStatusList []*SrvKeyspaceCacheStatus

// Len is part of sort.Interface
func (skcsl SrvKeyspaceCacheStatusList) Len() int {
	return len(skcsl)
}

// Less is part of sort.Interface
func (skcsl SrvKeyspaceCacheStatusList) Less(i, j int) bool {
	return skcsl[i].Cell+"."+skcsl[i].Keyspace <
		skcsl[j].Cell+"."+skcsl[j].Keyspace
}

// Swap is part of sort.Interface
func (skcsl SrvKeyspaceCacheStatusList) Swap(i, j int) {
	skcsl[i], skcsl[j] = skcsl[j], skcsl[i]
}

// ResilientSrvTopoServerCacheStatus has the full status of the cache
type ResilientSrvTopoServerCacheStatus struct {
	SrvKeyspaceNames SrvKeyspaceNamesCacheStatusList
	SrvKeyspaces     SrvKeyspaceCacheStatusList
}

// CacheStatus returns a displayable version of the cache
func (server *ResilientSrvTopoServer) CacheStatus() *ResilientSrvTopoServerCacheStatus {
	result := &ResilientSrvTopoServerCacheStatus{}
	server.mutex.Lock()

	for _, entry := range server.srvKeyspaceNamesCache {
		entry.mutex.Lock()
		result.SrvKeyspaceNames = append(result.SrvKeyspaceNames, &SrvKeyspaceNamesCacheStatus{
			Cell:         entry.cell,
			Value:        entry.value,
			LastError:    entry.lastError,
			LastErrorCtx: entry.lastErrorCtx,
		})
		entry.mutex.Unlock()
	}

	for _, entry := range server.srvKeyspaceCache {
		entry.mutex.RLock()
		result.SrvKeyspaces = append(result.SrvKeyspaces, &SrvKeyspaceCacheStatus{
			Cell:         entry.cell,
			Keyspace:     entry.keyspace,
			Value:        entry.value,
			LastError:    entry.lastError,
			LastErrorCtx: entry.lastErrorCtx,
		})
		entry.mutex.RUnlock()
	}

	server.mutex.Unlock()

	// do the sorting without the mutex
	sort.Sort(result.SrvKeyspaceNames)
	sort.Sort(result.SrvKeyspaces)

	return result
}
<commit_msg>vtgate: Fix that the tablet type is shown in the "Partitions:" list on the status page.<commit_after>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vtgate

import (
	"flag"
	"fmt"
	"html/template"
	"sort"
	"sync"
	"time"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/vt/topo"

	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)

var (
	srvTopoCacheTTL = flag.Duration("srv_topo_cache_ttl", 1*time.Second, "how long to use cached entries for topology")
	srvTopoTimeout  = flag.Duration("srv_topo_timeout", 2*time.Second, "topo server timeout")
)

const (
	queryCategory  = "query"
	cachedCategory = "cached"
	errorCategory  = "error"

	// TopoTemplate is the HTML to use to display the
	// ResilientSrvTopoServerCacheStatus object
	TopoTemplate = `

    SrvKeyspace Names Cache

    Cell
    SrvKeyspace Names

  {{range $i, $skn := .SrvKeyspaceNames}}

    {{github_com_youtube_vitess_vtctld_srv_cell $skn.Cell}}
    {{if $skn.LastError}}{{$skn.LastError}}{{else}}{{range $j, $value := $skn.Value}}{{github_com_youtube_vitess_vtctld_srv_keyspace $skn.Cell $value}} {{end}}{{end}}

  {{end}}

    SrvKeyspace Cache

    Cell
    Keyspace
    SrvKeyspace

  {{range $i, $sk := .SrvKeyspaces}}

    {{github_com_youtube_vitess_vtctld_srv_cell $sk.Cell}}
    {{github_com_youtube_vitess_vtctld_srv_keyspace $sk.Cell $sk.Keyspace}}
    {{if $sk.LastError}}{{$sk.LastError}}{{else}}{{$sk.StatusAsHTML}}{{end}}

  {{end}}

`
)

// ResilientSrvTopoServer is an implementation of SrvTopoServer based
// on a topo.Server that uses a cache for two purposes:
// - limit the QPS to the underlying topo.Server
// - return the last known value of the data if there is an error
type ResilientSrvTopoServer struct {
	topoServer topo.Server
	cacheTTL   time.Duration
	counts     *stats.Counters

	// mutex protects the cache map itself, not the individual
	// values in the cache.
	mutex                 sync.RWMutex
	srvKeyspaceNamesCache map[string]*srvKeyspaceNamesEntry
	srvKeyspaceCache      map[string]*srvKeyspaceEntry
}

type srvKeyspaceNamesEntry struct {
	// unmutable values
	cell string

	// the mutex protects any access to this structure (read or write)
	mutex sync.Mutex

	insertionTime time.Time
	value         []string
	lastError     error
	lastErrorCtx  context.Context
}

type srvKeyspaceEntry struct {
	// unmutable values
	cell     string
	keyspace string

	// the mutex protects any access to this structure (read or write)
	mutex sync.RWMutex

	// watchRunning describes if the watch go routine is running.
	// It is easier to have an explicit field instead of guessing
	// based on value and lastError.
	//
	// if watchrunning is not set, the next time we try to access the
	// keyspace, we will start a watch.
	// if watchrunning is set, we are guaranteed to have exactly one of
	// value or lastError be nil, and the other non-nil.
	watchRunning bool
	value        *topodatapb.SrvKeyspace
	lastError    error

	// lastErrorCtx tries to remember the context of the query
	// that failed to get the SrvKeyspace, so we can display it in
	// the status UI. The background routine that refreshes the
	// keyspace will not populate this field.
	// The intent is to have the source of a query that for instance
	// has a bad keyspace or cell name.
	lastErrorCtx context.Context
}

// NewResilientSrvTopoServer creates a new ResilientSrvTopoServer
// based on the provided topo.Server.
func NewResilientSrvTopoServer(base topo.Server, counterPrefix string) *ResilientSrvTopoServer {
	return &ResilientSrvTopoServer{
		topoServer: base,
		cacheTTL:   *srvTopoCacheTTL,
		counts:     stats.NewCounters(counterPrefix + "Counts"),

		srvKeyspaceNamesCache: make(map[string]*srvKeyspaceNamesEntry),
		srvKeyspaceCache:      make(map[string]*srvKeyspaceEntry),
	}
}

// GetSrvKeyspaceNames returns all keyspace names for the given cell.
func (server *ResilientSrvTopoServer) GetSrvKeyspaceNames(ctx context.Context, cell string) ([]string, error) {
	server.counts.Add(queryCategory, 1)

	// find the entry in the cache, add it if not there
	key := cell
	server.mutex.Lock()
	entry, ok := server.srvKeyspaceNamesCache[key]
	if !ok {
		entry = &srvKeyspaceNamesEntry{
			cell: cell,
		}
		server.srvKeyspaceNamesCache[key] = entry
	}
	server.mutex.Unlock()

	// Lock the entry, and do everything holding the lock.  This
	// means two concurrent requests will only issue one
	// underlying query.
	entry.mutex.Lock()
	defer entry.mutex.Unlock()

	// If the entry is fresh enough, return it
	if time.Now().Sub(entry.insertionTime) < server.cacheTTL {
		return entry.value, entry.lastError
	}

	// not in cache or too old, get the real value
	newCtx, cancel := context.WithTimeout(context.Background(), *srvTopoTimeout)
	defer cancel()

	result, err := server.topoServer.GetSrvKeyspaceNames(newCtx, cell)
	if err != nil {
		if entry.insertionTime.IsZero() {
			server.counts.Add(errorCategory, 1)
			log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (no cached value, caching and returning error)", newCtx, cell, err)
		} else {
			server.counts.Add(cachedCategory, 1)
			log.Warningf("GetSrvKeyspaceNames(%v, %v) failed: %v (returning cached value: %v %v)", newCtx, cell, err, entry.value, entry.lastError)
			return entry.value, entry.lastError
		}
	}

	// save the value we got and the current time in the cache
	entry.insertionTime = time.Now()
	entry.value = result
	entry.lastError = err
	entry.lastErrorCtx = newCtx
	return result, err
}

// WatchSrvVSchema is part of the SrvTopoServer API
func (server *ResilientSrvTopoServer) WatchSrvVSchema(ctx context.Context, cell string) (*topo.WatchSrvVSchemaData, <-chan *topo.WatchSrvVSchemaData, topo.CancelFunc) {
	return server.topoServer.WatchSrvVSchema(ctx, cell)
}

func (server *ResilientSrvTopoServer) getSrvKeyspaceEntry(cell, keyspace string) *srvKeyspaceEntry {
	// find the entry in the cache, add it if not there
	key := cell + "." + keyspace
	server.mutex.RLock()
	entry, ok := server.srvKeyspaceCache[key]
	if ok {
		server.mutex.RUnlock()
		return entry
	}
	server.mutex.RUnlock()

	server.mutex.Lock()
	entry, ok = server.srvKeyspaceCache[key]
	if !ok {
		entry = &srvKeyspaceEntry{
			cell:     cell,
			keyspace: keyspace,
		}
		server.srvKeyspaceCache[key] = entry
	}
	server.mutex.Unlock()
	return entry
}

// GetSrvKeyspace returns SrvKeyspace object for the given cell and keyspace.
func (server *ResilientSrvTopoServer) GetSrvKeyspace(ctx context.Context, cell, keyspace string) (*topodatapb.SrvKeyspace, error) {
	entry := server.getSrvKeyspaceEntry(cell, keyspace)

	// If the watch is already running, return the value
	entry.mutex.RLock()
	if entry.watchRunning {
		v, e := entry.value, entry.lastError
		entry.mutex.RUnlock()
		return v, e
	}
	entry.mutex.RUnlock()

	// Lock the entry, and do everything holding the lock.  This
	// means two concurrent requests will only issue one
	// underlying query.
	entry.mutex.Lock()
	defer entry.mutex.Unlock()

	// If the watch is already running, return the value
	if entry.watchRunning {
		return entry.value, entry.lastError
	}

	// Watch is not running, let's try to start it.
	// We use a background context, as starting the watch should keep going
	// even if the current query context is short-lived.
	newCtx := context.Background()
	current, changes, _ := server.topoServer.WatchSrvKeyspace(newCtx, cell, keyspace)
	if current.Err != nil {
		// lastError and lastErrorCtx will be visible from the UI
		// until the next try
		entry.value = nil
		entry.lastError = current.Err
		entry.lastErrorCtx = ctx
		log.Errorf("WatchSrvKeyspace failed for %v/%v: %v", cell, keyspace, current.Err)
		return nil, current.Err
	}

	// we are now watching, cache the first notification
	entry.watchRunning = true
	entry.value = current.Value
	entry.lastError = nil
	entry.lastErrorCtx = nil

	go func() {
		for c := range changes {
			if c.Err != nil {
				// Watch errored out. We log it, clear
				// our record, and return.
				err := fmt.Errorf("watch for SrvKeyspace %v in cell %v failed: %v", keyspace, cell, c.Err)
				log.Errorf("%v", err)
				entry.mutex.Lock()
				entry.watchRunning = false
				entry.value = nil
				entry.lastError = err
				entry.lastErrorCtx = nil
				entry.mutex.Unlock()
				return
			}

			// We got a new value, save it.
			entry.mutex.Lock()
			entry.value = c.Value
			entry.lastError = nil
			entry.lastErrorCtx = nil
			entry.mutex.Unlock()
		}
	}()

	return entry.value, entry.lastError
}

// The next few structures and methods are used to get a displayable
// version of the cache in a status page.

// SrvKeyspaceNamesCacheStatus is the current value for SrvKeyspaceNames
type SrvKeyspaceNamesCacheStatus struct {
	Cell         string
	Value        []string
	LastError    error
	LastErrorCtx context.Context
}

// SrvKeyspaceNamesCacheStatusList is used for sorting
type SrvKeyspaceNamesCacheStatusList []*SrvKeyspaceNamesCacheStatus

// Len is part of sort.Interface
func (skncsl SrvKeyspaceNamesCacheStatusList) Len() int {
	return len(skncsl)
}

// Less is part of sort.Interface
func (skncsl SrvKeyspaceNamesCacheStatusList) Less(i, j int) bool {
	return skncsl[i].Cell < skncsl[j].Cell
}

// Swap is part of sort.Interface
func (skncsl SrvKeyspaceNamesCacheStatusList) Swap(i, j int) {
	skncsl[i], skncsl[j] = skncsl[j], skncsl[i]
}

// SrvKeyspaceCacheStatus is the current value for a SrvKeyspace object
type SrvKeyspaceCacheStatus struct {
	Cell         string
	Keyspace     string
	Value        *topodatapb.SrvKeyspace
	LastError    error
	LastErrorCtx context.Context
}

// StatusAsHTML returns an HTML version of our status.
// It works best if there is data in the cache.
func (st *SrvKeyspaceCacheStatus) StatusAsHTML() template.HTML {
	if st.Value == nil {
		return template.HTML("No Data")
	}

	result := "Partitions:
"
	for _, keyspacePartition := range st.Value.Partitions {
		result += " " + keyspacePartition.ServedType.String() + ":"
		for _, shard := range keyspacePartition.ShardReferences {
			result += " " + shard.Name
		}
		result += "
"
	}

	if st.Value.ShardingColumnName != "" {
		result += "ShardingColumnName: " + st.Value.ShardingColumnName + "
"
		result += "ShardingColumnType: " + st.Value.ShardingColumnType.String() + "
"
	}

	if len(st.Value.ServedFrom) > 0 {
		result += "ServedFrom:
"
		for _, sf := range st.Value.ServedFrom {
			result += " " + sf.TabletType.String() + ": " + sf.Keyspace + "
"
		}
	}

	return template.HTML(result)
}

// SrvKeyspaceCacheStatusList is used for sorting
type SrvKeyspaceCacheStatusList []*SrvKeyspaceCacheStatus

// Len is part of sort.Interface
func (skcsl SrvKeyspaceCacheStatusList) Len() int {
	return len(skcsl)
}

// Less is part of sort.Interface
func (skcsl SrvKeyspaceCacheStatusList) Less(i, j int) bool {
	return skcsl[i].Cell+"."+skcsl[i].Keyspace <
		skcsl[j].Cell+"."+skcsl[j].Keyspace
}

// Swap is part of sort.Interface
func (skcsl SrvKeyspaceCacheStatusList) Swap(i, j int) {
	skcsl[i], skcsl[j] = skcsl[j], skcsl[i]
}

// ResilientSrvTopoServerCacheStatus has the full status of the cache
type ResilientSrvTopoServerCacheStatus struct {
	SrvKeyspaceNames SrvKeyspaceNamesCacheStatusList
	SrvKeyspaces     SrvKeyspaceCacheStatusList
}

// CacheStatus returns a displayable version of the cache
func (server *ResilientSrvTopoServer) CacheStatus() *ResilientSrvTopoServerCacheStatus {
	result := &ResilientSrvTopoServerCacheStatus{}
	server.mutex.Lock()

	for _, entry := range server.srvKeyspaceNamesCache {
		entry.mutex.Lock()
		result.SrvKeyspaceNames = append(result.SrvKeyspaceNames, &SrvKeyspaceNamesCacheStatus{
			Cell:         entry.cell,
			Value:        entry.value,
			LastError:    entry.lastError,
			LastErrorCtx: entry.lastErrorCtx,
		})
		entry.mutex.Unlock()
	}

	for _, entry := range server.srvKeyspaceCache {
		entry.mutex.RLock()
		result.SrvKeyspaces = append(result.SrvKeyspaces, &SrvKeyspaceCacheStatus{
			Cell:         entry.cell,
			Keyspace:     entry.keyspace,
			Value:        entry.value,
			LastError:    entry.lastError,
			LastErrorCtx: entry.lastErrorCtx,
		})
		entry.mutex.RUnlock()
	}

	server.mutex.Unlock()

	// do the sorting without the mutex
	sort.Sort(result.SrvKeyspaceNames)
	sort.Sort(result.SrvKeyspaces)

	return result
}
<|endoftext|>


<commit_before>package fire

import (
	"errors"
	"reflect"

	"github.com/asaskevich/govalidator"
	"gopkg.in/mgo.v2/bson"
)

// Model is the main interface implemented by every fire model embedding Base.
type Model interface {
	ID() bson.ObjectId
	MustGet(string) interface{}
	MustSet(string, interface{})
	Meta() *Meta

	initialize(Model)
}

// The ValidatableModel interface can be additionally implemented to provide
// a custom validation method that is used by the Validate function.
type ValidatableModel interface {
	Model

	// The Validate method that should return normal errors about invalid fields.
	Validate() error
}

// Validate uses the govalidator package to validate the model based on
// the "valid" struct tags. If the passed model also implements the
// ValidatableModel interface, Validate method will be invoked after the struct
// validation.
func Validate(m Model) error {
	// validate id
	if !m.ID().Valid() {
		return errors.New("Invalid ID")
	}

	// validate model
	_, err := govalidator.ValidateStruct(m)
	if err != nil {
		return err
	}

	// invoke custom validation method when available
	if validatableModel, ok := m.(ValidatableModel); ok {
		err = validatableModel.Validate()
		if err != nil {
			return err
		}
	}

	return nil
}

// Init initializes the internals of a model and should be called before using
// a newly created Model.
func Init(model Model) Model {
	model.initialize(model)
	return model
}

// InitSlice initializes all models in a slice of the form *[]*Post and returns
// a new slice that contains all initialized models.
func InitSlice(ptr interface{}) []Model {
	// get slice
	slice := reflect.ValueOf(ptr).Elem()

	// make model slice
	models := make([]Model, slice.Len())

	// iterate over entries
	for i := 0; i < slice.Len(); i++ {
		m := Init(slice.Index(i).Interface().(Model))
		models[i] = m
	}

	return models
}

// C is a short-hand function to extract the collection of a model.
func C(m Model) string {
	return Init(m).Meta().Collection
}

// Base is the base for every fire model.
type Base struct {
	DocID bson.ObjectId `json:"-" bson:"_id,omitempty"`

	model interface{}
	meta  *Meta
}

// ID returns the models id.
func (b *Base) ID() bson.ObjectId {
	return b.DocID
}

// MustGet returns the value of the given field.
//
// Note: MustGet will return the value of the first field that has a matching
// Name, JSONName, or BSONName and will panic if no field can be found.
func (b *Base) MustGet(name string) interface{} {
	field := b.meta.MustFindField(name)

	// read value from model struct
	structField := reflect.ValueOf(b.model).Elem().Field(field.index)
	return structField.Interface()
}

// MustSet will set the given field to the the passed valued.
//
// Note: MustSet will set the value of the first field that has a matching Name,
// JSONName, or BSONName and will panic if no field can been found. The method
// will also panic if the type of the field and the passed value do not match.
func (b *Base) MustSet(name string, value interface{}) {
	field := b.meta.MustFindField(name)

	// set the value on model struct
	reflect.ValueOf(b.model).Elem().Field(field.index).Set(reflect.ValueOf(value))
}

// Meta returns the models Meta structure.
func (b *Base) Meta() *Meta {
	return b.meta
}

func (b *Base) initialize(model Model) {
	b.model = model

	// set id if missing
	if !b.DocID.Valid() {
		b.DocID = bson.NewObjectId()
	}

	// assign meta
	b.meta = NewMeta(model)
}
<commit_msg>invoke custom validation method before struct validation<commit_after>package fire

import (
	"errors"
	"reflect"

	"github.com/asaskevich/govalidator"
	"gopkg.in/mgo.v2/bson"
)

// Model is the main interface implemented by every fire model embedding Base.
type Model interface {
	ID() bson.ObjectId
	MustGet(string) interface{}
	MustSet(string, interface{})
	Meta() *Meta

	initialize(Model)
}

// The ValidatableModel interface can be additionally implemented to provide
// a custom validation method that is used by the Validate function.
type ValidatableModel interface {
	Model

	// The Validate method that should return normal errors about invalid fields.
	Validate() error
}

// Validate uses the govalidator package to validate the model based on
// the "valid" struct tags. If the passed model also implements the
// ValidatableModel interface, Validate method will be invoked after the struct
// validation.
func Validate(m Model) error {
	// validate id
	if !m.ID().Valid() {
		return errors.New("Invalid ID")
	}

	// invoke custom validation method when available
	if validatableModel, ok := m.(ValidatableModel); ok {
		err := validatableModel.Validate()
		if err != nil {
			return err
		}
	}

	// validate model
	_, err := govalidator.ValidateStruct(m)
	if err != nil {
		return err
	}

	return nil
}

// Init initializes the internals of a model and should be called before using
// a newly created Model.
func Init(model Model) Model {
	model.initialize(model)
	return model
}

// InitSlice initializes all models in a slice of the form *[]*Post and returns
// a new slice that contains all initialized models.
func InitSlice(ptr interface{}) []Model {
	// get slice
	slice := reflect.ValueOf(ptr).Elem()

	// make model slice
	models := make([]Model, slice.Len())

	// iterate over entries
	for i := 0; i < slice.Len(); i++ {
		m := Init(slice.Index(i).Interface().(Model))
		models[i] = m
	}

	return models
}

// C is a short-hand function to extract the collection of a model.
func C(m Model) string {
	return Init(m).Meta().Collection
}

// Base is the base for every fire model.
type Base struct {
	DocID bson.ObjectId `json:"-" bson:"_id,omitempty"`

	model interface{}
	meta  *Meta
}

// ID returns the models id.
func (b *Base) ID() bson.ObjectId {
	return b.DocID
}

// MustGet returns the value of the given field.
//
// Note: MustGet will return the value of the first field that has a matching
// Name, JSONName, or BSONName and will panic if no field can be found.
func (b *Base) MustGet(name string) interface{} {
	field := b.meta.MustFindField(name)

	// read value from model struct
	structField := reflect.ValueOf(b.model).Elem().Field(field.index)
	return structField.Interface()
}

// MustSet will set the given field to the the passed valued.
//
// Note: MustSet will set the value of the first field that has a matching Name,
// JSONName, or BSONName and will panic if no field can been found. The method
// will also panic if the type of the field and the passed value do not match.
func (b *Base) MustSet(name string, value interface{}) {
	field := b.meta.MustFindField(name)

	// set the value on model struct
	reflect.ValueOf(b.model).Elem().Field(field.index).Set(reflect.ValueOf(value))
}

// Meta returns the models Meta structure.
func (b *Base) Meta() *Meta {
	return b.meta
}

func (b *Base) initialize(model Model) {
	b.model = model

	// set id if missing
	if !b.DocID.Valid() {
		b.DocID = bson.NewObjectId()
	}

	// assign meta
	b.meta = NewMeta(model)
}
<|endoftext|>


<commit_before>// fibonacci
package fibonacci

import (
	"bytes"
	"fmt"
	"strconv"
)

var fibonacci_numbers = [...]int{
	2, 3, 5, 8, 13, 21,
	34, 55, 89, 144, 233, 377,
	610, 987, 1597, 2584, 4181, 6765,
	10946, 17711, 28657, 46368, 75025, 121393,
	196418, 317811, 514229, 832040, 1346269, 2178309,
	3524578, 5702887, 9227465, 14930352, 24157817, 39088169,
	63245986, 102334155, 165580141, 267914296, 433494437, 701408733,
	1134903170, 1836311903}

var SKIP_CONSOLIDATION = [...]int{
	8, 8, 8, 8, 8, 8,
	8, 8, 8, 8, 8, 8,
	8, 8, 8, 8, 8, 8,
	89, 89, 89, 89, 89, 89,
	89, 89, 89, 89, 89, 89,
	114, 114, 114, 114, 233, 233,
	233, 233, 233, 233, 233, 233,
	233, 233,
}

type HeapNode struct {
	value  int
	marked bool
	degree int
	child  *HeapNode
	left   *HeapNode
	right  *HeapNode
}

func (h *HeapNode) String() string {
	return fmt.Sprintf("Node[Degree:%d Value:%v, Marked:%t]", h.degree, h.value, h.marked)
}

type Heap struct {
	Compare       func(int, int) int
	fiboIndex     int
	fiboTarget    int
	oldFiboTarget int
	size          int
	roots         int

	minimum *HeapNode
	degrees []*HeapNode
}

func NewHeap(comp func(int, int) int) *Heap {
	return &Heap{Compare: comp, size: 0, roots: 0, fiboIndex: 0,
		fiboTarget: fibonacci_numbers[0], oldFiboTarget: fibonacci_numbers[0],
		degrees: make([]*HeapNode, 8)}
}

func (h *Heap) Size() int {
	return h.size
}

func (h *Heap) String() string {
	if h.size == 0 {
		return "Heap[Size:0]"
	}

	buff := bytes.NewBufferString("Heap[Size:" + strconv.Itoa(h.size) + "\n")
	printNode(h.minimum, 1, buff)
	buff.WriteString("]")
	return buff.String()
}

func printNode(current *HeapNode, tabs int, buff *bytes.Buffer) {
	for i := 0; i < tabs; i++ {
		buff.WriteString("\t")
	}
	buff.WriteString(current.String())
	buff.WriteString("\n")
	if current.child != nil {
		printNode(current.child, tabs+1, buff)
	}

	for next := current.right; next != current; next = next.right {
		for i := 0; i < tabs; i++ {
			buff.WriteString("\t")
		}
		buff.WriteString(next.String())
		buff.WriteString("\n")
		if next.child != nil {
			printNode(next.child, tabs+1, buff)
		}
	}
}

func (h *Heap) peek() interface{} {
	if h.minimum != nil {
		return h.minimum.value
	}
	return nil
}

func (h *Heap) Insert(v int) {
	h.size++
	h.roots++

	node := &HeapNode{value: v}
	minimum := h.minimum
	if minimum != nil {
		node.left = minimum
		node.right = minimum.right
		minimum.right = node
		node.right.left = node

		if h.Compare(node.value, minimum.value) < 0 {
			h.minimum = node
		}
	} else {
		h.minimum = node
		node.left = node
		node.right = node
	}

	if h.size == h.fiboTarget {
		h.oldFiboTarget = h.fiboTarget
		h.fiboIndex++
		h.fiboTarget = fibonacci_numbers[h.fiboIndex]
		h.consolidate()
	}
}

func (h *Heap) Consolidate() {
	h.consolidate()
}

func (h *Heap) consolidate() {
	var current, same *HeapNode
	iter := h.minimum
	roots := h.roots
	degrees := h.degrees
	if len(degrees) <= h.fiboIndex+1 {
		degrees = make([]*HeapNode, h.fiboIndex+1)
		h.degrees = degrees
	}

	for i := 0; i < roots; i++ {
		current = iter
		iter = iter.right
		if degrees[current.degree] == nil {
			degrees[current.degree] = current
		} else {
			for degree := current.degree; degrees[degree] != nil; degree = current.degree {
				h.roots--
				same = degrees[degree]

				if h.Compare(same.value, current.value) > 0 {
					merge(current, same)
					if h.minimum == same {
						h.minimum = current
					}
				} else {
					merge(same, current)
					if h.minimum == current {
						h.minimum = same
					}
					current = same
				}

				degrees[degree] = nil
			}
			degrees[current.degree] = current
		}
	}

	current = h.minimum
	iter = current.right
	for i := 0; i < h.roots; i++ {
		degrees[iter.degree] = nil
		if h.Compare(iter.value, current.value) < 0 {
			current = iter
		}
		iter = iter.right
	}
	h.minimum = current
}

func merge(parent, child *HeapNode) {
	child.right.left = child.left
	child.left.right = child.right

	if parent.degree == 0 {
		parent.child = child
		child.left = child
		child.right = child
	} else {
		child.left = parent.child
		child.right = parent.child.right
		parent.child.right = child
		child.right.left = child
	}

	parent.degree++
}

func (h *Heap) RemoveMin() int {
	oldMin := h.minimum
	if oldMin != nil {
		h.size--
		h.roots--

		if oldMin.degree > 0 {
			h.roots += oldMin.degree
			child := oldMin.child

			if oldMin.right != oldMin {
				mL := oldMin.left
				mR := oldMin.right
				cL := child.left

				mL.right = child
				child.left = mL
				cL.right = mR
				mR.left = cL
			}

			if h.roots <= SKIP_CONSOLIDATION[h.fiboIndex] {
				iter := child.right
				for i := 0; i < h.roots; i++ {
					if h.Compare(iter.value, child.value) < 0 {
						child = iter
					}
					iter = iter.right
				}
				h.minimum = child
			} else {
				h.minimum = child
				h.consolidate()
			}
		} else if oldMin.right == oldMin {
			h.minimum = nil
		} else {
			oldMin.left.right = oldMin.right
			oldMin.right.left = oldMin.left
			h.minimum = oldMin.right
			h.consolidate()
		}

		if h.size == h.oldFiboTarget {
			h.fiboTarget = h.oldFiboTarget
			if h.fiboIndex > 0 {
				h.fiboIndex--
			}
			h.oldFiboTarget = fibonacci_numbers[h.fiboIndex]
		}

		oldMin.child = nil
		oldMin.left = nil
		oldMin.right = nil
		return oldMin.value
	}
	return 0
}
<commit_msg>Fixed golint issues<commit_after>// Package fibonacci by TimTheSinner
package fibonacci

import (
	"bytes"
	"fmt"
	"strconv"
)

var fibonacciNumbers = [...]int{
	2, 3, 5, 8, 13, 21,
	34, 55, 89, 144, 233, 377,
	610, 987, 1597, 2584, 4181, 6765,
	10946, 17711, 28657, 46368, 75025, 121393,
	196418, 317811, 514229, 832040, 1346269, 2178309,
	3524578, 5702887, 9227465, 14930352, 24157817, 39088169,
	63245986, 102334155, 165580141, 267914296, 433494437, 701408733,
	1134903170, 1836311903}

var skipConsolidation = [...]int{
	8, 8, 8, 8, 8, 8,
	8, 8, 8, 8, 8, 8,
	8, 8, 8, 8, 8, 8,
	89, 89, 89, 89, 89, 89,
	89, 89, 89, 89, 89, 89,
	114, 114, 114, 114, 233, 233,
	233, 233, 233, 233, 233, 233,
	233, 233,
}

//HeapNode heap storage node
type HeapNode struct {
	value  int
	marked bool
	degree int
	child  *HeapNode
	left   *HeapNode
	right  *HeapNode
}

func (h *HeapNode) String() string {
	return fmt.Sprintf("Node[Degree:%d Value:%v, Marked:%t]", h.degree, h.value, h.marked)
}

//Heap fibonacci heap
type Heap struct {
	Compare       func(int, int) int
	fiboIndex     int
	fiboTarget    int
	oldFiboTarget int
	size          int
	roots         int

	minimum *HeapNode
	degrees []*HeapNode
}

//NewHeap create a new FibonacciHeap
func NewHeap(comp func(int, int) int) *Heap {
	return &Heap{Compare: comp, size: 0, roots: 0, fiboIndex: 0,
		fiboTarget: fibonacciNumbers[0], oldFiboTarget: fibonacciNumbers[0],
		degrees: make([]*HeapNode, 8)}
}

//Size the size of the heap
func (h *Heap) Size() int {
	return h.size
}

func (h *Heap) String() string {
	if h.size == 0 {
		return "Heap[Size:0]"
	}

	buff := bytes.NewBufferString("Heap[Size:" + strconv.Itoa(h.size) + "\n")
	printNode(h.minimum, 1, buff)
	buff.WriteString("]")
	return buff.String()
}

func printNode(current *HeapNode, tabs int, buff *bytes.Buffer) {
	for i := 0; i < tabs; i++ {
		buff.WriteString("\t")
	}
	buff.WriteString(current.String())
	buff.WriteString("\n")
	if current.child != nil {
		printNode(current.child, tabs+1, buff)
	}

	for next := current.right; next != current; next = next.right {
		for i := 0; i < tabs; i++ {
			buff.WriteString("\t")
		}
		buff.WriteString(next.String())
		buff.WriteString("\n")
		if next.child != nil {
			printNode(next.child, tabs+1, buff)
		}
	}
}

func (h *Heap) peek() interface{} {
	if h.minimum != nil {
		return h.minimum.value
	}
	return nil
}

//Insert an object into the heap
func (h *Heap) Insert(v int) {
	h.size++
	h.roots++

	node := &HeapNode{value: v}
	minimum := h.minimum
	if minimum != nil {
		node.left = minimum
		node.right = minimum.right
		minimum.right = node
		node.right.left = node

		if h.Compare(node.value, minimum.value) < 0 {
			h.minimum = node
		}
	} else {
		h.minimum = node
		node.left = node
		node.right = node
	}

	if h.size == h.fiboTarget {
		h.oldFiboTarget = h.fiboTarget
		h.fiboIndex++
		h.fiboTarget = fibonacciNumbers[h.fiboIndex]
		h.consolidate()
	}
}

//Consolidate the heap
func (h *Heap) Consolidate() {
	h.consolidate()
}

func (h *Heap) consolidate() {
	var current, same *HeapNode
	iter := h.minimum
	roots := h.roots
	degrees := h.degrees
	if len(degrees) <= h.fiboIndex+1 {
		degrees = make([]*HeapNode, h.fiboIndex+1)
		h.degrees = degrees
	}

	for i := 0; i < roots; i++ {
		current = iter
		iter = iter.right
		if degrees[current.degree] == nil {
			degrees[current.degree] = current
		} else {
			for degree := current.degree; degrees[degree] != nil; degree = current.degree {
				h.roots--
				same = degrees[degree]

				if h.Compare(same.value, current.value) > 0 {
					merge(current, same)
					if h.minimum == same {
						h.minimum = current
					}
				} else {
					merge(same, current)
					if h.minimum == current {
						h.minimum = same
					}
					current = same
				}

				degrees[degree] = nil
			}
			degrees[current.degree] = current
		}
	}

	current = h.minimum
	iter = current.right
	for i := 0; i < h.roots; i++ {
		degrees[iter.degree] = nil
		if h.Compare(iter.value, current.value) < 0 {
			current = iter
		}
		iter = iter.right
	}
	h.minimum = current
}

func merge(parent, child *HeapNode) {
	child.right.left = child.left
	child.left.right = child.right

	if parent.degree == 0 {
		parent.child = child
		child.left = child
		child.right = child
	} else {
		child.left = parent.child
		child.right = parent.child.right
		parent.child.right = child
		child.right.left = child
	}

	parent.degree++
}

//RemoveMin removes the smallest value in the heap
func (h *Heap) RemoveMin() int {
	oldMin := h.minimum
	if oldMin != nil {
		h.size--
		h.roots--

		if oldMin.degree > 0 {
			h.roots += oldMin.degree
			child := oldMin.child

			if oldMin.right != oldMin {
				mL := oldMin.left
				mR := oldMin.right
				cL := child.left

				mL.right = child
				child.left = mL
				cL.right = mR
				mR.left = cL
			}

			if h.roots <= skipConsolidation[h.fiboIndex] {
				iter := child.right
				for i := 0; i < h.roots; i++ {
					if h.Compare(iter.value, child.value) < 0 {
						child = iter
					}
					iter = iter.right
				}
				h.minimum = child
			} else {
				h.minimum = child
				h.consolidate()
			}
		} else if oldMin.right == oldMin {
			h.minimum = nil
		} else {
			oldMin.left.right = oldMin.right
			oldMin.right.left = oldMin.left
			h.minimum = oldMin.right
			h.consolidate()
		}

		if h.size == h.oldFiboTarget {
			h.fiboTarget = h.oldFiboTarget
			if h.fiboIndex > 0 {
				h.fiboIndex--
			}
			h.oldFiboTarget = fibonacciNumbers[h.fiboIndex]
		}

		oldMin.child = nil
		oldMin.left = nil
		oldMin.right = nil
		return oldMin.value
	}
	return 0
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package file provides utility file operations.
package file

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

// ReadFile is like ioutil.ReadFile.
func ReadFile(name string) ([]byte, os.FileInfo, error) {
	fi, err := os.Stat(name)
	if err != nil {
		return nil, nil, err
	}

	data, err := ioutil.ReadFile(name)
	return data, fi, err
}

// WriteFile is like ioutil.WriteFile
func WriteFile(name string, data []byte) error {
	return ioutil.WriteFile(name, data, 0644)
}

// OpenReadFile is like os.Open.
func OpenReadFile(name string) io.ReadCloser {
	f, err := os.Open(name)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not open %s\n", name)
		os.Exit(1)
	}
	return f
}
<commit_msg>Add ReadFile variable in buildtools/file API. (#1062)<commit_after>/*
Copyright 2016 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package file provides utility file operations.
package file

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

// This variable can be updated from the caller to change the API
// for reading a file.
var ReadFile = readFile

// readFile is like ioutil.ReadFile.
func readFile(name string) ([]byte, os.FileInfo, error) {
	fi, err := os.Stat(name)
	if err != nil {
		return nil, nil, err
	}

	data, err := ioutil.ReadFile(name)
	return data, fi, err
}

// WriteFile is like ioutil.WriteFile
func WriteFile(name string, data []byte) error {
	return ioutil.WriteFile(name, data, 0644)
}

// OpenReadFile is like os.Open.
func OpenReadFile(name string) io.ReadCloser {
	f, err := os.Open(name)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not open %s\n", name)
		os.Exit(1)
	}
	return f
}
<|endoftext|>