<commit_before>package main

import (
	"./lib"
	"bytes"
	"fmt"
)

func String(arr []int) string {
	if len(arr) == 0 {
		return "[]"
	}
	var buffer bytes.Buffer
	sep := "[ "
	for _, elem := range arr {
		elemString := fmt.Sprintf("%d", elem)
		buffer.WriteString(sep)
		buffer.WriteString(elemString)
		sep = ", "
	}
	buffer.WriteString(" ]")
	return buffer.String()
}

func main() {
	numbers := []int{7, 7, 4, 0, 9, 8, 2, 4, 1, 9}
	fmt.Println("start", String(numbers))
	lib.Partition(numbers)
	fmt.Println(String(numbers))
}
<commit_msg>Added comments<commit_after>package main

import (
	"./lib"
	"bytes"
	"fmt"
)

// String returns the string representation of a slice. This does not use the
// builtin functionality in Go, since it is designed to return the same result
// as stringifying an array in Node.js.
func String(arr []int) string {
	if len(arr) == 0 {
		return "[]"
	}
	var buffer bytes.Buffer
	sep := "[ "
	for _, elem := range arr {
		elemString := fmt.Sprintf("%d", elem)
		buffer.WriteString(sep)
		buffer.WriteString(elemString)
		sep = ", "
	}
	buffer.WriteString(" ]")
	return buffer.String()
}

func main() {
	numbers := []int{7, 7, 4, 0, 9, 8, 2, 4, 1, 9}
	fmt.Println("start", String(numbers))
	lib.Partition(numbers)
	fmt.Println(String(numbers))
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
	"github.com/jacobsa/aws/s3/time"
	"net/url"
)

// NonExistentBucketError represents an error due to an attempt to work with a
// bucket that doesn't exist according to S3.
type NonExistentBucketError struct {
	s string
}

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version.
	StoreObject(key string, data []byte) error
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// If the supplied bucket doesn't exist, a *NonExistentBucketError is returned.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	// Create a connection to the given region's endpoint.
	endpoint := &url.URL{Scheme: "https", Host: string(region)}
	httpConn, err := http.NewConn(endpoint)
	if err != nil {
		return nil, fmt.Errorf("http.NewConn: %v", err)
	}

	// Create an appropriate request signer.
	signer, err := auth.NewSigner(&key)
	if err != nil {
		return nil, fmt.Errorf("auth.NewSigner: %v", err)
	}

	return openBucket(name, httpConn, signer, time.RealClock())
}

func (e *NonExistentBucketError) Error() string {
	return e.s
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer,
	clock time.Clock) (Bucket, error) {
	return &bucket{name, httpConn, signer, clock}, nil
}

type bucket struct {
	name string
	httpConn http.Conn
	signer auth.Signer
	clock time.Clock
}

func (b *bucket) GetObject(key string) (data []byte, err error) {
	return nil, fmt.Errorf("TODO: Implement bucket.GetObject.")
}

func (b *bucket) StoreObject(key string, data []byte) error {
	return fmt.Errorf("TODO: Implement bucket.StoreObject.")
}
<commit_msg>Added key checking.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
	"github.com/jacobsa/aws/s3/time"
	"net/url"
	"unicode/utf8"
)

// NonExistentBucketError represents an error due to an attempt to work with a
// bucket that doesn't exist according to S3.
type NonExistentBucketError struct {
	s string
}

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version.
	StoreObject(key string, data []byte) error
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// If the supplied bucket doesn't exist, a *NonExistentBucketError is returned.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	// Create a connection to the given region's endpoint.
	endpoint := &url.URL{Scheme: "https", Host: string(region)}
	httpConn, err := http.NewConn(endpoint)
	if err != nil {
		return nil, fmt.Errorf("http.NewConn: %v", err)
	}

	// Create an appropriate request signer.
	signer, err := auth.NewSigner(&key)
	if err != nil {
		return nil, fmt.Errorf("auth.NewSigner: %v", err)
	}

	return openBucket(name, httpConn, signer, time.RealClock())
}

func (e *NonExistentBucketError) Error() string {
	return e.s
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer,
	clock time.Clock) (Bucket, error) {
	return &bucket{name, httpConn, signer, clock}, nil
}

type bucket struct {
	name string
	httpConn http.Conn
	signer auth.Signer
	clock time.Clock
}

func (b *bucket) GetObject(key string) (data []byte, err error) {
	return nil, fmt.Errorf("TODO: Implement bucket.GetObject.")
}

func (b *bucket) StoreObject(key string, data []byte) error {
	// Validate the key.
	if len(key) > 1024 {
		return fmt.Errorf("Keys may be no longer than 1024 bytes.")
	}

	if !utf8.ValidString(key) {
		return fmt.Errorf("Keys must be valid UTF-8.")
	}

	return fmt.Errorf("TODO: Implement bucket.StoreObject.")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"reflect"
	"testing"
)

func createDB(test string) {
	var err error
	err = openDB(fmt.Sprintf("file:%s?mode=memory&cache=shared", test))
	if err != nil {
		log.Fatal(err)
	}
	// TODO(jamesog): Load schema from a file
	schema := []string{
		`CREATE TABLE scan (ip text, port integer, proto text, firstseen datetime, lastseen datetime)`,
		`CREATE TABLE users (email text UNIQUE NOT NULL)`,
		`CREATE TABLE groups (group_name text UNIQUE NOT NULL)`,
		`CREATE TABLE job (id int, cidr text NOT NULL, ports text, proto text, requested_by text, submitted datetime, received datetime, count int)`,
		`CREATE TABLE traceroute (dest text UNIQUE NOT NULL, path text)`,
	}
	for _, stmt := range schema {
		_, err = db.Exec(stmt)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func destroyDB() {
	db.Close()
}

func TestLoadDataWithNoResults(t *testing.T) {
	createDB("TestLoadDataWithNoResults")
	defer destroyDB()
	data, err := loadData(sqlFilter{})
	if err != nil {
		t.Fatalf("error from loadData: %v", err)
	}
	if len(data) != 0 {
		t.Errorf("expected len 0, got %v", len(data))
	}
}

func TestLoadTraceroutesWithNoResults(t *testing.T) {
	createDB("TestLoadTraceroutesWithNoResults")
	defer destroyDB()
	tr, err := loadTraceroutes()
	if err != nil {
		t.Fatal(err)
	}
	if len(tr) != 0 {
		t.Errorf("expected 0 results, got %v", len(tr))
	}
}

func TestLoadJobsWithNoResults(t *testing.T) {
	createDB("TestLoadJobsWithNoResults")
	defer destroyDB()
	data, err := loadJobs(sqlFilter{})
	if err != nil {
		t.Fatalf("error from loadJobs: %v", err)
	}
	if len(data) != 0 {
		t.Errorf("expected len 0, got %v", len(data))
	}
}

func TestSaveData(t *testing.T) {
	createDB("TestSaveData")
	defer destroyDB()
	results := []result{
		{IP: "192.0.2.1", Ports: []port{{Port: 80, Proto: "tcp", Status: "open"}}},
		{IP: "192.0.2.2", Ports: []port{{Port: 80, Proto: "tcp", Status: "open"}}},
		{IP: "192.0.2.3", Ports: []port{{Port: 80, Proto: "tcp", Status: "open"}}},
	}
	count, err := saveData(results)
	if err != nil {
		t.Fatal(err)
	}
	if count != int64(len(results)) {
		t.Errorf("expected count %d, got %d", len(results), count)
	}

}

func TestSaveJob(t *testing.T) {
	createDB("TestSaveJob")
	defer destroyDB()
	id, err := saveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	if id != 1 {
		t.Errorf("expected ID 1, got %d", id)
	}
}

func TestResultData(t *testing.T) {
	createDB("TestResultData")
	defer destroyDB()
	want := scanData{0, 0, 0, "0001-01-01 00:00", nil}
	data, err := resultData("", "", "")
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(data, want) {
		t.Errorf("want %+v, got %+v", want, data)
	}
}

func TestUpdateJob(t *testing.T) {
	createDB("TestUpdateJob")
	defer destroyDB()
	id, err := saveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	err = updateJob(string(id), 999)
	if err != nil {
		t.Errorf("error updating job: %v", err)
	}
}
<commit_msg>Add some HTTP handler tests<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strings"
	"testing"
)

func createDB(test string) {
	var err error
	err = openDB(fmt.Sprintf("file:%s?mode=memory&cache=shared", test))
	if err != nil {
		log.Fatal(err)
	}
	// TODO(jamesog): Load schema from a file
	schema := []string{
		`CREATE TABLE scan (ip text, port integer, proto text, firstseen datetime, lastseen datetime)`,
		`CREATE TABLE users (email text UNIQUE NOT NULL)`,
		`CREATE TABLE groups (group_name text UNIQUE NOT NULL)`,
		`CREATE TABLE job (id int, cidr text NOT NULL, ports text, proto text, requested_by text, submitted datetime, received datetime, count int)`,
		`CREATE TABLE traceroute (dest text UNIQUE NOT NULL, path text)`,
	}
	for _, stmt := range schema {
		_, err = db.Exec(stmt)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func destroyDB() {
	db.Close()
}

func TestLoadDataWithNoResults(t *testing.T) {
	createDB("TestLoadDataWithNoResults")
	defer destroyDB()
	data, err := loadData(sqlFilter{})
	if err != nil {
		t.Fatalf("error from loadData: %v", err)
	}
	if len(data) != 0 {
		t.Errorf("expected len 0, got %v", len(data))
	}
}

func TestLoadTraceroutesWithNoResults(t *testing.T) {
	createDB("TestLoadTraceroutesWithNoResults")
	defer destroyDB()
	tr, err := loadTraceroutes()
	if err != nil {
		t.Fatal(err)
	}
	if len(tr) != 0 {
		t.Errorf("expected 0 results, got %v", len(tr))
	}
}

func TestLoadJobsWithNoResults(t *testing.T) {
	createDB("TestLoadJobsWithNoResults")
	defer destroyDB()
	data, err := loadJobs(sqlFilter{})
	if err != nil {
		t.Fatalf("error from loadJobs: %v", err)
	}
	if len(data) != 0 {
		t.Errorf("expected len 0, got %v", len(data))
	}
}

func TestSaveData(t *testing.T) {
	createDB("TestSaveData")
	defer destroyDB()
	results := []result{
		{IP: "192.0.2.1", Ports: []port{{Port: 80, Proto: "tcp", Status: "open"}}},
		{IP: "192.0.2.2", Ports: []port{{Port: 80, Proto: "tcp", Status: "open"}}},
		{IP: "192.0.2.3", Ports: []port{{Port: 80, Proto: "tcp", Status: "open"}}},
	}
	count, err := saveData(results)
	if err != nil {
		t.Fatal(err)
	}
	if count != int64(len(results)) {
		t.Errorf("expected count %d, got %d", len(results), count)
	}

}

func TestSaveJob(t *testing.T) {
	createDB("TestSaveJob")
	defer destroyDB()
	id, err := saveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	if id != 1 {
		t.Errorf("expected ID 1, got %d", id)
	}
}

func TestResultData(t *testing.T) {
	createDB("TestResultData")
	defer destroyDB()
	want := scanData{0, 0, 0, "0001-01-01 00:00", nil}
	data, err := resultData("", "", "")
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(data, want) {
		t.Errorf("want %+v, got %+v", want, data)
	}
}

func TestUpdateJob(t *testing.T) {
	createDB("TestUpdateJob")
	defer destroyDB()
	id, err := saveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	err = updateJob(string(id), 999)
	if err != nil {
		t.Errorf("error updating job: %v", err)
	}
}

// TestIndexHandlerWithoutAuth tests fetching the index page with
// authentication disabled
func TestIndexHandlerWithoutAuth(t *testing.T) {
	createDB("TestIndexHandlerWithoutAuth")
	defer destroyDB()

	// We can't go through the OAuth2 login flow in tests
	authDisabled = true

	setupTemplates()

	r := httptest.NewRequest("GET", "/", nil)
	w := httptest.NewRecorder()
	index(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
}

// TestIPsHandler tests that we get expected JSON data
func TestIPsHandler(t *testing.T) {
	createDB("TestIPsHandler")
	defer destroyDB()

	r := httptest.NewRequest("GET", "/ips.json", nil)
	w := httptest.NewRecorder()
	ips(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
	ct := resp.Header.Get("Content-Type")
	if ct != "application/json" {
		t.Errorf("expected Content-Type: %s, got %v", "application/json", ct)
	}
}

func TestJobHandler(t *testing.T) {
	createDB("TestJobHandler")
	defer destroyDB()

	r := httptest.NewRequest("GET", "/job", nil)
	w := httptest.NewRecorder()
	newJob(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}

	v := url.Values{}
	v.Set("cidr", "192.0.2.0/24")
	v.Set("ports", "1-1024")
	v.Set("proto", "tcp")

	r = httptest.NewRequest("POST", "/job", strings.NewReader(v.Encode()))
	w = httptest.NewRecorder()
	newJob(w, r)

	resp = w.Result()
	body, _ = ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
}

func TestJobsHandler(t *testing.T) {
	createDB("TestJobsHandler")
	defer destroyDB()

	r := httptest.NewRequest("GET", "/jobs", nil)
	w := httptest.NewRecorder()
	jobs(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
	ct := resp.Header.Get("Content-Type")
	if ct != "application/json" {
		t.Errorf("expected Content-Type: %s, got %v", "application/json", ct)
	}
}

func TestResultsHandler(t *testing.T) {
	createDB("TestResultsHandler")
	defer destroyDB()

	data := bytes.NewBuffer([]byte(`[{"ip":"192.0.2.1","ports":[{"port":80,"proto":"tcp","status":"open","reason":"syn-ack","ttl":57}]}]`))

	r := httptest.NewRequest("POST", "/results", data)
	r.Header.Set("Content-Type", "application/json")
	w := httptest.NewRecorder()
	recvResults(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}

	filter := sqlFilter{Where: []string{"ip = ?"}, Values: []interface{}{"192.0.2.1"}}
	results, err := loadData(filter)
	if err != nil {
		t.Errorf("couldn't retrieve results from database: %v", err)
	}
	if len(results) > 1 {
		t.Errorf("expected 1 result, got %d", len(results))
	}
	if results[0].IP != "192.0.2.1" {
		t.Errorf("expected IP %s, got %v", "192.0.2.1", results[0].IP)
	}
	if results[0].Port != 80 {
		t.Errorf("expected port %d, got %v", 80, results[0].Port)
	}
	if results[0].Proto != "tcp" {
		t.Errorf("expected proto %s, got %v", "tcp", results[0].Proto)
	}

	// TODO(jamesog): We should test sending the same data to test the update
	// path in saveData() but this is currently difficult due to the way
	// times are stored in the database (as strings instead of raw values) -
	// it would require sleeping for some time, which isn't good.
	// ls := results[0].LastSeen
	// time.Sleep(70 * time.Second)
	// recvResults(w, r)

	// resp = w.Result()
	// body, _ = ioutil.ReadAll(resp.Body)
	// if resp.StatusCode != http.StatusOK {
	// 	t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	// }

	// results, _ = loadData(filter)
	// if results[0].LastSeen == ls {
	// 	t.Errorf("lastseen did not update, %v, %v", ls, results[0].LastSeen)
	// }
}

func TestJobResultsHandler(t *testing.T) {
	createDB("TestJobResultsHandler")
	defer destroyDB()

	data := bytes.NewBuffer([]byte(`[{"ip":"192.0.2.1","ports":[{"port":80,"proto":"tcp","status":"open","reason":"syn-ack","ttl":57}]}]`))

	mux := setupRouter()
	ts := httptest.NewServer(mux)
	defer ts.Close()

	req, err := http.NewRequest("PUT", ts.URL+"/results/1", data)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v", resp.StatusCode)
	}
}

// TestTracerouteHandler tests fetching a route, ensuring it fails, uploading
// that route then fetching it.
func TestTracerouteHandler(t *testing.T) {
	createDB("TestTracerouteHandler")
	defer destroyDB()

	route := `
traceroute to 192.0.2.1 (192.0.2.1), 64 hops max, 52 byte packets
1  router.internal (192.168.0.1)  1.308 ms  1.087 ms  0.929 ms
2  server.example.com (192.0.2.1)  8.134 ms !N  6.533 ms !N  6.295 ms !N
`

	mux := setupRouter()
	ts := httptest.NewServer(mux)
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL+"/traceroute/192.0.2.1", nil)
	if err != nil {
		t.Fatal(err)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusNotFound {
		t.Errorf("expected status %d, got %v: %s", http.StatusNotFound, resp.StatusCode, body)
	}

	postBody := new(bytes.Buffer)
	mp := multipart.NewWriter(postBody)
	mp.WriteField("dest", "192.0.2.1")
	ff, err := mp.CreateFormFile("traceroute", "traceroute")
	ff.Write([]byte(route))
	mp.Close()

	req = httptest.NewRequest("POST", "/traceroute", postBody)
	req.Header.Set("Content-Type", mp.FormDataContentType())
	w := httptest.NewRecorder()
	recvTraceroute(w, req)

	resp = w.Result()
	body, _ = ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusCreated {
		t.Errorf("expected status %d, got %v: %s", http.StatusCreated, resp.StatusCode, body)
	}

	req, err = http.NewRequest("GET", ts.URL+"/traceroute/192.0.2.1", nil)
	if err != nil {
		t.Fatal(err)
	}

	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	body, _ = ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status %d, got %v: %s", http.StatusOK, resp.StatusCode, body)
	}

	if string(body) != route {
		t.Errorf("expect %q, got %q", route, string(body))
	}
}
<|endoftext|>


<commit_before>package recurrent

import "time"

type Scheduler struct {
	target   func()
	interval time.Duration
	quit     chan bool
	signal   chan bool
}

func NewScheduler(interval time.Duration, target func()) *Scheduler {
	return newScheduler(interval, target, func(sched *Scheduler) {
		c := make(chan bool)
		q := make(chan bool)

		go func() {
			for {
				select {
				case <-q:
					break

				case c <- true:
				}
			}

			close(c)
		}()

		sched.run(c)
		q <- true
	})
}

func NewThrottledScheduler(interval time.Duration, minInterval time.Duration, target func()) *Scheduler {
	return newScheduler(interval, target, func(sched *Scheduler) {
		tick := time.NewTicker(minInterval)
		defer tick.Stop()

		sched.run(convert(tick.C))
	})
}

func (sched *Scheduler) Stop() {
	sched.quit <- true
}

func (sched *Scheduler) Signal() {
	select {
	case sched.signal <- true:

	default:
	}
}

func newScheduler(interval time.Duration, target func(), init func(*Scheduler)) *Scheduler {
	sched := &Scheduler{
		target:   target,
		interval: interval,
		quit:     make(chan bool),
		signal:   make(chan bool, 1),
	}

	go init(sched)
	return sched
}

func (sched *Scheduler) run(c chan bool) {
	t := throttle(c, sched.signal)

	for {
		select {
		case <-time.After(sched.interval):
			sched.Signal()

		case <-t:
			sched.target()

		case <-sched.quit:
			break
		}
	}
}

func convert(a <-chan time.Time) chan bool {
	c := make(chan bool)

	go func() {
		for _ = range a {
			c <- true
		}

		close(c)
	}()

	return c
}

func throttle(a chan bool, b chan bool) chan bool {
	c := make(chan bool)

	go func() {
		for _ = range a {
			c <- <-b
		}

		close(c)
	}()

	return c
}
<commit_msg>Add godocs.<commit_after>package recurrent

import "time"

type Scheduler struct {
	target   func()
	interval time.Duration
	quit     chan bool
	signal   chan bool
}

// Create a new scheduler which periodically executes the given function.
// This scheduler can be signaled to execute the function immediately as
// often as the user likes.
func NewScheduler(interval time.Duration, target func()) *Scheduler {
	return newScheduler(interval, target, func(sched *Scheduler) {
		c := make(chan bool)
		q := make(chan bool)

		go func() {
			for {
				select {
				case <-q:
					break

				case c <- true:
				}
			}

			close(c)
		}()

		sched.run(c)
		q <- true
	})
}

// Create a new scheduler which periodically executes the given function.
// The minInterval parameter specifies the frequency at which the signals
// for immediate execution are allowed. Any additional signals within the
// interval are ignored.
func NewThrottledScheduler(interval time.Duration, minInterval time.Duration, target func()) *Scheduler {
	return newScheduler(interval, target, func(sched *Scheduler) {
		tick := time.NewTicker(minInterval)
		defer tick.Stop()

		sched.run(convert(tick.C))
	})
}

// Stop the scheduler. No additional signals are meaningful.
func (sched *Scheduler) Stop() {
	sched.quit <- true
}

// Signal the scheduler to execute the function immediately. This method is
// always non-blocking, and may be ignored depending on if the scheduler is
// throttling signals or not.
func (sched *Scheduler) Signal() {
	select {
	case sched.signal <- true:

	default:
	}
}

func newScheduler(interval time.Duration, target func(), init func(*Scheduler)) *Scheduler {
	sched := &Scheduler{
		target:   target,
		interval: interval,
		quit:     make(chan bool),
		signal:   make(chan bool, 1),
	}

	go init(sched)
	return sched
}

func (sched *Scheduler) run(c chan bool) {
	t := throttle(c, sched.signal)

	for {
		select {
		case <-time.After(sched.interval):
			sched.Signal()

		case <-t:
			sched.target()

		case <-sched.quit:
			break
		}
	}
}

func convert(a <-chan time.Time) chan bool {
	c := make(chan bool)

	go func() {
		for _ = range a {
			c <- true
		}

		close(c)
	}()

	return c
}

func throttle(a chan bool, b chan bool) chan bool {
	c := make(chan bool)

	go func() {
		for _ = range a {
			c <- <-b
		}

		close(c)
	}()

	return c
}
<|endoftext|>


<commit_before>package goriak

import (
	"reflect"
	"testing"
)

func TestSetJSON(t *testing.T) {

	data := map[string]string{
		"Key": "Value",
		"AA":  "BB",
	}

	res, err := Bucket("json", "default").SetJSON(data).Run(con())

	if err != nil {
		t.Error(err)
	}

	var v map[string]string
	res, err = Bucket("json", "default").Key(res.Key).GetJSON(&v).Run(con())

	if err != nil {
		t.Error(err)
	}

	if !reflect.DeepEqual(data, v) {
		t.Error("Not equal")
	}
}

func TestSetJSONWithIndexes(t *testing.T) {
	type testType struct {
		Username string `goriakindex:"username_bin"`
		Name     string
	}

	val := testType{
		Username: "zegl",
		Name:     "Gustav",
	}

	setresult, err := Bucket("json", "default").SetJSON(val).Run(con())

	if err != nil {
		t.Error(err)
		return
	}

	foundCount := 0
	foundCorrent := false

	cb := func(key SecondaryIndexQueryResult) {
		if !key.IsComplete {
			foundCount++
		}

		if key.Key == setresult.Key {
			foundCorrent = true
		}

	}

	Bucket("json", "default").KeysInIndex("username_bin", "zegl", cb).Run(con())

	if foundCount != 1 {
		t.Error("Expected to find 1 item, found ", foundCount)
	}

	if !foundCorrent {
		t.Error("Did not find the correct item")
	}
}

func TestJSONWithIndexLimits(t *testing.T) {
	type testType struct {
		User string
		Age  string `goriakindex:"age_bin"`
	}

	users := []testType{
		{"A", "10"},
		{"B", "10"},
		{"C", "10"},
		{"D", "10"},
		{"E", "13"},
		{"F", "13"},
		{"G", "13"},
		{"H", "13"},
	}

	for _, u := range users {
		_, err := Bucket("json", "default").SetJSON(u).Run(con())

		if err != nil {
			t.Error(err)
			return
		}
	}

	foundCount := 0

	cb := func(key SecondaryIndexQueryResult) {
		if !key.IsComplete {
			foundCount++
		}
	}

	// With limit
	Bucket("json", "default").Limit(2).KeysInIndex("age_bin", "10", cb).Run(con())

	if foundCount != 2 {
		t.Error("Expected 2 results, got: ", foundCount)
	}

	foundCount = 0

	// Unlimited
	Bucket("json", "default").KeysInIndex("age_bin", "10", cb).Run(con())

	if foundCount != 4 {
		t.Error("Expected 4 results, got: ", foundCount)
	}
}

func TestJSONSetIndex(t *testing.T) {
	type testType struct {
		User string
		Age  string
	}

	users := []testType{
		{"A", "10"},
		{"B", "10"},
		{"C", "10"},
		{"D", "10"},
		{"E", "13"},
		{"F", "13"},
		{"G", "13"},
		{"H", "13"},
	}

	for _, u := range users {
		_, err := Bucket("json", "default").
			AddToIndex("ageC_bin", u.Age).
			SetJSON(u).
			Run(con())

		if err != nil {
			t.Error(err)
			return
		}
	}

	foundCount := 0

	cb := func(key SecondaryIndexQueryResult) {
		if !key.IsComplete {
			foundCount++
		}
	}

	// With limit
	Bucket("json", "default").Limit(2).KeysInIndex("ageC_bin", "10", cb).Run(con())

	if foundCount != 2 {
		t.Error("Expected 2 results, got: ", foundCount)
	}

	foundCount = 0

	// Unlimited
	Bucket("json", "default").KeysInIndex("ageC_bin", "10", cb).Run(con())

	if foundCount != 4 {
		t.Error("Expected 4 results, got: ", foundCount)
	}
}
<commit_msg>Added more tests to AddToIndex() and SetJSON()<commit_after>package goriak

import (
	"reflect"
	"testing"
)

func TestSetJSON(t *testing.T) {

	data := map[string]string{
		"Key": "Value",
		"AA":  "BB",
	}

	res, err := Bucket("json", "default").SetJSON(data).Run(con())

	if err != nil {
		t.Error(err)
	}

	var v map[string]string
	res, err = Bucket("json", "default").Key(res.Key).GetJSON(&v).Run(con())

	if err != nil {
		t.Error(err)
	}

	if !reflect.DeepEqual(data, v) {
		t.Error("Not equal")
	}
}

func TestSetJSONWithIndexes(t *testing.T) {
	type testType struct {
		Username string `goriakindex:"username_bin"`
		Name     string
	}

	val := testType{
		Username: "zegl",
		Name:     "Gustav",
	}

	setresult, err := Bucket("json", "default").SetJSON(val).Run(con())

	if err != nil {
		t.Error(err)
		return
	}

	foundCount := 0
	foundCorrent := false

	cb := func(key SecondaryIndexQueryResult) {
		if !key.IsComplete {
			foundCount++
		}

		if key.Key == setresult.Key {
			foundCorrent = true
		}

	}

	Bucket("json", "default").KeysInIndex("username_bin", "zegl", cb).Run(con())

	if foundCount != 1 {
		t.Error("Expected to find 1 item, found ", foundCount)
	}

	if !foundCorrent {
		t.Error("Did not find the correct item")
	}
}

func TestJSONWithIndexLimits(t *testing.T) {
	type testType struct {
		User string
		Age  string `goriakindex:"age_bin"`
	}

	users := []testType{
		{"A", "10"},
		{"B", "10"},
		{"C", "10"},
		{"D", "10"},
		{"E", "13"},
		{"F", "13"},
		{"G", "13"},
		{"H", "13"},
	}

	for _, u := range users {
		_, err := Bucket("json", "default").SetJSON(u).Run(con())

		if err != nil {
			t.Error(err)
			return
		}
	}

	foundCount := 0

	cb := func(key SecondaryIndexQueryResult) {
		if !key.IsComplete {
			foundCount++
		}
	}

	// With limit
	Bucket("json", "default").Limit(2).KeysInIndex("age_bin", "10", cb).Run(con())

	if foundCount != 2 {
		t.Error("Expected 2 results, got: ", foundCount)
	}

	foundCount = 0

	// Unlimited
	Bucket("json", "default").KeysInIndex("age_bin", "10", cb).Run(con())

	if foundCount != 4 {
		t.Error("Expected 4 results, got: ", foundCount)
	}
}

func TestJSONSetIndex(t *testing.T) {
	type testType struct {
		User string
		Age  string
	}

	users := []testType{
		{"A", "10"},
		{"B", "10"},
		{"C", "10"},
		{"D", "10"},
		{"E", "13"},
		{"F", "13"},
		{"G", "13"},
		{"H", "13"},
	}

	for _, u := range users {
		_, err := Bucket("json", "default").
			AddToIndex("ageC_bin", u.Age).
			SetJSON(u).
			Run(con())

		if err != nil {
			t.Error(err)
			return
		}
	}

	foundCount := 0

	cb := func(key SecondaryIndexQueryResult) {
		if !key.IsComplete {
			foundCount++
		}
	}

	// With limit
	Bucket("json", "default").Limit(2).KeysInIndex("ageC_bin", "10", cb).Run(con())

	if foundCount != 2 {
		t.Error("Expected 2 results, got: ", foundCount)
	}

	foundCount = 0

	// Unlimited
	Bucket("json", "default").KeysInIndex("ageC_bin", "10", cb).Run(con())

	if foundCount != 4 {
		t.Error("Expected 4 results, got: ", foundCount)
	}
}

func TestJSONWithSliceIndex(t *testing.T) {
	type testType struct {
		User string
		Ages []string `goriakindex:"ageslice_bin"`
	}

	users := []testType{
		{"A", []string{"10", "11"}},
		{"B", []string{"10", "11"}},
		{"C", []string{"10", "11"}},
		{"D", []string{"10", "11"}},
		{"E", []string{"13", "11"}},
		{"F", []string{"13", "11"}},
		{"G", []string{"13", "11"}},
		{"H", []string{"13", "11"}},
	}

	for _, u := range users {
		_, err := Bucket("json", "default").SetJSON(u).Run(con())

		if err != nil {
			t.Error(err)
			return
		}
	}

	foundCount := 0

	cb := func(key SecondaryIndexQueryResult) {
		if !key.IsComplete {
			foundCount++
		}
	}

	// With limit
	Bucket("json", "default").Limit(2).KeysInIndex("ageslice_bin", "10", cb).Run(con())

	if foundCount != 2 {
		t.Error("Expected 2 results, got: ", foundCount)
	}

	foundCount = 0

	// Unlimited
	Bucket("json", "default").KeysInIndex("ageslice_bin", "10", cb).Run(con())

	if foundCount != 4 {
		t.Error("Expected 4 results, got: ", foundCount)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

// waitForCmd launches a goroutine to wait on the supplied
// WaitGroup. A channel is returned which signals the completion of
// the wait.
func waitForCmd(wg *sync.WaitGroup) <-chan struct{} {
	cmdDone := make(chan struct{})
	go func() {
		wg.Wait()
		close(cmdDone)
	}()
	return cmdDone
}

// testCmdDone waits for the cmdDone channel to be closed for at most
// the specified wait duration. Returns true if the command finished in
// the allotted time, false otherwise.
func testCmdDone(cmdDone <-chan struct{}, wait time.Duration) bool {
	select {
	case <-cmdDone:
		return true
	case <-time.After(wait):
		return false
	}
}

func TestCommandQueue(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg := sync.WaitGroup{}

	// Try a command with no overlapping already-running commands.
	cq.GetWait(proto.Key("a"), nil, false, &wg)
	wg.Wait()
	cq.GetWait(proto.Key("a"), proto.Key("b"), false, &wg)
	wg.Wait()

	// Add a command and verify wait group is returned.
	wk := cq.Add(proto.Key("a"), nil, false)
	cq.GetWait(proto.Key("a"), nil, false, &wg)
	cmdDone := waitForCmd(&wg)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with command outstanding")
	}
	cq.Remove(wk)
	if !testCmdDone(cmdDone, 5*time.Millisecond) {
		t.Fatal("command should finish with no commands outstanding")
	}
}

func TestCommandQueueNoWaitOnReadOnly(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg := sync.WaitGroup{}
	// Add a read-only command.
	wk := cq.Add(proto.Key("a"), nil, true)
	// Verify no wait on another read-only command.
	cq.GetWait(proto.Key("a"), nil, true, &wg)
	wg.Wait()
	// Verify wait with a read-write command.
	cq.GetWait(proto.Key("a"), nil, false, &wg)
	cmdDone := waitForCmd(&wg)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with command outstanding")
	}
	cq.Remove(wk)
	if !testCmdDone(cmdDone, 5*time.Millisecond) {
		t.Fatal("command should finish with no commands outstanding")
	}
}

func TestCommandQueueMultipleExecutingCommands(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg := sync.WaitGroup{}

	// Add multiple commands and add a command which overlaps them all.
	wk1 := cq.Add(proto.Key("a"), nil, false)
	wk2 := cq.Add(proto.Key("b"), proto.Key("c"), false)
	wk3 := cq.Add(proto.Key("0"), proto.Key("d"), false)
	cq.GetWait(proto.Key("a"), proto.Key("cc"), false, &wg)
	cmdDone := waitForCmd(&wg)
	cq.Remove(wk1)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with two commands outstanding")
	}
	cq.Remove(wk2)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with one command outstanding")
	}
	cq.Remove(wk3)
	if !testCmdDone(cmdDone, 5*time.Millisecond) {
		t.Fatal("command should finish with no commands outstanding")
	}
}

func TestCommandQueueMultiplePendingCommands(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg1 := sync.WaitGroup{}
	wg2 := sync.WaitGroup{}
	wg3 := sync.WaitGroup{}

	// Add a command which will overlap all commands.
	wk := cq.Add(proto.Key("a"), proto.Key("d"), false)
	cq.GetWait(proto.Key("a"), nil, false, &wg1)
	cq.GetWait(proto.Key("b"), nil, false, &wg2)
	cq.GetWait(proto.Key("c"), nil, false, &wg3)
	cmdDone1 := waitForCmd(&wg1)
	cmdDone2 := waitForCmd(&wg2)
	cmdDone3 := waitForCmd(&wg3)

	if testCmdDone(cmdDone1, 1*time.Millisecond) ||
		testCmdDone(cmdDone2, 1*time.Millisecond) ||
		testCmdDone(cmdDone3, 1*time.Millisecond) {
		t.Fatal("no commands should finish with command outstanding")
	}
	cq.Remove(wk)
	if !testCmdDone(cmdDone1, 5*time.Millisecond) ||
		!testCmdDone(cmdDone2, 5*time.Millisecond) ||
		!testCmdDone(cmdDone3, 5*time.Millisecond) {
		t.Fatal("commands should finish with no commands outstanding")
	}
}

func TestCommandQueueClear(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg1 := sync.WaitGroup{}
	wg2 := sync.WaitGroup{}

	// Add multiple commands and commands which access each.
	cq.Add(proto.Key("a"), nil, false)
	cq.Add(proto.Key("b"), nil, false)
	cq.GetWait(proto.Key("a"), nil, false, &wg1)
	cq.GetWait(proto.Key("b"), nil, false, &wg2)
	cmdDone1 := waitForCmd(&wg1)
	cmdDone2 := waitForCmd(&wg2)

	// Clear the queue and verify both commands are signaled.
	cq.Clear()

	if !testCmdDone(cmdDone1, 100*time.Millisecond) ||
		!testCmdDone(cmdDone2, 100*time.Millisecond) {
		t.Fatal("commands should finish when clearing queue")
	}
}
<commit_msg>Add a test verifying that an end key is treated as exclusive end in the command queue<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

// waitForCmd launches a goroutine to wait on the supplied
// WaitGroup. A channel is returned which signals the completion of
// the wait.
func waitForCmd(wg *sync.WaitGroup) <-chan struct{} {
	cmdDone := make(chan struct{})
	go func() {
		wg.Wait()
		close(cmdDone)
	}()
	return cmdDone
}

// testCmdDone waits for the cmdDone channel to be closed for at most
// the specified wait duration. Returns true if the command finished in
// the allotted time, false otherwise.
func testCmdDone(cmdDone <-chan struct{}, wait time.Duration) bool {
	select {
	case <-cmdDone:
		return true
	case <-time.After(wait):
		return false
	}
}

func TestCommandQueue(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg := sync.WaitGroup{}

	// Try a command with no overlapping already-running commands.
	cq.GetWait(proto.Key("a"), nil, false, &wg)
	wg.Wait()
	cq.GetWait(proto.Key("a"), proto.Key("b"), false, &wg)
	wg.Wait()

	// Add a command and verify wait group is returned.
	wk := cq.Add(proto.Key("a"), nil, false)
	cq.GetWait(proto.Key("a"), nil, false, &wg)
	cmdDone := waitForCmd(&wg)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with command outstanding")
	}
	cq.Remove(wk)
	if !testCmdDone(cmdDone, 5*time.Millisecond) {
		t.Fatal("command should finish with no commands outstanding")
	}
}

func TestCommandQueueNoWaitOnReadOnly(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg := sync.WaitGroup{}
	// Add a read-only command.
	wk := cq.Add(proto.Key("a"), nil, true)
	// Verify no wait on another read-only command.
	cq.GetWait(proto.Key("a"), nil, true, &wg)
	wg.Wait()
	// Verify wait with a read-write command.
	cq.GetWait(proto.Key("a"), nil, false, &wg)
	cmdDone := waitForCmd(&wg)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with command outstanding")
	}
	cq.Remove(wk)
	if !testCmdDone(cmdDone, 5*time.Millisecond) {
		t.Fatal("command should finish with no commands outstanding")
	}
}

func TestCommandQueueMultipleExecutingCommands(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg := sync.WaitGroup{}

	// Add multiple commands and add a command which overlaps them all.
	wk1 := cq.Add(proto.Key("a"), nil, false)
	wk2 := cq.Add(proto.Key("b"), proto.Key("c"), false)
	wk3 := cq.Add(proto.Key("0"), proto.Key("d"), false)
	cq.GetWait(proto.Key("a"), proto.Key("cc"), false, &wg)
	cmdDone := waitForCmd(&wg)
	cq.Remove(wk1)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with two commands outstanding")
	}
	cq.Remove(wk2)
	if testCmdDone(cmdDone, 1*time.Millisecond) {
		t.Fatal("command should not finish with one command outstanding")
	}
	cq.Remove(wk3)
	if !testCmdDone(cmdDone, 5*time.Millisecond) {
		t.Fatal("command should finish with no commands outstanding")
	}
}

func TestCommandQueueMultiplePendingCommands(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg1 := sync.WaitGroup{}
	wg2 := sync.WaitGroup{}
	wg3 := sync.WaitGroup{}

	// Add a command which will overlap all commands.
	wk := cq.Add(proto.Key("a"), proto.Key("d"), false)
	cq.GetWait(proto.Key("a"), nil, false, &wg1)
	cq.GetWait(proto.Key("b"), nil, false, &wg2)
	cq.GetWait(proto.Key("c"), nil, false, &wg3)
	cmdDone1 := waitForCmd(&wg1)
	cmdDone2 := waitForCmd(&wg2)
	cmdDone3 := waitForCmd(&wg3)

	if testCmdDone(cmdDone1, 1*time.Millisecond) ||
		testCmdDone(cmdDone2, 1*time.Millisecond) ||
		testCmdDone(cmdDone3, 1*time.Millisecond) {
		t.Fatal("no commands should finish with command outstanding")
	}
	cq.Remove(wk)
	if !testCmdDone(cmdDone1, 5*time.Millisecond) ||
		!testCmdDone(cmdDone2, 5*time.Millisecond) ||
		!testCmdDone(cmdDone3, 5*time.Millisecond) {
		t.Fatal("commands should finish with no commands outstanding")
	}
}

func TestCommandQueueClear(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	wg1 := sync.WaitGroup{}
	wg2 := sync.WaitGroup{}

	// Add multiple commands and commands which access each.
	cq.Add(proto.Key("a"), nil, false)
	cq.Add(proto.Key("b"), nil, false)
	cq.GetWait(proto.Key("a"), nil, false, &wg1)
	cq.GetWait(proto.Key("b"), nil, false, &wg2)
	cmdDone1 := waitForCmd(&wg1)
	cmdDone2 := waitForCmd(&wg2)

	// Clear the queue and verify both commands are signaled.
	cq.Clear()

	if !testCmdDone(cmdDone1, 100*time.Millisecond) ||
		!testCmdDone(cmdDone2, 100*time.Millisecond) {
		t.Fatal("commands should finish when clearing queue")
	}
}

// TestCommandQueueExclusiveEnd verifies that an end key is treated as
// an exclusive end when GetWait calculates overlapping commands. Test
// it by calling GetWait with a command whose start key is equal to
// the end key of a previous command.
func TestCommandQueueExclusiveEnd(t *testing.T) {
	defer leaktest.AfterTest(t)
	cq := NewCommandQueue()
	cq.Add(proto.Key("a"), proto.Key("b"), false)

	wg := sync.WaitGroup{}
	cq.GetWait(proto.Key("b"), nil, false, &wg)
	// Verify no wait on the second writer command on "b" since
	// it does not overlap with the first command on ["a", "b").
	wg.Wait()
}
<|endoftext|>


<commit_before>package model

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"

	"github.com/oinume/lekcije/backend/domain/config"
	"github.com/oinume/lekcije/backend/errors"
)

const MaxFollowTeacherCount = 20

type FollowingTeacher struct {
	UserID    uint32 `gorm:"primary_key"`
	TeacherID uint32 `gorm:"primary_key"`
	CreatedAt time.Time
	UpdatedAt time.Time
}

func (*FollowingTeacher) TableName() string {
	return "following_teacher"
}

type FollowingTeacherService struct {
	db *gorm.DB
}

func NewFollowingTeacherService(db *gorm.DB) *FollowingTeacherService {
	return &FollowingTeacherService{db: db}
}

func (s *FollowingTeacherService) TableName() string {
	return (&FollowingTeacher{}).TableName()
}

func (s *FollowingTeacherService) FindTeachersByUserID(userID uint32) ([]*Teacher, error) { // TODO: delete
	limit := 100
	values := make([]*Teacher, 0, limit)
	sql := fmt.Sprintf(`
	SELECT t.* FROM teacher AS t
	INNER JOIN following_teacher AS ft ON t.id = ft.teacher_id
	WHERE ft.user_id = ?
	ORDER BY ft.updated_at DESC, t.id ASC
	LIMIT %d
	`, limit) // TODO: OFFSET

	if result := s.db.Raw(strings.TrimSpace(sql), userID).Scan(&values); result.Error != nil {
		if result.RecordNotFound() {
			return values, nil
		}
		return values, errors.NewInternalError(
			errors.WithError(result.Error),
		)
	}
	return values, nil
}

func (s *FollowingTeacherService) FindTeacherIDs() ([]uint32, error) {
	values := make([]*FollowingTeacher, 0, 5000)
	sql := `SELECT teacher_id FROM following_teacher LIMIT 5000`
	if result := s.db.Raw(sql).Scan(&values); result.Error != nil {
		if result.RecordNotFound() {
			return nil, nil
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessagef("failed to select teacher ids"),
		)
	}
	ids := make([]uint32, len(values))
	for i, t := range values {
		ids[i] = t.TeacherID
	}
	return ids, nil
}

func (s *FollowingTeacherService) FindTeacherIDsByUserID(
	ctx context.Context,
	userID uint32,
	fetchErrorCount int,
	lastLessonAt time.Time,
) ([]uint32, error) {
	_, span := otel.Tracer(config.DefaultTracerName).Start(ctx, "FollowingTeacherService.FindTeacherIDsByUserID")
	span.SetAttributes(attribute.KeyValue{
		Key:   "userID",
		Value: attribute.Int64Value(int64(userID)),
	})
	defer span.End()

	values := make([]*FollowingTeacher, 0, 1000)
	sql := `
	SELECT ft.teacher_id FROM following_teacher AS ft
	INNER JOIN teacher AS t ON ft.teacher_id = t.id
	WHERE
      user_id = ?
      AND t.fetch_error_count <= ?
      AND t.last_lesson_at >= ?
	`
	sql = strings.TrimSpace(sql)
	if result := s.db.Raw(sql, userID, fetchErrorCount, lastLessonAt.Format(dbDatetimeFormat)).Scan(&values); result.Error != nil {
		if result.RecordNotFound() {
			return nil, nil
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
		)
	}
	ids := make([]uint32, len(values))
	for i, t := range values {
		ids[i] = t.TeacherID
	}
	return ids, nil
}

func (s *FollowingTeacherService) CountFollowingTeachersByUserID(userID uint32) (int, error) {
	count := struct {
		Count int
	}{}
	sql := `SELECT COUNT(*) AS count FROM following_teacher WHERE user_id = ?`
	if err := s.db.Raw(sql, userID).Scan(&count).Error; err != nil {
		return 0, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("count failed"),
			errors.WithResource(errors.NewResource("following_teacher", "userID", userID)),
		)
	}
	return count.Count, nil
}

func (s FollowingTeacherService) ReachesFollowingTeacherLimit(userID uint32, additionalTeachers int) (bool, error) {
	count, err := s.CountFollowingTeachersByUserID(userID)
	if err != nil {
		return false, err
	}
	return count+additionalTeachers > MaxFollowTeacherCount, nil // TODO: Refer plan's limit
}

func (s *FollowingTeacherService) FollowTeacher(
	userID uint32, teacher *Teacher, timestamp time.Time,
) (*FollowingTeacher, error) {
	// Create teacher at first
	teacher.CreatedAt = timestamp
	teacher.UpdatedAt = timestamp
	teacherService := NewTeacherService(s.db)
	if err := teacherService.CreateOrUpdate(teacher); err != nil {
		return nil, err
	}

	ft := &FollowingTeacher{
		UserID:    userID,
		TeacherID: teacher.ID,
		CreatedAt: timestamp,
		UpdatedAt: timestamp,
	}
	if err := s.db.FirstOrCreate(ft).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithResource(
				errors.NewResourceWithEntries(ft.TableName(), []errors.ResourceEntry{
					{Key: "userID", Value: userID},
					{Key: "teacherID", Value: teacher.ID},
				}),
			),
		)
	}
	return ft, nil
}

func (s *FollowingTeacherService) DeleteTeachersByUserIDAndTeacherIDs(
	userID uint32,
	teacherIDs []uint32,
) (int, error) {
	if len(teacherIDs) == 0 {
		return 0, nil
	}

	placeholder := strings.TrimRight(strings.Repeat("?,", len(teacherIDs)), ",")
	sql := fmt.Sprintf("DELETE FROM %s WHERE user_id = ? AND teacher_id IN (%s)", s.TableName(), placeholder)
	args := []interface{}{userID}
	for _, teacherID := range teacherIDs {
		args = append(args, teacherID)
	}
	if result := s.db.Exec(sql, args...); result.Error != nil {
		return 0, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to delete following_teacher"),
			errors.WithResource(errors.NewResourceWithEntries(s.TableName(), []errors.ResourceEntry{
				{Key: "userID", Value: userID},
				{Key: "teacherIDs", Value: teacherIDs},
			})),
		)
	} else {
		return int(result.RowsAffected), nil
	}
}
<commit_msg>Add TODO comment<commit_after>package model

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"

	"github.com/oinume/lekcije/backend/domain/config"
	"github.com/oinume/lekcije/backend/errors"
)

const MaxFollowTeacherCount = 20

type FollowingTeacher struct {
	UserID    uint32 `gorm:"primary_key"`
	TeacherID uint32 `gorm:"primary_key"`
	CreatedAt time.Time
	UpdatedAt time.Time
}

func (*FollowingTeacher) TableName() string {
	return "following_teacher"
}

type FollowingTeacherService struct {
	db *gorm.DB
}

func NewFollowingTeacherService(db *gorm.DB) *FollowingTeacherService {
	return &FollowingTeacherService{db: db}
}

func (s *FollowingTeacherService) TableName() string {
	return (&FollowingTeacher{}).TableName()
}

func (s *FollowingTeacherService) FindTeachersByUserID(userID uint32) ([]*Teacher, error) { // TODO: delete
	limit := 100
	values := make([]*Teacher, 0, limit)
	sql := fmt.Sprintf(`
	SELECT t.* FROM teacher AS t
	INNER JOIN following_teacher AS ft ON t.id = ft.teacher_id
	WHERE ft.user_id = ?
	ORDER BY ft.updated_at DESC, t.id ASC
	LIMIT %d
	`, limit) // TODO: OFFSET

	if result := s.db.Raw(strings.TrimSpace(sql), userID).Scan(&values); result.Error != nil {
		if result.RecordNotFound() {
			return values, nil
		}
		return values, errors.NewInternalError(
			errors.WithError(result.Error),
		)
	}
	return values, nil
}

func (s *FollowingTeacherService) FindTeacherIDs() ([]uint32, error) {
	values := make([]*FollowingTeacher, 0, 5000)
	sql := `SELECT teacher_id FROM following_teacher LIMIT 5000`
	if result := s.db.Raw(sql).Scan(&values); result.Error != nil {
		if result.RecordNotFound() {
			return nil, nil
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessagef("failed to select teacher ids"),
		)
	}
	ids := make([]uint32, len(values))
	for i, t := range values {
		ids[i] = t.TeacherID
	}
	return ids, nil
}

func (s *FollowingTeacherService) FindTeacherIDsByUserID(
	ctx context.Context,
	userID uint32,
	fetchErrorCount int,
	lastLessonAt time.Time,
) ([]uint32, error) {
	_, span := otel.Tracer(config.DefaultTracerName).Start(ctx, "FollowingTeacherService.FindTeacherIDsByUserID")
	span.SetAttributes(attribute.KeyValue{
		Key:   "userID",
		Value: attribute.Int64Value(int64(userID)),
	})
	defer span.End()

	values := make([]*FollowingTeacher, 0, 1000)
	sql := `
	SELECT ft.teacher_id FROM following_teacher AS ft
	INNER JOIN teacher AS t ON ft.teacher_id = t.id
	WHERE
      user_id = ?
      AND t.fetch_error_count <= ?
      AND t.last_lesson_at >= ?
	`
	sql = strings.TrimSpace(sql)
	if result := s.db.Raw(sql, userID, fetchErrorCount, lastLessonAt.Format(dbDatetimeFormat)).Scan(&values); result.Error != nil {
		if result.RecordNotFound() {
			return nil, nil
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
		)
	}
	ids := make([]uint32, len(values))
	for i, t := range values {
		ids[i] = t.TeacherID
	}
	return ids, nil
}

func (s *FollowingTeacherService) CountFollowingTeachersByUserID(userID uint32) (int, error) {
	count := struct {
		Count int
	}{}
	sql := `SELECT COUNT(*) AS count FROM following_teacher WHERE user_id = ?`
	if err := s.db.Raw(sql, userID).Scan(&count).Error; err != nil {
		return 0, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("count failed"),
			errors.WithResource(errors.NewResource("following_teacher", "userID", userID)),
		)
	}
	return count.Count, nil
}

func (s FollowingTeacherService) ReachesFollowingTeacherLimit(userID uint32, additionalTeachers int) (bool, error) {
	count, err := s.CountFollowingTeachersByUserID(userID)
	if err != nil {
		return false, err
	}
	return count+additionalTeachers > MaxFollowTeacherCount, nil // TODO: Refer plan's limit
}

func (s *FollowingTeacherService) FollowTeacher( // TODO: delete no longer used
	userID uint32, teacher *Teacher, timestamp time.Time,
) (*FollowingTeacher, error) {
	// Create teacher at first
	teacher.CreatedAt = timestamp
	teacher.UpdatedAt = timestamp
	teacherService := NewTeacherService(s.db)
	if err := teacherService.CreateOrUpdate(teacher); err != nil {
		return nil, err
	}

	ft := &FollowingTeacher{
		UserID:    userID,
		TeacherID: teacher.ID,
		CreatedAt: timestamp,
		UpdatedAt: timestamp,
	}
	if err := s.db.FirstOrCreate(ft).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithResource(
				errors.NewResourceWithEntries(ft.TableName(), []errors.ResourceEntry{
					{Key: "userID", Value: userID},
					{Key: "teacherID", Value: teacher.ID},
				}),
			),
		)
	}
	return ft, nil
}

func (s *FollowingTeacherService) DeleteTeachersByUserIDAndTeacherIDs(
	userID uint32,
	teacherIDs []uint32,
) (int, error) {
	if len(teacherIDs) == 0 {
		return 0, nil
	}

	placeholder := strings.TrimRight(strings.Repeat("?,", len(teacherIDs)), ",")
	sql := fmt.Sprintf("DELETE FROM %s WHERE user_id = ? AND teacher_id IN (%s)", s.TableName(), placeholder)
	args := []interface{}{userID}
	for _, teacherID := range teacherIDs {
		args = append(args, teacherID)
	}
	if result := s.db.Exec(sql, args...); result.Error != nil {
		return 0, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to delete following_teacher"),
			errors.WithResource(errors.NewResourceWithEntries(s.TableName(), []errors.ResourceEntry{
				{Key: "userID", Value: userID},
				{Key: "teacherIDs", Value: teacherIDs},
			})),
		)
	} else {
		return int(result.RowsAffected), nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/mat/besticon/besticon"
)

func TestGetIndex(t *testing.T) {
	req, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	indexHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), "<title>The Favicon Finder</title>")
}

func TestGetIcons(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?url=apple.com", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconsHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "max-age=86400", w.Header().Get("Cache-Control"))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), "Icons on apple.com")

	assertStringContains(t, w.Body.String(), "<img src='http://www.apple.com/favicon.ico'")
	assertStringContains(t, w.Body.String(), "")
	assertStringContains(t, w.Body.String(), "32x32	")
}

func TestGetIcon(t *testing.T) {
	req, err := http.NewRequest("GET", "/icon?url=apple.com&size=120", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "max-age=86400", w.Header().Get("Cache-Control"))
	assertStringEquals(t, "http://www.apple.com/apple-touch-icon.png", w.Header().Get("Location"))
}

func TestGetIconWithFallBackURL(t *testing.T) {
	req, err := http.NewRequest("GET", "/icon?url=apple.com&size=400&fallback_icon_url=http%3A%2F%2Fexample.com", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "max-age=86400", w.Header().Get("Cache-Control"))
	assertStringEquals(t, "http://example.com", w.Header().Get("Location"))
}

func TestGetIconWith404Page(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?size=32&url=httpbin.org/status/404", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/lettericons/H-32.png", w.Header().Get("Location"))
}

func TestGet404IconWithFallbackColor(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?size=32&url=httpbin.org/status/404&fallback_icon_color=123456", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/lettericons/H-32-123456.png", w.Header().Get("Location"))
}

func TestGet404IconWithInvalidFallbackColor(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?size=32&url=httpbin.org/status/404&fallback_icon_color=zz", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/lettericons/H-32.png", w.Header().Get("Location"))
}

func TestGetAllIcons(t *testing.T) {
	req, err := http.NewRequest("GET", "/allicons.json?url=apple.com", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	alliconsHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "application/json", w.Header().Get("Content-Type"))
	assertStringEquals(t, "max-age=3600", w.Header().Get("Cache-Control"))

	assertStringContains(t, w.Body.String(), `"url":"http://www.apple.com/favicon.ico"`)
	assertStringContains(t, w.Body.String(), `"width":32`)
	assertStringContains(t, w.Body.String(), `"height":32`)

	// Make sure we don't return inlined image data
	assertDoesNotExceed(t, len(w.Body.String()), 2000)
}

func TestGetPopular(t *testing.T) {
	req, err := http.NewRequest("GET", "/popular", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	popularHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), `Icon Examples`)
	assertStringContains(t, w.Body.String(), `github.com`)
}

func TestGetLetterIcon(t *testing.T) {
	req, err := http.NewRequest("GET", "/lettericons/M-144-EFC25D.png", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	lettericonHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "image/png", w.Header().Get("Content-Type"))
	assertStringEquals(t, "max-age=31536000", w.Header().Get("Cache-Control"))
	assertIntegerInInterval(t, 1500, 1800, w.Body.Len())
}

func TestGetObsoleteApiRedirect(t *testing.T) {
	req, err := http.NewRequest("GET", "/api/icons?url=http%3A%2F%2Fapple.com&i_am_feeling_lucky=yes", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	obsoleteAPIHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/icon?size=120&url=http%3A%2F%2Fapple.com&i_am_feeling_lucky=yes", w.Header().Get("Location"))
}

func TestGet404(t *testing.T) {
	req, err := http.NewRequest("GET", "/does-not-exist", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	indexHandler(w, req)

	assertStringEquals(t, "404", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), "The requested page does not exist :-(")
}

func assertStringContains(t *testing.T, haystack string, needle string) {
	if !strings.Contains(haystack, needle) {
		fail(t, fmt.Sprintf("Expected '%s' to be contained in '%s'", needle, haystack))
	}
}

func assertStringEquals(t *testing.T, expected string, actual string) {
	if expected != actual {
		fail(t, fmt.Sprintf("Expected '%s' to be '%s'", actual, expected))
	}
}

func assertIntegerEquals(t *testing.T, expected int, actual int) {
	if expected != actual {
		fail(t, fmt.Sprintf("Expected %d to be %d", actual, expected))
	}
}

func assertIntegerInInterval(t *testing.T, lower int, upper int, actual int) {
	if actual < lower || actual > upper {
		fail(t, fmt.Sprintf("Expected %d to be in interval [%d,%d]", actual, lower, upper))
	}
}

func assertDoesNotExceed(t *testing.T, actual int, maximum int) {
	if actual >= maximum {
		fail(t, fmt.Sprintf("Expected '%d' to be < '%d'", actual, maximum))
	}
}

func fail(t *testing.T, failureMessage string) {
	t.Errorf("\t%s\n"+
		"\r\t",
		failureMessage)
}

func init() {
	besticon.SetLogOutput(ioutil.Discard)
}
<commit_msg>Update apple favicon tests<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/mat/besticon/besticon"
)

func TestGetIndex(t *testing.T) {
	req, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	indexHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), "<title>The Favicon Finder</title>")
}

func TestGetIcons(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?url=apple.com", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconsHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "max-age=86400", w.Header().Get("Cache-Control"))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), "Icons on apple.com")

	assertStringContains(t, w.Body.String(), "<img src='https://www.apple.com/favicon.ico'")
	assertStringContains(t, w.Body.String(), "")
	assertStringContains(t, w.Body.String(), "32x32	")
}

func TestGetIcon(t *testing.T) {
	req, err := http.NewRequest("GET", "/icon?url=apple.com&size=120", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "max-age=86400", w.Header().Get("Cache-Control"))
	assertStringEquals(t, "https://www.apple.com/apple-touch-icon.png", w.Header().Get("Location"))
}

func TestGetIconWithFallBackURL(t *testing.T) {
	req, err := http.NewRequest("GET", "/icon?url=apple.com&size=400&fallback_icon_url=http%3A%2F%2Fexample.com", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "max-age=86400", w.Header().Get("Cache-Control"))
	assertStringEquals(t, "http://example.com", w.Header().Get("Location"))
}

func TestGetIconWith404Page(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?size=32&url=httpbin.org/status/404", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/lettericons/H-32.png", w.Header().Get("Location"))
}

func TestGet404IconWithFallbackColor(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?size=32&url=httpbin.org/status/404&fallback_icon_color=123456", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/lettericons/H-32-123456.png", w.Header().Get("Location"))
}

func TestGet404IconWithInvalidFallbackColor(t *testing.T) {
	req, err := http.NewRequest("GET", "/icons?size=32&url=httpbin.org/status/404&fallback_icon_color=zz", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	iconHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/lettericons/H-32.png", w.Header().Get("Location"))
}

func TestGetAllIcons(t *testing.T) {
	req, err := http.NewRequest("GET", "/allicons.json?url=apple.com", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	alliconsHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "application/json", w.Header().Get("Content-Type"))
	assertStringEquals(t, "max-age=3600", w.Header().Get("Cache-Control"))

	assertStringContains(t, w.Body.String(), `"url":"https://www.apple.com/favicon.ico"`)
	assertStringContains(t, w.Body.String(), `"width":32`)
	assertStringContains(t, w.Body.String(), `"height":32`)

	// Make sure we don't return inlined image data
	assertDoesNotExceed(t, len(w.Body.String()), 2000)
}

func TestGetPopular(t *testing.T) {
	req, err := http.NewRequest("GET", "/popular", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	popularHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), `Icon Examples`)
	assertStringContains(t, w.Body.String(), `github.com`)
}

func TestGetLetterIcon(t *testing.T) {
	req, err := http.NewRequest("GET", "/lettericons/M-144-EFC25D.png", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	lettericonHandler(w, req)

	assertStringEquals(t, "200", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "image/png", w.Header().Get("Content-Type"))
	assertStringEquals(t, "max-age=31536000", w.Header().Get("Cache-Control"))
	assertIntegerInInterval(t, 1500, 1800, w.Body.Len())
}

func TestGetObsoleteApiRedirect(t *testing.T) {
	req, err := http.NewRequest("GET", "/api/icons?url=http%3A%2F%2Fapple.com&i_am_feeling_lucky=yes", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	obsoleteAPIHandler(w, req)

	assertStringEquals(t, "302", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "/icon?size=120&url=http%3A%2F%2Fapple.com&i_am_feeling_lucky=yes", w.Header().Get("Location"))
}

func TestGet404(t *testing.T) {
	req, err := http.NewRequest("GET", "/does-not-exist", nil)
	if err != nil {
		log.Fatal(err)
	}

	w := httptest.NewRecorder()
	indexHandler(w, req)

	assertStringEquals(t, "404", fmt.Sprintf("%d", w.Code))
	assertStringEquals(t, "text/html; charset=utf-8", w.Header().Get("Content-Type"))

	assertStringContains(t, w.Body.String(), "The requested page does not exist :-(")
}

func assertStringContains(t *testing.T, haystack string, needle string) {
	if !strings.Contains(haystack, needle) {
		fail(t, fmt.Sprintf("Expected '%s' to be contained in '%s'", needle, haystack))
	}
}

func assertStringEquals(t *testing.T, expected string, actual string) {
	if expected != actual {
		fail(t, fmt.Sprintf("Expected '%s' to be '%s'", actual, expected))
	}
}

func assertIntegerEquals(t *testing.T, expected int, actual int) {
	if expected != actual {
		fail(t, fmt.Sprintf("Expected %d to be %d", actual, expected))
	}
}

func assertIntegerInInterval(t *testing.T, lower int, upper int, actual int) {
	if actual < lower || actual > upper {
		fail(t, fmt.Sprintf("Expected %d to be in interval [%d,%d]", actual, lower, upper))
	}
}

func assertDoesNotExceed(t *testing.T, actual int, maximum int) {
	if actual >= maximum {
		fail(t, fmt.Sprintf("Expected '%d' to be < '%d'", actual, maximum))
	}
}

func fail(t *testing.T, failureMessage string) {
	t.Errorf("\t%s\n"+
		"\r\t",
		failureMessage)
}

func init() {
	besticon.SetLogOutput(ioutil.Discard)
}
<|endoftext|>


<commit_before>package sdm630

import (
	"math"
)

type Scheduler interface {
	Produce()
}

type RoundRobinScheduler struct {
	out    QuerySnipChannel
	devids []uint8
}

func NewRoundRobinScheduler(
	out QuerySnipChannel,
	devices []uint8,
) *RoundRobinScheduler {
	return &RoundRobinScheduler{out: out,
		devids: devices,
	}
}

func (s *RoundRobinScheduler) Produce() {
	for {
		for _, devid := range s.devids {
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Voltage,
				Value: math.NaN(), Description: "L1 Voltage (V)", IEC61850: "VolLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Voltage,
				Value: math.NaN(), Description: "L2 Voltage (V)", IEC61850: "VolLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Voltage,
				Value: math.NaN(), Description: "L3 Voltage (V)", IEC61850: "VolLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Current,
				Value: math.NaN(), Description: "L1 Current (A)", IEC61850: "AmpLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Current,
				Value: math.NaN(), Description: "L2 Current (A)", IEC61850: "AmpLocPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Current,
				Value: math.NaN(), Description: "L3 Current (A)", IEC61850: "AmpLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Power, Value: math.NaN(), Description: "L1 Power (W)", IEC61850: "WLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Power, Value: math.NaN(), Description: "L2 Power (W)", IEC61850: "WLocPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Power, Value: math.NaN(), Description: "L3 Power (W)", IEC61850: "WLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Cosphi, Value: math.NaN(), Description: "L1 Cosphi", IEC61850: "AngLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Cosphi, Value: math.NaN(), Description: "L2 Cosphi", IEC61850: "AngLocPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Cosphi, Value: math.NaN(), Description: "L3 Cosphi", IEC61850: "AngLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Import, Value: math.NaN(), Description: "L1 Import (kWh)", IEC61850: "TotkWhImportPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Import, Value: math.NaN(), Description: "L2 Import (kWh)",
				IEC61850: "TotkWhImportPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Import, Value: math.NaN(), Description: "L3 Import (kWh)", IEC61850: "TotkWhImportPhsC"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeTotalImport, Value: math.NaN(), Description: "Total Import (kWh)", IEC61850: "TotkWhImport"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Export, Value: math.NaN(), Description: "L1 Export (kWh)", IEC61850: "TotkWhExportPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Export, Value: math.NaN(), Description: "L2 Export (kWh)", IEC61850: "TotkWhExportPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Export, Value: math.NaN(), Description: "L3 Export (kWh)", IEC61850: "TotkWhExportPhsC"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeTotalExport, Value: math.NaN(), Description: "Total Export (kWh)", IEC61850: "TotkWhExport"}

			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1THDCurrent, Value: math.NaN(), Description: "L1 Current THD (%)", IEC61850: "ThdAPhsA"}
			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2THDCurrent, Value: math.NaN(), Description: "L2 Current THD (%)", IEC61850: "ThdAPhsB"}
			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3THDCurrent, Value: math.NaN(), Description: "L3 Current THD (%)", IEC61850: "ThdAPhsC"}
			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeAvgTHDCurrent, Value: math.NaN(), Description: "Average current to neutral THD (%)", IEC61850: "ThdAmp"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1THDVoltageNeutral, Value: math.NaN(), Description: "L1 Voltage to neutral THD (%)", IEC61850: "ThdVolPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2THDVoltageNeutral, Value: math.NaN(), Description: "L2 Voltage to neutral THD (%)", IEC61850: "ThdVolPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3THDVoltageNeutral, Value: math.NaN(), Description: "L3 Voltage to neutral THD (%)", IEC61850: "ThdVolPhsC"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeAvgTHDVoltageNeutral, Value: math.NaN(), Description: "Average voltage to neutral THD (%)", IEC61850: "ThdVol"}
		}
	}
}
<commit_msg>fixed typo: IEC61850 Voltage labels for L1 and L2 were identical<commit_after>package sdm630

import (
	"math"
)

type Scheduler interface {
	Produce()
}

type RoundRobinScheduler struct {
	out    QuerySnipChannel
	devids []uint8
}

func NewRoundRobinScheduler(
	out QuerySnipChannel,
	devices []uint8,
) *RoundRobinScheduler {
	return &RoundRobinScheduler{out: out,
		devids: devices,
	}
}

func (s *RoundRobinScheduler) Produce() {
	for {
		for _, devid := range s.devids {
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Voltage,
				Value: math.NaN(), Description: "L1 Voltage (V)", IEC61850: "VolLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Voltage,
				Value: math.NaN(), Description: "L2 Voltage (V)", IEC61850: "VolLocPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Voltage,
				Value: math.NaN(), Description: "L3 Voltage (V)", IEC61850: "VolLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Current,
				Value: math.NaN(), Description: "L1 Current (A)", IEC61850: "AmpLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Current,
				Value: math.NaN(), Description: "L2 Current (A)", IEC61850: "AmpLocPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Current,
				Value: math.NaN(), Description: "L3 Current (A)", IEC61850: "AmpLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Power, Value: math.NaN(), Description: "L1 Power (W)", IEC61850: "WLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Power, Value: math.NaN(), Description: "L2 Power (W)", IEC61850: "WLocPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Power, Value: math.NaN(), Description: "L3 Power (W)", IEC61850: "WLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Cosphi, Value: math.NaN(), Description: "L1 Cosphi", IEC61850: "AngLocPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Cosphi, Value: math.NaN(), Description: "L2 Cosphi", IEC61850: "AngLocPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Cosphi, Value: math.NaN(), Description: "L3 Cosphi", IEC61850: "AngLocPhsC"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Import, Value: math.NaN(), Description: "L1 Import (kWh)", IEC61850: "TotkWhImportPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Import, Value: math.NaN(), Description: "L2 Import (kWh)",
				IEC61850: "TotkWhImportPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Import, Value: math.NaN(), Description: "L3 Import (kWh)", IEC61850: "TotkWhImportPhsC"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeTotalImport, Value: math.NaN(), Description: "Total Import (kWh)", IEC61850: "TotkWhImport"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1Export, Value: math.NaN(), Description: "L1 Export (kWh)", IEC61850: "TotkWhExportPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2Export, Value: math.NaN(), Description: "L2 Export (kWh)", IEC61850: "TotkWhExportPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3Export, Value: math.NaN(), Description: "L3 Export (kWh)", IEC61850: "TotkWhExportPhsC"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeTotalExport, Value: math.NaN(), Description: "Total Export (kWh)", IEC61850: "TotkWhExport"}

			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1THDCurrent, Value: math.NaN(), Description: "L1 Current THD (%)", IEC61850: "ThdAPhsA"}
			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2THDCurrent, Value: math.NaN(), Description: "L2 Current THD (%)", IEC61850: "ThdAPhsB"}
			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3THDCurrent, Value: math.NaN(), Description: "L3 Current THD (%)", IEC61850: "ThdAPhsC"}
			//	s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeAvgTHDCurrent, Value: math.NaN(), Description: "Average current to neutral THD (%)", IEC61850: "ThdAmp"}

			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL1THDVoltageNeutral, Value: math.NaN(), Description: "L1 Voltage to neutral THD (%)", IEC61850: "ThdVolPhsA"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL2THDVoltageNeutral, Value: math.NaN(), Description: "L2 Voltage to neutral THD (%)", IEC61850: "ThdVolPhsB"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeL3THDVoltageNeutral, Value: math.NaN(), Description: "L3 Voltage to neutral THD (%)", IEC61850: "ThdVolPhsC"}
			s.out <- QuerySnip{DeviceId: devid, OpCode: OpCodeAvgTHDVoltageNeutral, Value: math.NaN(), Description: "Average voltage to neutral THD (%)", IEC61850: "ThdVol"}
		}
	}
}
<|endoftext|>


<commit_before>package sudoku

import (
	"fmt"
	"log"
	"strconv"
	"strings"
)

//DifficultySignals is a collection of names to float64 values, representing
//the various signals extracted from a SolveDirections, and used for the
//Difficulty calculation. Generally not useful to package users.
type DifficultySignals map[string]float64

//A difficulty signal generator can return more than one difficutly signal, so
//it doesn't just return float64 Each signal generator should always return a
//map with the SAME keys--so if you've called it once you know what the next
//calls will have as keys.
type difficultySignalGenerator func(directions SolveDirections) DifficultySignals

const _DIFFICULTY_WEIGHT_FILENAME = "difficulties.csv"

var difficultySignalGenerators []difficultySignalGenerator

//These are the weights that will be used to turn a list of signals into a
//difficulty. starting weights are set in hs_difficulty_weights.go, which is
//auto-generated. Generate those now: go:generate python util/difficulty-
//convert.py
var difficultySignalWeights map[string]float64

func init() {
	difficultySignalGenerators = []difficultySignalGenerator{
		signalTechnique,
		signalNumberOfSteps,
		signalTechniquePercentage,
		signalPercentageFilledSteps,
		signalNumberUnfilled,
		signalStepsUntilNonFill,
	}
}

//Stats returns a printout of interesting statistics about the
//SolveDirections, including number of steps, difficulty (based on this solve
//description alone), how unrelated the cells in subsequent steps are, and the
//values of all of the signals used to generate the difficulty.
func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	var lastStep *SolveStep
	dissimilarityAccum := 0.0
	for _, step := range self.Steps {
		if lastStep != nil {
			dissimilarityAccum += step.TargetCells.chainDissimilarity(lastStep.TargetCells)
		}
		techniqueCount[step.Technique.Name()] += 1
		lastStep = step
	}
	dissimilarityAccum /= float64(len(self.Steps))

	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	//TODO: we shouldn't even include this... it's not meaningful to report the difficulty of a single solve.
	result = append(result, fmt.Sprintf("Difficulty : %f", self.Signals().difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self.Steps)))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Avg Dissimilarity: %f", dissimilarityAccum))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		//TODO: pad the technique name with enough spaces so the colon lines up.
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

//Description returns a comprehensive prose description of the
//SolveDirections, including reasoning for each step, that if followed would
//lead to the grid being solved. Unlike Walkthrough, Description() does not
//include diagrams for each step.
func (self SolveDirections) Description() []string {

	if len(self.Steps) == 0 {
		return []string{"The puzzle is already solved."}
	}

	var descriptions []string

	//Hints have a special preamble.
	if self.IsHint {
		lastStep := self.Steps[len(self.Steps)-1]
		//TODO: this terminology is too tuned for the Online Sudoku use case.
		//it practice it should probably name the cell in text.
		descriptions = append(descriptions, "Based on the other numbers you've entered, this cell can only be a "+strconv.Itoa(lastStep.TargetNums[0])+".")
		descriptions = append(descriptions, "How do we know that?")
		if len(self.Steps) > 1 {
			descriptions = append(descriptions, "We can't fill any cells right away so first we need to cull some possibilities.")
		}
	}

	for i, step := range self.Steps {
		intro := ""
		description := step.Description()
		if len(self.Steps) > 1 {
			description = strings.ToLower(description)
			switch i {
			case 0:
				intro = "First, "
			case len(self.Steps) - 1:
				intro = "Finally, "
			default:
				//TODO: switch between "then" and "next" randomly.
				intro = "Next, "
			}
		}

		descriptions = append(descriptions, intro+description)

	}
	return descriptions
}

//Walkthrough prints an exhaustive set of human-readable directions that
//includes diagrams at each step to make it easier to follow.
func (self SolveDirections) Walkthrough() string {

	//TODO: test this.

	if len(self.Steps) == 0 {
		return "The puzzle could not be solved with any of the techniques we're aware of."
	}

	clone := self.Grid()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self.Steps))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self.Steps))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self.Steps[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

//Signals returns the DifficultySignals for this set of SolveDirections.
func (self SolveDirections) Signals() DifficultySignals {
	//Because of the contract of a DifficultySignalGenerator (that it always
	//returns the same keys), as long as DifficultySignalGenerators stays
	//constant it's reasonable for callers to assume that one call to
	//Signals() will return all of the string keys you'll see any time you
	//call Signals()
	result := DifficultySignals{}
	for _, generator := range difficultySignalGenerators {
		result.add(generator(self))
	}
	return result
}

//This will overwrite colliding values
//TODO: this is confusingly named
func (self DifficultySignals) add(other DifficultySignals) {
	for key, val := range other {
		self[key] = val
	}
}

//For keys in both, will sum them together.
//TODO: this is confusingly named (compared to Add)  Do we really need both Sum and Add?
func (self DifficultySignals) sum(other DifficultySignals) {
	for key, val := range other {
		self[key] += val
	}
}

func (self DifficultySignals) difficulty() float64 {
	accum := 0.0

	if constant, ok := difficultySignalWeights["Constant"]; ok {
		accum = constant
	} else {
		log.Println("Didn't have the constant term loaded.")
	}

	for signal, val := range self {
		//We can discard the OK because 0 is a reasonable thing to do with weights we aren't aware of.
		weight, _ := difficultySignalWeights[signal]

		accum += val * weight
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

//Rest of file is different Signals

//TODO: now that SolveDirections includes gridSnapshot, think if there are any
//additional Signals we can generate.

//This technique returns a count of how many each type of technique is seen.
//Different techniques are different "difficulties" so seeing more of a hard
//technique will Lead to a higher overall difficulty.
func signalTechnique(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Count"] = 0.0
	}
	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Count"]++
	}
	return result
}

//This signal is just number of steps. More steps is PROBABLY a harder puzzle.
func signalNumberOfSteps(directions SolveDirections) DifficultySignals {
	return DifficultySignals{
		"Number of Steps": float64(len(directions.Steps)),
	}
}

//This signal is like signalTechnique, except it returns the count divided by
//the TOTAL number of steps.
func signalTechniquePercentage(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Percentage"] = 0.0
	}

	count := len(directions.Steps)

	if count == 0 {
		return result
	}

	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Percentage"]++
	}

	//Now normalize all of them
	for name := range result {
		result[name] /= float64(count)
	}

	return result
}

//This signal is how many steps are filled out of all steps. Presumably harder
//puzzles will have more non-fill steps.
func signalPercentageFilledSteps(directions SolveDirections) DifficultySignals {
	numerator := 0.0
	denominator := float64(len(directions.Steps))

	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			numerator += 1.0
		}
	}

	return DifficultySignals{
		"Percentage Fill Steps": numerator / denominator,
	}
}

//This signal is how many cells are unfilled at the beginning. Presumably
//harder puzzles will have fewer cells filled (although obviously this isn't
//necessarily true)
func signalNumberUnfilled(directions SolveDirections) DifficultySignals {

	//We don't have access to the underlying grid, so we'll just count how
	//many fill steps (since each can only add one number, and no numbers are
	//ever unfilled)

	count := 0.0
	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			count++
		}
	}

	return DifficultySignals{
		"Number Unfilled Cells": count,
	}
}

//This signal is how many steps into the solve directions before you encounter
//your first non-fill step. Non-fill steps are harder, so this signal captures
//how easy the start of the puzzle is.
func signalStepsUntilNonFill(directions SolveDirections) DifficultySignals {
	count := 0.0
	for _, step := range directions.Steps {
		if !step.Technique.IsFill() {
			break
		}
		count++
	}

	return DifficultySignals{
		"Steps Until Nonfill": count,
	}

}
<commit_msg>Added a TODO<commit_after>package sudoku

import (
	"fmt"
	"log"
	"strconv"
	"strings"
)

//DifficultySignals is a collection of names to float64 values, representing
//the various signals extracted from a SolveDirections, and used for the
//Difficulty calculation. Generally not useful to package users.
type DifficultySignals map[string]float64

//A difficulty signal generator can return more than one difficutly signal, so
//it doesn't just return float64 Each signal generator should always return a
//map with the SAME keys--so if you've called it once you know what the next
//calls will have as keys.
type difficultySignalGenerator func(directions SolveDirections) DifficultySignals

const _DIFFICULTY_WEIGHT_FILENAME = "difficulties.csv"

var difficultySignalGenerators []difficultySignalGenerator

//These are the weights that will be used to turn a list of signals into a
//difficulty. starting weights are set in hs_difficulty_weights.go, which is
//auto-generated. Generate those now: go:generate python util/difficulty-
//convert.py
var difficultySignalWeights map[string]float64

func init() {
	difficultySignalGenerators = []difficultySignalGenerator{
		signalTechnique,
		signalNumberOfSteps,
		signalTechniquePercentage,
		signalPercentageFilledSteps,
		signalNumberUnfilled,
		signalStepsUntilNonFill,
	}
}

//Stats returns a printout of interesting statistics about the
//SolveDirections, including number of steps, difficulty (based on this solve
//description alone), how unrelated the cells in subsequent steps are, and the
//values of all of the signals used to generate the difficulty.
func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	var lastStep *SolveStep
	dissimilarityAccum := 0.0
	for _, step := range self.Steps {
		if lastStep != nil {
			dissimilarityAccum += step.TargetCells.chainDissimilarity(lastStep.TargetCells)
		}
		techniqueCount[step.Technique.Name()] += 1
		lastStep = step
	}
	dissimilarityAccum /= float64(len(self.Steps))

	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	//TODO: we shouldn't even include this... it's not meaningful to report the difficulty of a single solve.
	result = append(result, fmt.Sprintf("Difficulty : %f", self.Signals().difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self.Steps)))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Avg Dissimilarity: %f", dissimilarityAccum))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		//TODO: pad the technique name with enough spaces so the colon lines up.
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

//Description returns a comprehensive prose description of the
//SolveDirections, including reasoning for each step, that if followed would
//lead to the grid being solved. Unlike Walkthrough, Description() does not
//include diagrams for each step.
func (self SolveDirections) Description() []string {

	//TODO: the IsHint directions don't sound as good as the old ones in OnlineSudoku did.

	if len(self.Steps) == 0 {
		return []string{"The puzzle is already solved."}
	}

	var descriptions []string

	//Hints have a special preamble.
	if self.IsHint {
		lastStep := self.Steps[len(self.Steps)-1]
		//TODO: this terminology is too tuned for the Online Sudoku use case.
		//it practice it should probably name the cell in text.
		descriptions = append(descriptions, "Based on the other numbers you've entered, this cell can only be a "+strconv.Itoa(lastStep.TargetNums[0])+".")
		descriptions = append(descriptions, "How do we know that?")
		if len(self.Steps) > 1 {
			descriptions = append(descriptions, "We can't fill any cells right away so first we need to cull some possibilities.")
		}
	}

	for i, step := range self.Steps {
		intro := ""
		description := step.Description()
		if len(self.Steps) > 1 {
			description = strings.ToLower(description)
			switch i {
			case 0:
				intro = "First, "
			case len(self.Steps) - 1:
				intro = "Finally, "
			default:
				//TODO: switch between "then" and "next" randomly.
				intro = "Next, "
			}
		}

		descriptions = append(descriptions, intro+description)

	}
	return descriptions
}

//Walkthrough prints an exhaustive set of human-readable directions that
//includes diagrams at each step to make it easier to follow.
func (self SolveDirections) Walkthrough() string {

	//TODO: test this.

	if len(self.Steps) == 0 {
		return "The puzzle could not be solved with any of the techniques we're aware of."
	}

	clone := self.Grid()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self.Steps))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self.Steps))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self.Steps[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

//Signals returns the DifficultySignals for this set of SolveDirections.
func (self SolveDirections) Signals() DifficultySignals {
	//Because of the contract of a DifficultySignalGenerator (that it always
	//returns the same keys), as long as DifficultySignalGenerators stays
	//constant it's reasonable for callers to assume that one call to
	//Signals() will return all of the string keys you'll see any time you
	//call Signals()
	result := DifficultySignals{}
	for _, generator := range difficultySignalGenerators {
		result.add(generator(self))
	}
	return result
}

//This will overwrite colliding values
//TODO: this is confusingly named
func (self DifficultySignals) add(other DifficultySignals) {
	for key, val := range other {
		self[key] = val
	}
}

//For keys in both, will sum them together.
//TODO: this is confusingly named (compared to Add)  Do we really need both Sum and Add?
func (self DifficultySignals) sum(other DifficultySignals) {
	for key, val := range other {
		self[key] += val
	}
}

func (self DifficultySignals) difficulty() float64 {
	accum := 0.0

	if constant, ok := difficultySignalWeights["Constant"]; ok {
		accum = constant
	} else {
		log.Println("Didn't have the constant term loaded.")
	}

	for signal, val := range self {
		//We can discard the OK because 0 is a reasonable thing to do with weights we aren't aware of.
		weight, _ := difficultySignalWeights[signal]

		accum += val * weight
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

//Rest of file is different Signals

//TODO: now that SolveDirections includes gridSnapshot, think if there are any
//additional Signals we can generate.

//This technique returns a count of how many each type of technique is seen.
//Different techniques are different "difficulties" so seeing more of a hard
//technique will Lead to a higher overall difficulty.
func signalTechnique(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Count"] = 0.0
	}
	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Count"]++
	}
	return result
}

//This signal is just number of steps. More steps is PROBABLY a harder puzzle.
func signalNumberOfSteps(directions SolveDirections) DifficultySignals {
	return DifficultySignals{
		"Number of Steps": float64(len(directions.Steps)),
	}
}

//This signal is like signalTechnique, except it returns the count divided by
//the TOTAL number of steps.
func signalTechniquePercentage(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Percentage"] = 0.0
	}

	count := len(directions.Steps)

	if count == 0 {
		return result
	}

	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Percentage"]++
	}

	//Now normalize all of them
	for name := range result {
		result[name] /= float64(count)
	}

	return result
}

//This signal is how many steps are filled out of all steps. Presumably harder
//puzzles will have more non-fill steps.
func signalPercentageFilledSteps(directions SolveDirections) DifficultySignals {
	numerator := 0.0
	denominator := float64(len(directions.Steps))

	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			numerator += 1.0
		}
	}

	return DifficultySignals{
		"Percentage Fill Steps": numerator / denominator,
	}
}

//This signal is how many cells are unfilled at the beginning. Presumably
//harder puzzles will have fewer cells filled (although obviously this isn't
//necessarily true)
func signalNumberUnfilled(directions SolveDirections) DifficultySignals {

	//We don't have access to the underlying grid, so we'll just count how
	//many fill steps (since each can only add one number, and no numbers are
	//ever unfilled)

	count := 0.0
	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			count++
		}
	}

	return DifficultySignals{
		"Number Unfilled Cells": count,
	}
}

//This signal is how many steps into the solve directions before you encounter
//your first non-fill step. Non-fill steps are harder, so this signal captures
//how easy the start of the puzzle is.
func signalStepsUntilNonFill(directions SolveDirections) DifficultySignals {
	count := 0.0
	for _, step := range directions.Steps {
		if !step.Technique.IsFill() {
			break
		}
		count++
	}

	return DifficultySignals{
		"Steps Until Nonfill": count,
	}

}
<|endoftext|>