<commit_before>package resource

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Repository struct {
	dir     string
	keyFile string
	source  string
}

func NewRepository(source string, dir string, keyFile string) Repository {
	return Repository{
		dir:     dir,
		keyFile: keyFile,
		source:  source,
	}
}

func (r Repository) Clone() error {
	_, err := os.Stat(r.dir)
	if !os.IsNotExist(err) {
		// repository is already cloned
		return nil
	}

	cmd := exec.Command("git", "clone", r.source, r.dir)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=/usr/bin/ssh -i %s", r.keyFile))
	}
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("Could not clone repository: git clone %s %s failed: %s\n[STDERR]\n%s", r.source, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) Fetch() error {
	err := r.runRepoCmd("git", "fetch", "origin")
	if err != nil {
		return fmt.Errorf("Could not fetch origin: %s", err)
	}
	return nil
}

func (r Repository) CheckoutRef(ref string) error {
	err := r.runRepoCmd("git", "checkout", ref)
	if err != nil {
		return fmt.Errorf("Could not checkout %s: %s", ref, err)
	}
	return nil
}

func (r Repository) RemoteBranches() ([]string, error) {
	branchesOutput, err := r.runRepoCmdOutput("git", "branch", "-r")
	if err != nil {
		return []string{}, fmt.Errorf("Could not list remote branches: %s", err)
	}
	branches := strings.Split(branchesOutput, "\n")

	trimmedBranches := []string{}
	for _, branch := range branches {
		if !strings.Contains(branch, "origin/HEAD ->") {
			trimmedBranches = append(trimmedBranches, strings.TrimSpace(branch))
		}
	}
	return trimmedBranches, nil
}

func (r Repository) RefAuthorName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%an\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefAuthorDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ai\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%cn\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefCommitDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ci\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitTimestamp(ref string) (int64, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ct\"", ref)
	if err != nil {
		return 0, fmt.Errorf("Could not show committer timestamp for %s: %s", ref, err)
	}
	timeString := strings.Trim(timeOutput, "\"")
	timestamp, err := strconv.ParseInt(timeString, 10, 64)
	if err != nil {
		return 0, fmt.Errorf("Could not parse committer timestamp (%s) for %s: %s", timeString, ref, err)
	}
	return timestamp, nil
}

func (r Repository) RefMessage(ref string) (string, error) {
	msgOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%B\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show message for %s: %s", ref, err)
	}
	return strings.Trim(msgOutput, "\""), nil
}

func (r Repository) LatestRef(branch string) (string, error) {
	refOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%H\"", branch)
	if err != nil {
		return "", fmt.Errorf("Could not show SHA for %s: %s", branch, err)
	}
	return strings.Trim(refOutput, "\""), nil
}

func (r Repository) RefsSinceTimestamp(branch string, timestamp int64) ([]string, error) {
	refsOutput, err := r.runRepoCmdOutput("git", "log", fmt.Sprintf("--since=%d", timestamp), "--format=\"%H\"", branch)
	if err != nil {
		return []string{}, fmt.Errorf("Could not list refs since %d for %s: %s", timestamp, branch, err)
	}
	return strings.Split(refsOutput, "\n"), nil
}

func (r Repository) runRepoCmd(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=/usr/bin/ssh -i %s", r.keyFile))
	}
	cmd.Dir = r.dir
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) runRepoCmdOutput(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=/usr/bin/ssh -i %s", r.keyFile))
	}
	cmd.Dir = r.dir
	var outputBytes bytes.Buffer
	cmd.Stdout = &outputBytes
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return strings.TrimSpace(outputBytes.String()), nil
}

func CreateKeyFile(privateKey string) (string, error) {
	keyFile, err := ioutil.TempFile("", "tracker-git-branch-resource")
	if err != nil {
		return "", fmt.Errorf("Could not create keyfile: %s", err)
	}
	keyFile.Chmod(0600)
	_, err = keyFile.WriteString(privateKey)
	if err != nil {
		return "", fmt.Errorf("Could not write keyfile %s: %s", keyFile, err)
	}
	return keyFile.Name(), nil
}
<commit_msg>Handle repository target dir existence check correctly<commit_after>package resource

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Repository struct {
	dir     string
	keyFile string
	source  string
}

func NewRepository(source string, dir string, keyFile string) Repository {
	return Repository{
		dir:     dir,
		keyFile: keyFile,
		source:  source,
	}
}

func (r Repository) Clone() error {
	_, err := os.Stat(r.dir)
	if err == nil {
		// repository is already cloned
		return nil
	}
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not stat repository dir %s: %s", r.dir, err)
	}

	cmd := exec.Command("git", "clone", r.source, r.dir)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=/usr/bin/ssh -i %s", r.keyFile))
	}
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("Could not clone repository: git clone %s %s failed: %s\n[STDERR]\n%s", r.source, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) Fetch() error {
	err := r.runRepoCmd("git", "fetch", "origin")
	if err != nil {
		return fmt.Errorf("Could not fetch origin: %s", err)
	}
	return nil
}

func (r Repository) CheckoutRef(ref string) error {
	err := r.runRepoCmd("git", "checkout", ref)
	if err != nil {
		return fmt.Errorf("Could not checkout %s: %s", ref, err)
	}
	return nil
}

func (r Repository) RemoteBranches() ([]string, error) {
	branchesOutput, err := r.runRepoCmdOutput("git", "branch", "-r")
	if err != nil {
		return []string{}, fmt.Errorf("Could not list remote branches: %s", err)
	}
	branches := strings.Split(branchesOutput, "\n")

	trimmedBranches := []string{}
	for _, branch := range branches {
		if !strings.Contains(branch, "origin/HEAD ->") {
			trimmedBranches = append(trimmedBranches, strings.TrimSpace(branch))
		}
	}
	return trimmedBranches, nil
}

func (r Repository) RefAuthorName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%an\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefAuthorDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ai\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%cn\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefCommitDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ci\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitTimestamp(ref string) (int64, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ct\"", ref)
	if err != nil {
		return 0, fmt.Errorf("Could not show committer timestamp for %s: %s", ref, err)
	}
	timeString := strings.Trim(timeOutput, "\"")
	timestamp, err := strconv.ParseInt(timeString, 10, 64)
	if err != nil {
		return 0, fmt.Errorf("Could not parse committer timestamp (%s) for %s: %s", timeString, ref, err)
	}
	return timestamp, nil
}

func (r Repository) RefMessage(ref string) (string, error) {
	msgOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%B\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show message for %s: %s", ref, err)
	}
	return strings.Trim(msgOutput, "\""), nil
}

func (r Repository) LatestRef(branch string) (string, error) {
	refOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%H\"", branch)
	if err != nil {
		return "", fmt.Errorf("Could not show SHA for %s: %s", branch, err)
	}
	return strings.Trim(refOutput, "\""), nil
}

func (r Repository) RefsSinceTimestamp(branch string, timestamp int64) ([]string, error) {
	refsOutput, err := r.runRepoCmdOutput("git", "log", fmt.Sprintf("--since=%d", timestamp), "--format=\"%H\"", branch)
	if err != nil {
		return []string{}, fmt.Errorf("Could not list refs since %d for %s: %s", timestamp, branch, err)
	}
	return strings.Split(refsOutput, "\n"), nil
}

func (r Repository) runRepoCmd(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=/usr/bin/ssh -i %s", r.keyFile))
	}
	cmd.Dir = r.dir
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) runRepoCmdOutput(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=/usr/bin/ssh -i %s", r.keyFile))
	}
	cmd.Dir = r.dir
	var outputBytes bytes.Buffer
	cmd.Stdout = &outputBytes
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return strings.TrimSpace(outputBytes.String()), nil
}

func CreateKeyFile(privateKey string) (string, error) {
	keyFile, err := ioutil.TempFile("", "tracker-git-branch-resource")
	if err != nil {
		return "", fmt.Errorf("Could not create keyfile: %s", err)
	}
	keyFile.Chmod(0600)
	_, err = keyFile.WriteString(privateKey)
	if err != nil {
		return "", fmt.Errorf("Could not write keyfile %s: %s", keyFile, err)
	}
	return keyFile.Name(), nil
}
<|endoftext|>


<commit_before><commit_msg>add a test that parseDfLines ignores the docker mapper storage<commit_after><|endoftext|>


<commit_before>package main

import (
	"math"
	"testing"
)

const MaxInt = int(^uint(0) >> 1)

func smallestMultiple() int {
	divisible := false
	answer := 0
	for i := 20; i < MaxInt; i++ {
		for j := 1; j <= 20; j++ {
			divisible = math.Remainder(float64(i), float64(j)) == float64(0)
			if divisible == false {
				break
			}
		}
		if divisible == true {
			answer = i
			break
		}
	}
	return answer
}

func TestSmallestMultiple(t *testing.T) {
	x := smallestMultiple()
	answer := 232792560
	if x != answer {
		t.Errorf("result = %v, want %v", x, answer)
	}
}

<commit_msg>Speed up problem 5 solution<commit_after>package main

import (
	"math"
	"testing"
)

// From mathblog.dk/project-euler-problem-5
func generatePrimes(limit int) []int {
	var primes []int
	var j int
	isPrime := false

	primes = append(primes, 2)

	for i := 3; i <= limit; i += 2 {
		j = 0
		isPrime = true
		for primes[j]*primes[j] <= i {
			if i%primes[j] == 0 {
				isPrime = false
				break
			}
			j++
		}
		if isPrime {
			primes = append(primes, i)
		}
	}

	return primes
}

func smallestMultiple() int {
	divisorMax := 20
	var primes []int = generatePrimes(divisorMax)
	result := 1

	for i := 0; i < len(primes); i++ {
		a := math.Floor(math.Log(float64(divisorMax)) / math.Log(float64(primes[i])))
		result = result * int(math.Pow(float64(primes[i]), a))
	}
	return result
}

func TestSmallestMultiple(t *testing.T) {
	x := smallestMultiple()
	answer := 232792560
	if x != answer {
		t.Errorf("result = %v, want %v", x, answer)
	}
}
<|endoftext|>


<commit_before>package goshim

import (
	"path/filepath"
	"reflect"
	"strings"
	"testing"
)

func TestSrcList(t *testing.T) {
	files := []string{"main.go", "type.go"}

	dir := "testdata/testproj"
	list, _ := srcList(dir)
	if !reflect.DeepEqual(list, files) {
		t.Errorf("failed to get srcList. got: %+v, expected: %+v", list, files)
	}

	dir = "./" + dir
	list, _ = srcList(dir)
	if !reflect.DeepEqual(list, files) {
		t.Errorf("failed to get srcList. got: %+v, expected: %+v", list, files)
	}

	dir, _ = filepath.Abs(dir)
	list, _ = srcList(dir)
	if !reflect.DeepEqual(list, files) {
		t.Errorf("failed to get srcList. got: %+v, expected: %+v", list, files)
	}

	dir = "testdata/non-existence"
	_, err := srcList(dir)
	if err == nil || !strings.HasPrefix(err.Error(), "can't load package: package ./testdata/non-existence: open") {
		t.Errorf("something went wrong")
	}

	dir = "testdata/notgo"
	_, err = srcList(dir)
	if err == nil || !strings.HasPrefix(err.Error(), "can't load package: package ./testdata/notgo: no buildable Go source files in") {
		t.Errorf("something went wrong")
	}
}

func TestRun(t *testing.T) {
	dir := "testdata/testproj"
	ret := Run([]string{dir})
	if ret != 0 {
		t.Errorf("something went wrong")
	}
}
<commit_msg>fix test<commit_after>package goshim

import (
	"path/filepath"
	"reflect"
	"strings"
	"testing"
)

func TestSrcList(t *testing.T) {
	files := []string{"main.go", "type.go"}

	dir := "testdata/testproj"
	list, _ := srcList(dir)
	if !reflect.DeepEqual(list, files) {
		t.Errorf("failed to get srcList. got: %+v, expected: %+v", list, files)
	}

	dir = "./" + dir
	list, _ = srcList(dir)
	if !reflect.DeepEqual(list, files) {
		t.Errorf("failed to get srcList. got: %+v, expected: %+v", list, files)
	}

	dir, _ = filepath.Abs(dir)
	list, _ = srcList(dir)
	if !reflect.DeepEqual(list, files) {
		t.Errorf("failed to get srcList. got: %+v, expected: %+v", list, files)
	}

	dir = "testdata/non-existence"
	_, err := srcList(dir)
	if err == nil || !strings.HasPrefix(err.Error(), "can't load package: package ./testdata/non-existence: open") {
		t.Errorf("something went wrong: %v", err)
	}

	dir = "testdata/notgo"
	_, err = srcList(dir)
	if err == nil || !strings.HasPrefix(err.Error(), "can't load package: package ./testdata/notgo: no buildable Go source files in") {
		t.Errorf("something went wrong %v:", err)
	}
}

func TestRun(t *testing.T) {
	dir := "testdata/testproj"
	ret := Run([]string{dir})
	if ret != 0 {
		t.Errorf("something went wrong: %v", ret)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/blevesearch/bleve"

	log "github.com/couchbaselabs/clog"
)

func docIDLookup(req *http.Request) string {
	return muxVariableLookup(req, "docID")
}

func indexNameLookup(req *http.Request) string {
	return muxVariableLookup(req, "indexName")
}

func indexAlias(mgr *Manager, indexName, indexUUID string) (bleve.IndexAlias, error) {
	nodeDefs, _, err := CfgGetNodeDefs(mgr.Cfg(), NODE_DEFS_WANTED)
	if err != nil {
		return nil, fmt.Errorf("could not retrieve wanted nodeDefs, err: %v", err)
	}

	// Returns true if the node has the "pindex" tag.
	nodeDoesPIndexes := func(nodeUUID string) (*NodeDef, bool) {
		for _, nodeDef := range nodeDefs.NodeDefs {
			if nodeDef.UUID == nodeUUID {
				if len(nodeDef.Tags) <= 0 {
					return nodeDef, true
				}
				for _, tag := range nodeDef.Tags {
					if tag == "pindex" {
						return nodeDef, true
					}
				}
			}
		}
		return nil, false
	}

	_, allPlanPIndexes, err := mgr.GetPlanPIndexes(false)
	if err != nil {
		return nil, fmt.Errorf("could not retrieve allPlanPIndexes, err: %v", err)
	}

	planPIndexes, exists := allPlanPIndexes[indexName]
	if !exists || len(planPIndexes) <= 0 {
		return nil, fmt.Errorf("no planPIndexes for indexName: %s", indexName)
	}

	_, pindexes := mgr.CurrentMaps()

	selfUUID := mgr.UUID()
	_, selfDoesPIndexes := nodeDoesPIndexes(selfUUID)

	// TODO: need some abstractions to allow for non-bleve pindexes, perhaps.
	alias := bleve.NewIndexAlias()

build_alias_loop:
	for _, planPIndex := range planPIndexes {
		// First check whether this local node serves that planPIndex.
		if selfDoesPIndexes &&
			strings.Contains(planPIndex.NodeUUIDs[selfUUID], PLAN_PINDEX_NODE_READ) {
			localPIndex, exists := pindexes[planPIndex.Name]
			if exists &&
				localPIndex != nil &&
				localPIndex.Name == planPIndex.Name &&
				localPIndex.IndexType == "bleve" &&
				localPIndex.IndexName == indexName &&
				(indexUUID == "" || localPIndex.IndexUUID == indexUUID) {
				bindex, ok := localPIndex.Impl.(bleve.Index)
				if ok && bindex != nil {
					alias.Add(bindex)
					continue build_alias_loop
				}
			}
		}

		// Otherwise, look for a remote node that serves that planPIndex.
		//
		// TODO: We should favor the most up-to-date node rather than
		// the first one that we run into here?  But, perhaps the most
		// up-to-date node is also the most overloaded?
		for nodeUUID, nodeState := range planPIndex.NodeUUIDs {
			if nodeUUID != selfUUID {
				nodeDef, ok := nodeDoesPIndexes(nodeUUID)
				if ok &&
					strings.Contains(nodeState, PLAN_PINDEX_NODE_READ) {
					baseURL := "http://" + nodeDef.HostPort + "/api/pindex/" + planPIndex.Name
					// TODO: Propagate auth to bleve client.
					// TODO: Propagate consistency requirements to bleve client.
					alias.Add(&BleveClient{
						SearchURL:   baseURL + "/search",
						DocCountURL: baseURL + "/count",
					})
					continue build_alias_loop
				}
			}
		}

		return nil, fmt.Errorf("no node provides planPIndex: %#v", planPIndex)
	}

	return alias, nil
}

// ------------------------------------------------------------------

type ListIndexHandler struct {
	mgr *Manager
}

func NewListIndexHandler(mgr *Manager) *ListIndexHandler {
	return &ListIndexHandler{mgr: mgr}
}

func (h *ListIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexDefs, _, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	rv := struct {
		Status    string     `json:"status"`
		IndexDefs *IndexDefs `json:"indexDefs"`
	}{
		Status:    "ok",
		IndexDefs: indexDefs,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type GetIndexHandler struct {
	mgr *Manager
}

func NewGetIndexHandler(mgr *Manager) *GetIndexHandler {
	return &GetIndexHandler{mgr: mgr}
}

func (h *GetIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	_, indexDefsByName, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	indexDef, exists := indexDefsByName[indexName]
	if !exists || indexDef == nil {
		showError(w, req, "not an index", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")
	if indexUUID != "" && indexUUID != indexDef.UUID {
		showError(w, req, "wrong index UUID", 400)
		return
	}

	m := map[string]interface{}{}
	if indexDef.Schema != "" {
		if err := json.Unmarshal([]byte(indexDef.Schema), &m); err != nil {
			showError(w, req, "could not unmarshal mapping", 500)
			return
		}
	}

	rv := struct {
		Status       string                 `json:"status"`
		IndexDef     *IndexDef              `json:"indexDef"`
		IndexMapping map[string]interface{} `json:"indexMapping"`
	}{
		Status:       "ok",
		IndexDef:     indexDef,
		IndexMapping: m,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type CountHandler struct {
	mgr *Manager
}

func NewCountHandler(mgr *Manager) *CountHandler {
	return &CountHandler{mgr: mgr}
}

func (h *CountHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	alias, err := indexAlias(h.mgr, indexName, indexUUID)
	if err != nil {
		showError(w, req, fmt.Sprintf("index alias: %v", err), 500)
		return
	}

	docCount, err := alias.DocCount()
	if err != nil {
		showError(w, req, fmt.Sprintf("error counting docs: %v", err), 500)
		return
	}

	rv := struct {
		Status string `json:"status"`
		Count  uint64 `json:"count"`
	}{
		Status: "ok",
		Count:  docCount,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type SearchHandler struct {
	mgr *Manager
}

func NewSearchHandler(mgr *Manager) *SearchHandler {
	return &SearchHandler{mgr: mgr}
}

func (h *SearchHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	log.Printf("rest search request: %s", indexName)

	alias, err := indexAlias(h.mgr, indexName, indexUUID)
	if err != nil {
		showError(w, req, fmt.Sprintf("index alias: %v", err), 500)
		return
	}

	// read the request body
	requestBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400)
		return
	}

	log.Printf("rest search request body: %s", requestBody)

	// parse the request
	var searchRequest bleve.SearchRequest
	err = json.Unmarshal(requestBody, &searchRequest)
	if err != nil {
		showError(w, req, fmt.Sprintf("error parsing query: %v", err), 400)
		return
	}

	log.Printf("rest search parsed request %#v", searchRequest)

	// varlidate the query
	err = searchRequest.Query.Validate()
	if err != nil {
		showError(w, req, fmt.Sprintf("error validating query: %v", err), 400)
		return
	}

	// execute the query
	searchResponse, err := alias.Search(&searchRequest)
	if err != nil {
		showError(w, req, fmt.Sprintf("error executing query: %v", err), 500)
		return
	}

	// encode the response
	mustEncode(w, searchResponse)
}
<commit_msg>added TODO comment on querying user-defined index aliases<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/blevesearch/bleve"

	log "github.com/couchbaselabs/clog"
)

func docIDLookup(req *http.Request) string {
	return muxVariableLookup(req, "docID")
}

func indexNameLookup(req *http.Request) string {
	return muxVariableLookup(req, "indexName")
}

// Return an indexAlias that represents all the PIndexes for the index.
//
// TODO: But, what if the index is an explicit, user-defined alias?
// Then, this should instead be an alias to indexes, and we should
// instead use table-lookup driven logic here to switch behavior based
// on index type.
func indexAlias(mgr *Manager, indexName, indexUUID string) (bleve.IndexAlias, error) {
	nodeDefs, _, err := CfgGetNodeDefs(mgr.Cfg(), NODE_DEFS_WANTED)
	if err != nil {
		return nil, fmt.Errorf("could not retrieve wanted nodeDefs, err: %v", err)
	}

	// Returns true if the node has the "pindex" tag.
	nodeDoesPIndexes := func(nodeUUID string) (*NodeDef, bool) {
		for _, nodeDef := range nodeDefs.NodeDefs {
			if nodeDef.UUID == nodeUUID {
				if len(nodeDef.Tags) <= 0 {
					return nodeDef, true
				}
				for _, tag := range nodeDef.Tags {
					if tag == "pindex" {
						return nodeDef, true
					}
				}
			}
		}
		return nil, false
	}

	_, allPlanPIndexes, err := mgr.GetPlanPIndexes(false)
	if err != nil {
		return nil, fmt.Errorf("could not retrieve allPlanPIndexes, err: %v", err)
	}

	planPIndexes, exists := allPlanPIndexes[indexName]
	if !exists || len(planPIndexes) <= 0 {
		return nil, fmt.Errorf("no planPIndexes for indexName: %s", indexName)
	}

	_, pindexes := mgr.CurrentMaps()

	selfUUID := mgr.UUID()
	_, selfDoesPIndexes := nodeDoesPIndexes(selfUUID)

	// TODO: need some abstractions to allow for non-bleve pindexes, perhaps.
	alias := bleve.NewIndexAlias()

build_alias_loop:
	for _, planPIndex := range planPIndexes {
		// First check whether this local node serves that planPIndex.
		if selfDoesPIndexes &&
			strings.Contains(planPIndex.NodeUUIDs[selfUUID], PLAN_PINDEX_NODE_READ) {
			localPIndex, exists := pindexes[planPIndex.Name]
			if exists &&
				localPIndex != nil &&
				localPIndex.Name == planPIndex.Name &&
				localPIndex.IndexType == "bleve" &&
				localPIndex.IndexName == indexName &&
				(indexUUID == "" || localPIndex.IndexUUID == indexUUID) {
				bindex, ok := localPIndex.Impl.(bleve.Index)
				if ok && bindex != nil {
					alias.Add(bindex)
					continue build_alias_loop
				}
			}
		}

		// Otherwise, look for a remote node that serves that planPIndex.
		//
		// TODO: We should favor the most up-to-date node rather than
		// the first one that we run into here?  But, perhaps the most
		// up-to-date node is also the most overloaded?
		for nodeUUID, nodeState := range planPIndex.NodeUUIDs {
			if nodeUUID != selfUUID {
				nodeDef, ok := nodeDoesPIndexes(nodeUUID)
				if ok &&
					strings.Contains(nodeState, PLAN_PINDEX_NODE_READ) {
					baseURL := "http://" + nodeDef.HostPort + "/api/pindex/" + planPIndex.Name
					// TODO: Propagate auth to bleve client.
					// TODO: Propagate consistency requirements to bleve client.
					alias.Add(&BleveClient{
						SearchURL:   baseURL + "/search",
						DocCountURL: baseURL + "/count",
					})
					continue build_alias_loop
				}
			}
		}

		return nil, fmt.Errorf("no node provides planPIndex: %#v", planPIndex)
	}

	return alias, nil
}

// ------------------------------------------------------------------

type ListIndexHandler struct {
	mgr *Manager
}

func NewListIndexHandler(mgr *Manager) *ListIndexHandler {
	return &ListIndexHandler{mgr: mgr}
}

func (h *ListIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexDefs, _, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	rv := struct {
		Status    string     `json:"status"`
		IndexDefs *IndexDefs `json:"indexDefs"`
	}{
		Status:    "ok",
		IndexDefs: indexDefs,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type GetIndexHandler struct {
	mgr *Manager
}

func NewGetIndexHandler(mgr *Manager) *GetIndexHandler {
	return &GetIndexHandler{mgr: mgr}
}

func (h *GetIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	_, indexDefsByName, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	indexDef, exists := indexDefsByName[indexName]
	if !exists || indexDef == nil {
		showError(w, req, "not an index", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")
	if indexUUID != "" && indexUUID != indexDef.UUID {
		showError(w, req, "wrong index UUID", 400)
		return
	}

	m := map[string]interface{}{}
	if indexDef.Schema != "" {
		if err := json.Unmarshal([]byte(indexDef.Schema), &m); err != nil {
			showError(w, req, "could not unmarshal mapping", 500)
			return
		}
	}

	rv := struct {
		Status       string                 `json:"status"`
		IndexDef     *IndexDef              `json:"indexDef"`
		IndexMapping map[string]interface{} `json:"indexMapping"`
	}{
		Status:       "ok",
		IndexDef:     indexDef,
		IndexMapping: m,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type CountHandler struct {
	mgr *Manager
}

func NewCountHandler(mgr *Manager) *CountHandler {
	return &CountHandler{mgr: mgr}
}

func (h *CountHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	alias, err := indexAlias(h.mgr, indexName, indexUUID)
	if err != nil {
		showError(w, req, fmt.Sprintf("index alias: %v", err), 500)
		return
	}

	docCount, err := alias.DocCount()
	if err != nil {
		showError(w, req, fmt.Sprintf("error counting docs: %v", err), 500)
		return
	}

	rv := struct {
		Status string `json:"status"`
		Count  uint64 `json:"count"`
	}{
		Status: "ok",
		Count:  docCount,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type SearchHandler struct {
	mgr *Manager
}

func NewSearchHandler(mgr *Manager) *SearchHandler {
	return &SearchHandler{mgr: mgr}
}

func (h *SearchHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	log.Printf("rest search request: %s", indexName)

	alias, err := indexAlias(h.mgr, indexName, indexUUID)
	if err != nil {
		showError(w, req, fmt.Sprintf("index alias: %v", err), 500)
		return
	}

	// read the request body
	requestBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400)
		return
	}

	log.Printf("rest search request body: %s", requestBody)

	// parse the request
	var searchRequest bleve.SearchRequest
	err = json.Unmarshal(requestBody, &searchRequest)
	if err != nil {
		showError(w, req, fmt.Sprintf("error parsing query: %v", err), 400)
		return
	}

	log.Printf("rest search parsed request %#v", searchRequest)

	// varlidate the query
	err = searchRequest.Query.Validate()
	if err != nil {
		showError(w, req, fmt.Sprintf("error validating query: %v", err), 400)
		return
	}

	// execute the query
	searchResponse, err := alias.Search(&searchRequest)
	if err != nil {
		showError(w, req, fmt.Sprintf("error executing query: %v", err), 500)
		return
	}

	// encode the response
	mustEncode(w, searchResponse)
}
<|endoftext|>


<commit_before>package rest

import (
	"net/url"

	"github.com/lfq7413/tomato/config"
	"github.com/lfq7413/tomato/errs"
	"github.com/lfq7413/tomato/mail"
	"github.com/lfq7413/tomato/orm"
	"github.com/lfq7413/tomato/types"
	"github.com/lfq7413/tomato/utils"
)

var adapter mail.Adapter

func init() {
	a := config.TConfig.MailAdapter
	if a == "smtp" {
		adapter = mail.NewSMTPAdapter()
	} else {
		adapter = mail.NewSMTPAdapter()
	}
}

// shouldVerifyEmails 根据配置参数确定是否需要验证邮箱
func shouldVerifyEmails() bool {
	return config.TConfig.VerifyUserEmails
}

// SetEmailVerifyToken 设置需要验证的 token
func SetEmailVerifyToken(user types.M) {
	if shouldVerifyEmails() {
		user["_email_verify_token"] = utils.CreateToken()
		user["emailVerified"] = false
	}
}

// SendVerificationEmail 发送验证邮件
func SendVerificationEmail(user types.M) {
	if shouldVerifyEmails() == false {
		return
	}
	user = getUserIfNeeded(user)
	if user == nil {
		return
	}
	user["className"] = "_User"
	token := url.QueryEscape(user["_email_verify_token"].(string))
	username := url.QueryEscape(user["username"].(string))
	link := config.TConfig.VerifyEmailURL + "?token=" + token + "&username=" + username
	options := types.M{
		"appName": config.TConfig.AppName,
		"link":    link,
		"user":    user,
	}
	adapter.SendMail(defaultVerificationEmail(options))
}

// getUserIfNeeded 把 user 填充完整，如果无法完成则返回 nil
func getUserIfNeeded(user types.M) types.M {
	if user["username"] != nil && user["email"] != nil {
		return user
	}
	where := types.M{}
	if user["username"] != nil {
		where["username"] = user["username"]
	}
	if user["email"] != nil {
		where["email"] = user["email"]
	}

	response, err := NewQuery(Master(), "_User", where, types.M{}).Execute()
	if err != nil {
		return nil
	}
	if utils.HasResults(response) == false {
		return nil
	}

	return response["results"].([]interface{})[0].(map[string]interface{})
}

func defaultVerificationEmail(options types.M) types.M {
	user := utils.MapInterface(options["user"])
	text := "Hi,\n\n"
	text += "You are being asked to confirm the e-mail address " + user["email"].(string)
	text += " with " + options["appName"].(string) + "\n\n"
	text += "Click here to confirm it:\n" + options["link"].(string)
	to := user["email"].(string)
	subject := "Please verify your e-mail for " + options["appName"].(string)
	return types.M{
		"text":    text,
		"to":      to,
		"subject": subject,
	}
}

// SendPasswordResetEmail 发送密码重置邮件
func SendPasswordResetEmail(email string) error {
	user := setPasswordResetToken(email)
	if user == nil || len(user) == 0 {
		return errs.E(errs.EmailMissing, "you must provide an email")
	}
	user["className"] = "_User"
	token := url.QueryEscape(user["_perishable_token"].(string))
	username := url.QueryEscape(user["username"].(string))
	link := config.TConfig.RequestResetPasswordURL + "?token=" + token + "&username=" + username
	options := types.M{
		"appName": config.TConfig.AppName,
		"link":    link,
		"user":    user,
	}
	adapter.SendMail(defaultResetPasswordEmail(options))
	return nil
}

func setPasswordResetToken(email string) types.M {
	token := utils.CreateToken()
	collection := orm.AdaptiveCollection("_User")
	where := types.M{"email": email}
	update := types.M{
		"$set": types.M{"_perishable_token": token},
	}
	return collection.FindOneAndUpdate(where, update)
}

func defaultResetPasswordEmail(options types.M) types.M {
	// TODO
	return nil
}
<commit_msg>完成 defaultResetPasswordEmail<commit_after>package rest

import (
	"net/url"

	"github.com/lfq7413/tomato/config"
	"github.com/lfq7413/tomato/errs"
	"github.com/lfq7413/tomato/mail"
	"github.com/lfq7413/tomato/orm"
	"github.com/lfq7413/tomato/types"
	"github.com/lfq7413/tomato/utils"
)

var adapter mail.Adapter

func init() {
	a := config.TConfig.MailAdapter
	if a == "smtp" {
		adapter = mail.NewSMTPAdapter()
	} else {
		adapter = mail.NewSMTPAdapter()
	}
}

// shouldVerifyEmails 根据配置参数确定是否需要验证邮箱
func shouldVerifyEmails() bool {
	return config.TConfig.VerifyUserEmails
}

// SetEmailVerifyToken 设置需要验证的 token
func SetEmailVerifyToken(user types.M) {
	if shouldVerifyEmails() {
		user["_email_verify_token"] = utils.CreateToken()
		user["emailVerified"] = false
	}
}

// SendVerificationEmail 发送验证邮件
func SendVerificationEmail(user types.M) {
	if shouldVerifyEmails() == false {
		return
	}
	user = getUserIfNeeded(user)
	if user == nil {
		return
	}
	user["className"] = "_User"
	token := url.QueryEscape(user["_email_verify_token"].(string))
	username := url.QueryEscape(user["username"].(string))
	link := config.TConfig.VerifyEmailURL + "?token=" + token + "&username=" + username
	options := types.M{
		"appName": config.TConfig.AppName,
		"link":    link,
		"user":    user,
	}
	adapter.SendMail(defaultVerificationEmail(options))
}

// getUserIfNeeded 把 user 填充完整，如果无法完成则返回 nil
func getUserIfNeeded(user types.M) types.M {
	if user["username"] != nil && user["email"] != nil {
		return user
	}
	where := types.M{}
	if user["username"] != nil {
		where["username"] = user["username"]
	}
	if user["email"] != nil {
		where["email"] = user["email"]
	}

	response, err := NewQuery(Master(), "_User", where, types.M{}).Execute()
	if err != nil {
		return nil
	}
	if utils.HasResults(response) == false {
		return nil
	}

	return response["results"].([]interface{})[0].(map[string]interface{})
}

func defaultVerificationEmail(options types.M) types.M {
	user := utils.MapInterface(options["user"])
	text := "Hi,\n\n"
	text += "You are being asked to confirm the e-mail address " + user["email"].(string)
	text += " with " + options["appName"].(string) + "\n\n"
	text += "Click here to confirm it:\n" + options["link"].(string)
	to := user["email"].(string)
	subject := "Please verify your e-mail for " + options["appName"].(string)
	return types.M{
		"text":    text,
		"to":      to,
		"subject": subject,
	}
}

// SendPasswordResetEmail 发送密码重置邮件
func SendPasswordResetEmail(email string) error {
	user := setPasswordResetToken(email)
	if user == nil || len(user) == 0 {
		return errs.E(errs.EmailMissing, "you must provide an email")
	}
	user["className"] = "_User"
	token := url.QueryEscape(user["_perishable_token"].(string))
	username := url.QueryEscape(user["username"].(string))
	link := config.TConfig.RequestResetPasswordURL + "?token=" + token + "&username=" + username
	options := types.M{
		"appName": config.TConfig.AppName,
		"link":    link,
		"user":    user,
	}
	adapter.SendMail(defaultResetPasswordEmail(options))
	return nil
}

func setPasswordResetToken(email string) types.M {
	token := utils.CreateToken()
	collection := orm.AdaptiveCollection("_User")
	where := types.M{"email": email}
	update := types.M{
		"$set": types.M{"_perishable_token": token},
	}
	return collection.FindOneAndUpdate(where, update)
}

func defaultResetPasswordEmail(options types.M) types.M {
	user := utils.MapInterface(options["user"])
	text := "Hi,\n\n"
	text += "You requested to reset your password for " + options["appName"].(string) + "\n\n"
	text += "Click here to reset it:\n" + options["link"].(string)
	to := user["email"].(string)
	subject := "Password Reset for " + options["appName"].(string)
	return types.M{
		"text":    text,
		"to":      to,
		"subject": subject,
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// Package graph contains the graph data structure to represent dependency graphs.
// Nodes are fmt.Stringers, and are uniquely identified by their String() method.
// Edges don't have a public interface, but the various graph.*Edge methods work with them.
package graph

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/SimplicityApks/depgrapher/syntax"
	"runtime"
	"strings"
	"sync"
)

// Node represents a single data point stored in a Graph. Its String() method should return a unique string identifier.
// The String() method call should be fast (ideally inline), as it will be called often!
type Node fmt.Stringer

// node is a simple string type, created only by the Graph input methods.
type node string

func (s node) String() string {
	return string(s)
}

type edge struct {
	source Node
	target Node
}

func (e *edge) String() string {
	return e.source.String() + " => " + e.target.String()
}

// Interface defines a basic graph-like data structure.
// The stored data is accessed by the String() method, so every Node needs to implement fmt.Stringer.
type Interface interface {
	// GetNode returns the Node with the specified name, or nil if the name couldn't be found in the graph.
	GetNode(name string) Node
	// GetNodes returns all Node objects saved in the graph as a slice. If the graph is empty, an empty slice is returned.
	GetNodes() []Node

	// AddEdge adds a new edge from source to target to the graph. The nodes are added to the graph if they were not present.
	AddEdge(source, target Node)
	// HasEdge returns true if the graph has an edge from the source Node to the target Node, false otherwise.
	HasEdge(source, target string) bool
	// GetDependencies returns a slice containing all dependencies of the Node with the given string.
	GetDependencies(node string) []Node
	// GetDependencies returns a slice containing all dependants of the Node with the given string.
	GetDependants(node string) []Node

	// Copy returns a deep copy of the graph.
	Copy() Interface
}

// Graph represents the dependency graph in memory. It internally uses a mutex to make read and write access concurrency-safe
// It satisfies the graph.Interface and fmt.Stringer interfaces.
// The zero value is an empty graph with no dependencies.
type Graph struct {
	nodes []Node
	edges []edge
	mutex sync.Mutex
}

// GetNode returns the Node with the specified name, or nil if the name couldn't be found in g.
func (g *Graph) GetNode(name string) Node {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, node := range g.nodes {
		if node.String() == name {
			return node
		}
	}
	return nil
}

// GetNodes returns all Node objects saved in the Graph as a slice. If it is empty, an empty slice is returned.
func (g *Graph) GetNodes() []Node {
	return g.nodes
}

// AddEdge adds a new edge from source to target to the Graph. The nodes are added to the Graph if they were not present.
func (g *Graph) AddEdge(source, target Node) {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	// add the nodes if they aren't already in place
	var foundSource, foundTarget bool
	for _, n := range g.nodes {
		if n.String() == source.String() {
			foundSource = true
		} else if n.String() == target.String() {
			foundTarget = true
		}
	}
	if !foundSource {
		g.nodes = append(g.nodes, source)
	}
	if !foundTarget {
		g.nodes = append(g.nodes, target)
	}
	g.edges = append(g.edges, edge{source: source, target: target})
}

// HasEdge returns true if g has an edge from the source Node to the target Node, false otherwise.
func (g *Graph) HasEdge(source, target string) bool {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, edge := range g.edges {
		if edge.source.String() == source && edge.target.String() == target {
			return true
		}
	}
	return false
}

// GetDependencies returns a slice containing all dependencies of the Node with the given string.
// The Graph contains an edge from the Node to each item in the returned dependencies.
func (g *Graph) GetDependencies(node string) []Node {
	deps := make([]Node, 0)
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, edge := range g.edges {
		if edge.source.String() == node {
			deps = append(deps, edge.target)
		}
	}
	return deps
}

// GetDependencies returns a slice containing all dependants of the Node with the given string.
// The Graph contains an edge from each item in dependants to the Node.
func (g *Graph) GetDependants(node string) []Node {
	deps := make([]Node, 0)
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, edge := range g.edges {
		if edge.target.String() == node {
			deps = append(deps, edge.source)
		}
	}
	return deps
}

// Copy returns a deep copy of the Graph.
func (g *Graph) Copy() Interface {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	result := &Graph{
		nodes: make([]Node, len(g.nodes)),
		edges: make([]edge, len(g.edges)),
	}
	copy(result.nodes, g.nodes)
	copy(result.edges, g.edges)
	return result
}

// scanDependencies adds the given dependency line with the given syntax as edges to g.
func (g *Graph) scanDependencies(line string, syntax *syntax.Syntax) {
	infixIndex := strings.Index(line, syntax.EdgeInfix)
	sources := []string{line[:infixIndex]}
	if syntax.SourceDelimiter != "" {
		sources = strings.Split(line[:infixIndex], syntax.SourceDelimiter)
	}
	targets := strings.Split(line[infixIndex+len(syntax.EdgeInfix):], syntax.TargetDelimiter)
	for _, source := range sources {
		if syntax.StripWhitespace {
			source = strings.TrimSpace(source)
		}
		if source != "" {
			for _, target := range targets {
				if syntax.StripWhitespace {
					target = strings.TrimSpace(target)
				}
				if target != "" {
					g.AddEdge(node(source), node(target))
				}
			}
		}
	}
}

// FromScanner reads data from the given scanner, building up the dependency tree.
func (g *Graph) FromScanner(scanner *bufio.Scanner, syntaxes ...*syntax.Syntax) (*Graph, error) {
	if len(syntaxes) == 0 {
		panic("FromScanner: At least one syntax required!")
	}
	scanner.Split(scanLineWithEscape)
	activeSyntaxes := make(map[*syntax.Syntax]struct{}, len(syntaxes))
	// for running concurrently, we'll add a pool of worker goroutines
	numWorkers := runtime.GOMAXPROCS(0)
	// we need to wait for our goroutines to finish
	waitGroup := sync.WaitGroup{}
	waitGroup.Add(numWorkers)
	defer waitGroup.Wait()
	type Task struct {
		line   string
		syntax *syntax.Syntax
	}
	tasks := make(chan Task, numWorkers)
	defer close(tasks)
	for i := 0; i < numWorkers; i++ {
		go func() {
			defer waitGroup.Done()
			for task := range tasks {
				g.scanDependencies(task.line, task.syntax)
			}
		}()
	}
	for scanner.Scan() {
		if scanner.Err() != nil {
			return g, scanner.Err()
		}
		line := scanner.Text()
		for _, syntax := range syntaxes {
			if strings.Contains(line, syntax.GraphPrefix) {
				activeSyntaxes[syntax] = struct{}{}
			} else if _, active := activeSyntaxes[syntax]; !active {
				continue
			}
			if syntax.GraphSuffix != "" && strings.Contains(line, syntax.GraphSuffix) {
				delete(activeSyntaxes, syntax)
			}
			prefIndex := strings.Index(line, syntax.EdgePrefix)
			infixIndex := strings.Index(line, syntax.EdgeInfix)
			suffixIndex := strings.LastIndex(line, syntax.EdgeSuffix)
			if prefIndex >= 0 && infixIndex >= 0 && suffixIndex >= 0 {
				tasks <- Task{line[prefIndex+len(syntax.EdgePrefix) : suffixIndex], syntax}
				break
			}
		}
	}
	return g, nil
}

// String returns a simple string representation consisting of all edges.
func (g *Graph) String() string {
	if len(g.nodes) == 0 {
		return "{empty graph}"
	}
	var buffer bytes.Buffer
	for _, edge := range g.edges {
		buffer.WriteString(edge.String())
		buffer.WriteString("; ")
	}
	return buffer.String()
}

// scanLineWithEscape is a drop-in replacement for bufio.ScanLines, appending the next line if the last byte is a backslash '\'
func scanLineWithEscape(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanLines(data, atEOF)
	for err == nil && len(token) > 0 && token[len(token)-1] == '\\' {
		// omit the trailing backslash
		token = token[:len(token)-1]
		nextData := data[advance:]
		var nextAdvance int
		var nextToken []byte
		nextAdvance, nextToken, err = bufio.ScanLines(nextData, atEOF)
		advance += nextAdvance
		token = append(token, nextToken...)
	}
	return
}
<commit_msg>Add RemoveEdge method in Graph<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// Package graph contains the graph data structure to represent dependency graphs.
// Nodes are fmt.Stringers, and are uniquely identified by their String() method.
// Edges don't have a public interface, but the various graph.*Edge methods work with them.
package graph

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/SimplicityApks/depgrapher/syntax"
	"runtime"
	"strings"
	"sync"
)

// Node represents a single data point stored in a Graph. Its String() method should return a unique string identifier.
// The String() method call should be fast (ideally inline), as it will be called often!
type Node fmt.Stringer

// node is a simple string type, created only by the Graph input methods.
type node string

func (s node) String() string {
	return string(s)
}

type edge struct {
	source Node
	target Node
}

func (e *edge) String() string {
	return e.source.String() + " => " + e.target.String()
}

// Interface defines a basic graph-like data structure.
// The stored data is accessed by the String() method, so every Node needs to implement fmt.Stringer.
type Interface interface {
	// GetNode returns the Node with the specified name, or nil if the name couldn't be found in the graph.
	GetNode(name string) Node
	// GetNodes returns all Node objects saved in the graph as a slice. If the graph is empty, an empty slice is returned.
	GetNodes() []Node

	// AddEdge adds a new edge from source to target to the graph. The nodes are added to the graph if they were not present.
	AddEdge(source, target Node)
	// HasEdge returns true if the graph has an edge from the source Node to the target Node, false otherwise.
	HasEdge(source, target string) bool
	// RemoveEdge removes the edge from the source Node to the target Node.
	// The nodes are removed as well if they don't have any other dependencies or dependants.
	// Returns false if the graph didn't have an edge from source to target, true otherwise.
	RemoveEdge(source, target string) bool
	// GetDependencies returns a slice containing all dependencies of the Node with the given string.
	GetDependencies(node string) []Node
	// GetDependencies returns a slice containing all dependants of the Node with the given string.
	GetDependants(node string) []Node

	// Copy returns a deep copy of the graph.
	Copy() Interface
}

// Graph represents the dependency graph in memory. It internally uses a mutex to make read and write access concurrency-safe
// It satisfies the graph.Interface and fmt.Stringer interfaces.
// The zero value is an empty graph with no dependencies.
//
// This implementation prefers fast(-ish) node lookups over edge removals because it keeps a list of all nodes ready.
type Graph struct {
	nodes []Node
	edges []edge
	mutex sync.Mutex
}

// GetNode returns the Node with the specified name, or nil if the name couldn't be found in g.
func (g *Graph) GetNode(name string) Node {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, node := range g.nodes {
		if node.String() == name {
			return node
		}
	}
	return nil
}

// GetNodes returns all Node objects saved in the Graph as a slice. If it is empty, an empty slice is returned.
func (g *Graph) GetNodes() []Node {
	return g.nodes
}

// AddEdge adds a new edge from source to target to the Graph. The nodes are added to the Graph if they were not present.
func (g *Graph) AddEdge(source, target Node) {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	// add the nodes if they aren't already in place
	var foundSource, foundTarget bool
	for _, n := range g.nodes {
		if n.String() == source.String() {
			foundSource = true
		} else if n.String() == target.String() {
			foundTarget = true
		}
	}
	if !foundSource {
		g.nodes = append(g.nodes, source)
	}
	if !foundTarget {
		g.nodes = append(g.nodes, target)
	}
	g.edges = append(g.edges, edge{source: source, target: target})
}

// HasEdge returns true if g has an edge from the source Node to the target Node, false otherwise.
func (g *Graph) HasEdge(source, target string) bool {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, edge := range g.edges {
		if edge.source.String() == source && edge.target.String() == target {
			return true
		}
	}
	return false
}

// RemoveEdge removes the edge from the source Node to the target Node.
// The nodes are removed as well if they don't have any other dependencies or dependants.
// Returns false if the graph didn't have an edge from source to target, true otherwise.
func (g *Graph) RemoveEdge(source, target string) bool {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for index, e := range g.edges {
		if e.source.String() == source && e.target.String() == target {
			// delete without preserving order as it is faster
			last := len(g.edges) - 1
			g.edges[index] = g.edges[last]
			// set the last edge to nil so the nodes can be garbage collected
			g.edges[last] = edge{}
			g.edges = g.edges[:last]
			// remove the nodes as well if they have no reason to live any more
			if len(g.GetDependants(source)) == 0 && len(g.GetDependencies(source)) == 0 {
				g.deleteNode(e.source)
			}
			if len(g.GetDependants(target)) == 0 && len(g.GetDependencies(target)) == 0 {
				g.deleteNode(e.target)
			}
			return true
		}
	}
	return false
}

func (g *Graph) deleteNode(n Node) bool {
	for index, current := range g.nodes {
		if current == n {
			last := len(g.nodes) - 1
			g.nodes[index] = g.nodes[last]
			g.nodes = g.nodes[:last]
			return true
		}
	}
	return false
}

// GetDependencies returns a slice containing all dependencies of the Node with the given string.
// The Graph contains an edge from the Node to each item in the returned dependencies.
func (g *Graph) GetDependencies(node string) []Node {
	deps := make([]Node, 0)
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, edge := range g.edges {
		if edge.source.String() == node {
			deps = append(deps, edge.target)
		}
	}
	return deps
}

// GetDependencies returns a slice containing all dependants of the Node with the given string.
// The Graph contains an edge from each item in dependants to the Node.
func (g *Graph) GetDependants(node string) []Node {
	deps := make([]Node, 0)
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, edge := range g.edges {
		if edge.target.String() == node {
			deps = append(deps, edge.source)
		}
	}
	return deps
}

// Copy returns a deep copy of the Graph.
func (g *Graph) Copy() Interface {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	result := &Graph{
		nodes: make([]Node, len(g.nodes)),
		edges: make([]edge, len(g.edges)),
	}
	copy(result.nodes, g.nodes)
	copy(result.edges, g.edges)
	return result
}

// scanDependencies adds the given dependency line with the given syntax as edges to g.
func (g *Graph) scanDependencies(line string, syntax *syntax.Syntax) {
	infixIndex := strings.Index(line, syntax.EdgeInfix)
	sources := []string{line[:infixIndex]}
	if syntax.SourceDelimiter != "" {
		sources = strings.Split(line[:infixIndex], syntax.SourceDelimiter)
	}
	targets := strings.Split(line[infixIndex+len(syntax.EdgeInfix):], syntax.TargetDelimiter)
	for _, source := range sources {
		if syntax.StripWhitespace {
			source = strings.TrimSpace(source)
		}
		if source != "" {
			for _, target := range targets {
				if syntax.StripWhitespace {
					target = strings.TrimSpace(target)
				}
				if target != "" {
					g.AddEdge(node(source), node(target))
				}
			}
		}
	}
}

// FromScanner reads data from the given scanner, building up the dependency tree.
func (g *Graph) FromScanner(scanner *bufio.Scanner, syntaxes ...*syntax.Syntax) (*Graph, error) {
	if len(syntaxes) == 0 {
		panic("FromScanner: At least one syntax required!")
	}
	scanner.Split(scanLineWithEscape)
	activeSyntaxes := make(map[*syntax.Syntax]struct{}, len(syntaxes))
	// for running concurrently, we'll add a pool of worker goroutines
	numWorkers := runtime.GOMAXPROCS(0)
	// we need to wait for our goroutines to finish
	waitGroup := sync.WaitGroup{}
	waitGroup.Add(numWorkers)
	defer waitGroup.Wait()
	type Task struct {
		line   string
		syntax *syntax.Syntax
	}
	tasks := make(chan Task, numWorkers)
	defer close(tasks)
	for i := 0; i < numWorkers; i++ {
		go func() {
			defer waitGroup.Done()
			for task := range tasks {
				g.scanDependencies(task.line, task.syntax)
			}
		}()
	}
	for scanner.Scan() {
		if scanner.Err() != nil {
			return g, scanner.Err()
		}
		line := scanner.Text()
		for _, syntax := range syntaxes {
			if strings.Contains(line, syntax.GraphPrefix) {
				activeSyntaxes[syntax] = struct{}{}
			} else if _, active := activeSyntaxes[syntax]; !active {
				continue
			}
			if syntax.GraphSuffix != "" && strings.Contains(line, syntax.GraphSuffix) {
				delete(activeSyntaxes, syntax)
			}
			prefIndex := strings.Index(line, syntax.EdgePrefix)
			infixIndex := strings.Index(line, syntax.EdgeInfix)
			suffixIndex := strings.LastIndex(line, syntax.EdgeSuffix)
			if prefIndex >= 0 && infixIndex >= 0 && suffixIndex >= 0 {
				tasks <- Task{line[prefIndex+len(syntax.EdgePrefix) : suffixIndex], syntax}
				break
			}
		}
	}
	return g, nil
}

// String returns a simple string representation consisting of all edges.
func (g *Graph) String() string {
	if len(g.nodes) == 0 {
		return "{empty graph}"
	}
	var buffer bytes.Buffer
	for _, edge := range g.edges {
		buffer.WriteString(edge.String())
		buffer.WriteString("; ")
	}
	return buffer.String()
}

// scanLineWithEscape is a drop-in replacement for bufio.ScanLines, appending the next line if the last byte is a backslash '\'
func scanLineWithEscape(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanLines(data, atEOF)
	for err == nil && len(token) > 0 && token[len(token)-1] == '\\' {
		// omit the trailing backslash
		token = token[:len(token)-1]
		nextData := data[advance:]
		var nextAdvance int
		var nextToken []byte
		nextAdvance, nextToken, err = bufio.ScanLines(nextData, atEOF)
		advance += nextAdvance
		token = append(token, nextToken...)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package samples

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

// A file system with a fixed structure that looks like this:
//
//     hello
//     dir/
//         world
//
// Each file contains the string "Hello, world!".
type HelloFS struct {
	fuseutil.NotImplementedFileSystem
	Clock timeutil.Clock
}

var _ fuse.FileSystem = &HelloFS{}

func (fs *HelloFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	// We always allow opening the root directory.
	if req.Inode == fuse.RootInodeID {
		resp = &fuse.OpenDirResponse{}
		return
	}

	// TODO(jacobsa): Handle others.
	err = fuse.ENOSYS
	return
}
<commit_msg>Defined inode IDs.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package samples

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

// A file system with a fixed structure that looks like this:
//
//     hello
//     dir/
//         world
//
// Each file contains the string "Hello, world!".
type HelloFS struct {
	fuseutil.NotImplementedFileSystem
	Clock timeutil.Clock
}

var _ fuse.FileSystem = &HelloFS{}

const (
	rootInode fuse.InodeID = fuse.RootInodeID + iota
	helloInode
	dirInode
	worldInode
)

func (fs *HelloFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	// We always allow opening the root directory.
	if req.Inode == rootInode {
		resp = &fuse.OpenDirResponse{}
		return
	}

	// TODO(jacobsa): Handle others.
	err = fuse.ENOSYS
	return
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/png"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
)

type File struct {
	Name string
	Size int64
}

// LimitedDir tracks child files of a directory and ensure there are at most
// maxCount of them or the total size is less than maxSize. Otherwise, oldest
// one are deleted until the conditions are matched. LimitedDir can be used
// concurrently.
//
// Known limitations:
// - Adding an existing file count as a new one. This is not a problem in
//   gribouillis as saved drawings always carry new name, and if they do not, the
//   LimitedDir will be a bit more punitive.
// - Empty files are tolerated. Again, not a problem since gribouillis store
//   valid PNG files.
type LimitedDir struct {
	path     string
	maxSize  int64
	maxCount int
	lock     sync.Mutex
	files    []File
	size     int64
}

type sortedFiles []os.FileInfo

func (s sortedFiles) Len() int {
	return len(s)
}

func (s sortedFiles) Less(i, j int) bool {
	ti := s[i].ModTime()
	tj := s[j].ModTime()
	return ti != tj && tj.After(ti)
}

func (s sortedFiles) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

// OpenLimitedDir returns a LimitedDir initialized on supplied directory.
func OpenLimitedDir(path string, maxSize int64, maxCount int) (*LimitedDir, error) {
	err := os.MkdirAll(path, 755)
	if err != nil {
		return nil, err
	}
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	sort.Sort(sortedFiles(entries))
	files := make([]File, len(entries))
	total := int64(0)
	for i, e := range entries {
		if !e.Mode().IsRegular() {
			continue
		}
		files[i] = File{
			Name: e.Name(),
			Size: e.Size(),
		}
		total += files[i].Size
	}
	d := &LimitedDir{
		path:     path,
		maxCount: maxCount,
		files:    files,
		size:     total,
		maxSize:  maxSize,
	}
	err = d.shrink()
	if err != nil {
		return nil, err
	}
	return d, err
}

func (d *LimitedDir) Path() string {
	return d.path
}

func (d *LimitedDir) shrink() error {
	for (d.size > d.maxSize && len(d.files) > 0) || len(d.files) > d.maxCount {
		f := d.files[0]
		p := filepath.Join(d.path, f.Name)
		log.Printf("removing %s", f.Name)
		err := os.Remove(p)
		if err != nil && !os.IsNotExist(err) {
			return err
		} else if err == nil {
			d.size -= f.Size
		}
		d.files = d.files[1:]
	}
	return nil
}

// Add registers a new file in the LimitedDir and applies the maxCount/maxSize
// policy. Note that adding an existing files works like adding a new one.
func (d *LimitedDir) Add(name string) error {
	path := filepath.Join(d.path, name)
	st, err := os.Stat(path)
	if err != nil {
		return err
	}

	d.lock.Lock()
	defer d.lock.Unlock()
	d.files = append(d.files, File{
		Name: name,
		Size: st.Size(),
	})
	d.size += st.Size()
	return d.shrink()
}

// List returns the list of tracked files in deletion order.
func (d *LimitedDir) List() []string {
	d.lock.Lock()
	defer d.lock.Unlock()
	names := []string{}
	for _, f := range d.files {
		names = append(names, f.Name)
	}
	return names
}

// fixImage decode input data as PNG, pad it with white at each borders and
// write it again as PNG on output write.
func fixImage(w io.Writer, r io.Reader, padding int) error {
	src, err := png.Decode(r)
	if err != nil {
		return err
	}
	srcRect := src.Bounds()
	dstRect := image.Rect(srcRect.Min.X-padding, srcRect.Min.Y-padding,
		srcRect.Max.X+padding, srcRect.Max.Y+padding)
	dst := image.NewRGBA(dstRect)
	white := color.RGBA{255, 255, 255, 255}
	for j := dstRect.Min.Y; j < dstRect.Max.Y; j++ {
		for i := dstRect.Min.X; i < dstRect.Max.X; i++ {
			if i >= srcRect.Min.X && i < srcRect.Max.X &&
				j >= srcRect.Min.Y && j < srcRect.Max.Y {
				dst.Set(i, j, src.At(i, j))
			} else {
				dst.Set(i, j, white)
			}
		}
	}
	return png.Encode(w, dst)
}

// save decode posted PNG and save it with a random name into imgDir. It returns
// a JSON response with the absolute path of the saved image.
func save(imgURL string, imgDir *LimitedDir, maxImgSize int64, w http.ResponseWriter,
	r *http.Request) error {

	buf := make([]byte, 16)
	_, err := rand.Read(buf)
	if err != nil {
		return err
	}
	name := fmt.Sprintf("%x", buf) + ".png"
	path := filepath.Join(imgDir.Path(), name)
	log.Printf("writing %s", path)
	fp, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		if fp != nil {
			fp.Close()
			os.Remove(path)
		}
	}()

	err = fixImage(fp, &io.LimitedReader{
		R: r.Body,
		N: int64(maxImgSize),
	}, 20)
	if err != nil {
		return err
	}
	err = fp.Close()
	if err != nil {
		return err
	}
	fp = nil
	err = imgDir.Add(name)
	if err != nil {
		return err
	}
	rsp := struct {
		Path string `json:"path"`
	}{
		Path: imgURL + name,
	}
	w.Header().Set("Content-Type", "image/png")
	return json.NewEncoder(w).Encode(&rsp)
}

func gribouillis() error {
	flag.Usage = func() {
		fmt.Println(`Usage: gribouillis [OPTIONS]

gribouillis starts a web server on -http and exposes a "literallycanvas" web
drawing canvas on root URL. Saved images are serialized on disk in "images/"
relatively to the working directory and accessible with random URLs in "saved/"
subpath.

Use -base-url to set the web server base URL (useful when proxying).
`)
		os.Exit(1)
	}
	addr := flag.String("http", "localhost:5001", "HTTP host:port")
	baseURL := flag.String("base-url", "", "web server base URL")
	maxImgSizeStr := flag.String("max-image-size", "10MB", "maximum image size")
	minDelayStr := flag.String("min-delay", "5s", "minimum delay between two records")
	maxSizeStr := flag.String("max-size", "50MB",
		"maximum combined size of saved drawings")
	maxCount := flag.Int("max-count", 500, "maximum number of saved drawings")
	flag.Parse()
	if flag.NArg() != 0 {
		return fmt.Errorf("no argument expected")
	}
	trimmed := strings.TrimRight(*baseURL, "/")
	baseURL = &trimmed
	maxImgSize, err := humanize.ParseBytes(*maxImgSizeStr)
	if err != nil {
		return err
	}
	maxSize, err := humanize.ParseBytes(*maxSizeStr)
	if err != nil {
		return err
	}
	minDelay, err := time.ParseDuration(*minDelayStr)
	if err != nil {
		return err
	}
	lastTimeMutex := sync.Mutex{}
	lastTime := time.Now()

	imgURL := *baseURL + "/saved/"
	imgDir, err := OpenLimitedDir("images", int64(maxSize), *maxCount)
	if err != nil {
		return err
	}
	http.Handle(imgURL, http.StripPrefix(imgURL,
		http.FileServer(http.Dir(imgDir.Path()))))
	http.HandleFunc(*baseURL+"/save/", func(w http.ResponseWriter, r *http.Request) {
		now := time.Now()
		lastTimeMutex.Lock()
		last := lastTime
		lastTimeMutex.Unlock()
		if now.Sub(last) < minDelay {
			log.Printf("rate limited")
			w.WriteHeader(429)
			w.Write([]byte("rate limited"))
			return
		}
		lastTimeMutex.Lock()
		lastTime = now
		lastTimeMutex.Unlock()

		err := save(imgURL, imgDir, int64(maxImgSize), w, r)
		if err != nil {
			log.Printf("save error: %s", err)
			w.WriteHeader(500)
			w.Write([]byte(fmt.Sprintf("could not save image: %s", err)))
		}
	})
	http.Handle(*baseURL+"/", http.StripPrefix(*baseURL+"/",
		http.FileServer(http.Dir("literallycanvas"))))
	return http.ListenAndServe(*addr, nil)
}

func main() {
	err := gribouillis()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>gribouillis: make --help print options<commit_after>package main

import (
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/png"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
)

type File struct {
	Name string
	Size int64
}

// LimitedDir tracks child files of a directory and ensure there are at most
// maxCount of them or the total size is less than maxSize. Otherwise, oldest
// one are deleted until the conditions are matched. LimitedDir can be used
// concurrently.
//
// Known limitations:
// - Adding an existing file count as a new one. This is not a problem in
//   gribouillis as saved drawings always carry new name, and if they do not, the
//   LimitedDir will be a bit more punitive.
// - Empty files are tolerated. Again, not a problem since gribouillis store
//   valid PNG files.
type LimitedDir struct {
	path     string
	maxSize  int64
	maxCount int
	lock     sync.Mutex
	files    []File
	size     int64
}

type sortedFiles []os.FileInfo

func (s sortedFiles) Len() int {
	return len(s)
}

func (s sortedFiles) Less(i, j int) bool {
	ti := s[i].ModTime()
	tj := s[j].ModTime()
	return ti != tj && tj.After(ti)
}

func (s sortedFiles) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

// OpenLimitedDir returns a LimitedDir initialized on supplied directory.
func OpenLimitedDir(path string, maxSize int64, maxCount int) (*LimitedDir, error) {
	err := os.MkdirAll(path, 755)
	if err != nil {
		return nil, err
	}
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	sort.Sort(sortedFiles(entries))
	files := make([]File, len(entries))
	total := int64(0)
	for i, e := range entries {
		if !e.Mode().IsRegular() {
			continue
		}
		files[i] = File{
			Name: e.Name(),
			Size: e.Size(),
		}
		total += files[i].Size
	}
	d := &LimitedDir{
		path:     path,
		maxCount: maxCount,
		files:    files,
		size:     total,
		maxSize:  maxSize,
	}
	err = d.shrink()
	if err != nil {
		return nil, err
	}
	return d, err
}

func (d *LimitedDir) Path() string {
	return d.path
}

func (d *LimitedDir) shrink() error {
	for (d.size > d.maxSize && len(d.files) > 0) || len(d.files) > d.maxCount {
		f := d.files[0]
		p := filepath.Join(d.path, f.Name)
		log.Printf("removing %s", f.Name)
		err := os.Remove(p)
		if err != nil && !os.IsNotExist(err) {
			return err
		} else if err == nil {
			d.size -= f.Size
		}
		d.files = d.files[1:]
	}
	return nil
}

// Add registers a new file in the LimitedDir and applies the maxCount/maxSize
// policy. Note that adding an existing files works like adding a new one.
func (d *LimitedDir) Add(name string) error {
	path := filepath.Join(d.path, name)
	st, err := os.Stat(path)
	if err != nil {
		return err
	}

	d.lock.Lock()
	defer d.lock.Unlock()
	d.files = append(d.files, File{
		Name: name,
		Size: st.Size(),
	})
	d.size += st.Size()
	return d.shrink()
}

// List returns the list of tracked files in deletion order.
func (d *LimitedDir) List() []string {
	d.lock.Lock()
	defer d.lock.Unlock()
	names := []string{}
	for _, f := range d.files {
		names = append(names, f.Name)
	}
	return names
}

// fixImage decode input data as PNG, pad it with white at each borders and
// write it again as PNG on output write.
func fixImage(w io.Writer, r io.Reader, padding int) error {
	src, err := png.Decode(r)
	if err != nil {
		return err
	}
	srcRect := src.Bounds()
	dstRect := image.Rect(srcRect.Min.X-padding, srcRect.Min.Y-padding,
		srcRect.Max.X+padding, srcRect.Max.Y+padding)
	dst := image.NewRGBA(dstRect)
	white := color.RGBA{255, 255, 255, 255}
	for j := dstRect.Min.Y; j < dstRect.Max.Y; j++ {
		for i := dstRect.Min.X; i < dstRect.Max.X; i++ {
			if i >= srcRect.Min.X && i < srcRect.Max.X &&
				j >= srcRect.Min.Y && j < srcRect.Max.Y {
				dst.Set(i, j, src.At(i, j))
			} else {
				dst.Set(i, j, white)
			}
		}
	}
	return png.Encode(w, dst)
}

// save decode posted PNG and save it with a random name into imgDir. It returns
// a JSON response with the absolute path of the saved image.
func save(imgURL string, imgDir *LimitedDir, maxImgSize int64, w http.ResponseWriter,
	r *http.Request) error {

	buf := make([]byte, 16)
	_, err := rand.Read(buf)
	if err != nil {
		return err
	}
	name := fmt.Sprintf("%x", buf) + ".png"
	path := filepath.Join(imgDir.Path(), name)
	log.Printf("writing %s", path)
	fp, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		if fp != nil {
			fp.Close()
			os.Remove(path)
		}
	}()

	err = fixImage(fp, &io.LimitedReader{
		R: r.Body,
		N: int64(maxImgSize),
	}, 20)
	if err != nil {
		return err
	}
	err = fp.Close()
	if err != nil {
		return err
	}
	fp = nil
	err = imgDir.Add(name)
	if err != nil {
		return err
	}
	rsp := struct {
		Path string `json:"path"`
	}{
		Path: imgURL + name,
	}
	w.Header().Set("Content-Type", "image/png")
	return json.NewEncoder(w).Encode(&rsp)
}

func gribouillis() error {
	flag.Usage = func() {
		fmt.Println(`Usage: gribouillis [OPTIONS]

gribouillis starts a web server on -http and exposes a "literallycanvas" web
drawing canvas on root URL. Saved images are serialized on disk in "images/"
relatively to the working directory and accessible with random URLs in "saved/"
subpath.

Use -base-url to set the web server base URL (useful when proxying).
`)
		flag.PrintDefaults()
		os.Exit(1)
	}
	addr := flag.String("http", "localhost:5001", "HTTP host:port")
	baseURL := flag.String("base-url", "", "web server base URL")
	maxImgSizeStr := flag.String("max-image-size", "10MB", "maximum image size")
	minDelayStr := flag.String("min-delay", "5s", "minimum delay between two records")
	maxSizeStr := flag.String("max-size", "50MB",
		"maximum combined size of saved drawings")
	maxCount := flag.Int("max-count", 500, "maximum number of saved drawings")
	flag.Parse()
	if flag.NArg() != 0 {
		return fmt.Errorf("no argument expected")
	}
	trimmed := strings.TrimRight(*baseURL, "/")
	baseURL = &trimmed
	maxImgSize, err := humanize.ParseBytes(*maxImgSizeStr)
	if err != nil {
		return err
	}
	maxSize, err := humanize.ParseBytes(*maxSizeStr)
	if err != nil {
		return err
	}
	minDelay, err := time.ParseDuration(*minDelayStr)
	if err != nil {
		return err
	}
	lastTimeMutex := sync.Mutex{}
	lastTime := time.Now()

	imgURL := *baseURL + "/saved/"
	imgDir, err := OpenLimitedDir("images", int64(maxSize), *maxCount)
	if err != nil {
		return err
	}
	http.Handle(imgURL, http.StripPrefix(imgURL,
		http.FileServer(http.Dir(imgDir.Path()))))
	http.HandleFunc(*baseURL+"/save/", func(w http.ResponseWriter, r *http.Request) {
		now := time.Now()
		lastTimeMutex.Lock()
		last := lastTime
		lastTimeMutex.Unlock()
		if now.Sub(last) < minDelay {
			log.Printf("rate limited")
			w.WriteHeader(429)
			w.Write([]byte("rate limited"))
			return
		}
		lastTimeMutex.Lock()
		lastTime = now
		lastTimeMutex.Unlock()

		err := save(imgURL, imgDir, int64(maxImgSize), w, r)
		if err != nil {
			log.Printf("save error: %s", err)
			w.WriteHeader(500)
			w.Write([]byte(fmt.Sprintf("could not save image: %s", err)))
		}
	})
	http.Handle(*baseURL+"/", http.StripPrefix(*baseURL+"/",
		http.FileServer(http.Dir("literallycanvas"))))
	return http.ListenAndServe(*addr, nil)
}

func main() {
	err := gribouillis()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package scheduler

import (
	"sync/atomic"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/mesos"
)

type OfferFilter func(*mesos.Offer) bool

type OfferRegistry interface {
	// Initialize the instance, spawning necessary housekeeping go routines.
	Init()
	Add([]*mesos.Offer)
	AwaitNew(id string, f OfferFilter) <-chan empty

	// invoked when offers are rescinded or expired
	Delete(string)

	Get(offerId string) (PerishableOffer, bool)

	Walk(Walker) error

	// invalidate one or all (when offerId="") offers; offers are not declined,
	// but are simply flagged as expired in the offer history
	Invalidate(offerId string)
}

// callback that is invoked during a walk through a series of active offers,
// returning with stop=true when the walk should stop.
type Walker func(offer PerishableOffer) (stop bool, err error)

type OfferRegistryConfig struct {
	declineOffer  func(offerId string) error
	ttl           time.Duration // determines a perishable offer's expiration deadline: now+ttl
	lingerTtl     time.Duration // if zero, offers will not linger in the FIFO past their expiration deadline
	walkDelay     time.Duration // specifies the sleep time between expired offer harvests
	listenerDelay time.Duration // specifies the sleep time between offer listener notifications
}

type offerStorage struct {
	OfferRegistryConfig
	offers    *cache.FIFO // collection of timedOffer's
	listeners *cache.FIFO // collection of <-chan empty
}

type timedOffer struct {
	*mesos.Offer
	expiration time.Time
	acquired   int32 // 1 = acquired, 0 = free
}

type expiredOffer struct{}

type PerishableOffer interface {
	// returns true if this offer has expired
	hasExpired() bool
	// if not yet expired, return mesos offer details; otherwise nil
	details() *mesos.Offer
	// mark this offer as acquired, returning true if it was previously unacquired. thread-safe.
	acquire() bool
	// mark this offer as un-acquired. thread-safe.
	release()
}

func (e *expiredOffer) hasExpired() bool {
	return true
}

func (e *expiredOffer) details() *mesos.Offer {
	return nil
}

func (e *expiredOffer) acquire() bool {
	return false
}

func (e *expiredOffer) release() {}

func (to *timedOffer) hasExpired() bool {
	return time.Now().After(to.expiration)
}

func (to *timedOffer) details() *mesos.Offer {
	return to.Offer
}

func (to *timedOffer) acquire() bool {
	return atomic.CompareAndSwapInt32(&to.acquired, 0, 1) && !to.hasExpired()
}

func (to *timedOffer) release() {
	atomic.CompareAndSwapInt32(&to.acquired, 1, 0)
}

func CreateOfferRegistry(c OfferRegistryConfig) OfferRegistry {
	return &offerStorage{c, cache.NewFIFO(), cache.NewFIFO()}
}

func (s *offerStorage) Add(offers []*mesos.Offer) {
	now := time.Now()
	for _, offer := range offers {
		offerId := offer.Id.GetValue()
		log.V(3).Infof("Receiving offer %v", offerId)
		s.offers.Add(offerId, &timedOffer{offer, now.Add(s.ttl), 0})
	}
}

// delete an offer from storage, meaning that we expire it
func (s *offerStorage) Delete(offerId string) {
	if offer, ok := s.Get(offerId); ok {
		log.V(3).Infof("Deleting offer %v", offerId)
		// attempt to block others from consuming the offer. if it's already been
		// claimed and is not yet lingering then don't decline it - just mark it as
		// expired in the history: allow a prior claimant to attempt to launch with it
		myoffer := offer.acquire()
		if offer.details() != nil && myoffer {
			log.V(3).Infof("Declining offer %v", offerId)
			if err := s.declineOffer(offerId); err != nil {
				log.Warningf("Failed to decline offer %v: %v", offerId, err)
			}
		}
		s.expireOffer(offer)
	} // else, ignore offers not in the history
}

// expire all known, live offers
func (s *offerStorage) Invalidate(offerId string) {
	if offerId != "" {
		s.invalidateOne(offerId)
		return
	}
	obj := s.offers.List()
	for _, o := range obj {
		offer, ok := o.(PerishableOffer)
		if !ok {
			log.Errorf("Expected perishable offer, not %v", o)
			continue
		}
		offer.acquire() // attempt to block others from using it
		s.expireOffer(offer)
		// don't reject it from the controller, we already know that it's an invalid offer
	}
}

func (s *offerStorage) invalidateOne(offerId string) {
	if offer, ok := s.Get(offerId); ok {
		offer.acquire() // attempt to block others from using it
		s.expireOffer(offer)
		// don't reject it from the controller, we already know that it's an invalid offer
	}
}

// Walk the collection of offers, Delete()ing those that have expired,
// visiting those that haven't with the Walked. The walk stops either as
// indicated by the Walker or when the end of the offer list is reached.
func (s *offerStorage) Walk(w Walker) error {
	for offerId := range s.offers.Contains() {
		offer, ok := s.Get(offerId)
		if !ok {
			// offer disappeared...
			continue
		}
		// delete any non-lingering offers that have expired
		if offer.hasExpired() && offer.details() != nil {
			s.Delete(offerId)
			continue
		}
		if stop, err := w(offer); err != nil {
			return err
		} else if stop {
			return nil
		}
	}
	return nil
}

func (s *offerStorage) expireOffer(offer PerishableOffer) {
	// the offer may or may not be expired due to TTL so check for details
	// since that's a more reliable determinant of lingering status
	if details := offer.details(); details != nil {
		// recently expired, should linger
		offerId := details.Id.GetValue()
		log.V(3).Infof("Expiring offer %v", offerId)
		if s.lingerTtl > 0 {
			log.V(3).Infof("offer will linger: %v", offerId)
			s.offers.Update(offerId, &expiredOffer{})
			go func() {
				time.Sleep(s.lingerTtl)
				log.V(3).Infof("Delete lingering offer: %v", offerId)
				s.offers.Delete(offerId)
			}()
		} else {
			log.V(3).Infof("Permanently deleting offer %v", offerId)
			s.offers.Delete(offerId)
		}
	} // else, it's still lingering...
}

func (s *offerStorage) Get(id string) (PerishableOffer, bool) {
	if obj, ok := s.offers.Get(id); !ok {
		return nil, false
	} else {
		to, ok := obj.(PerishableOffer)
		if !ok {
			log.Errorf("invalid offer object in fifo '%v'", obj)
		}
		return to, ok
	}
}

type offerListener struct {
	id     string
	filter OfferFilter
	notify chan<- empty
	age    int
}

// register a listener for new offers, whom we'll notify upon receiving such.
func (s *offerStorage) AwaitNew(id string, f OfferFilter) <-chan empty {
	if f == nil {
		return nil
	}
	ch := make(chan empty, 1)
	listen := &offerListener{
		id:     id,
		filter: f,
		notify: ch,
		age:    0,
	}
	log.V(3).Infof("Registering offer listener %s", listen.id)
	s.listeners.Add(id, listen)
	return ch
}

func (s *offerStorage) makeOfferHarvestFunc() func() {
	return func() {
		s.Walk(func(PerishableOffer) (bool, error) {
			// noop; simply walking will expire offers past their TTL
			return false, nil
		})
	}
}

func (s *offerStorage) makeOfferListenerFunc() func() {
	return func() {
		var listen *offerListener
		var ok bool
		// get the next offer listener
		for {
			obj := s.listeners.Pop()
			if listen, ok = obj.(*offerListener); ok {
				break
			}
			log.Warningf("unexpected listener object %v", obj)
		}
		// notify if we find an acceptable offer
		for id := range s.offers.Contains() {
			var offer PerishableOffer
			if offer, ok = s.Get(id); !ok || offer.hasExpired() {
				continue
			}
			if listen.filter(offer.details()) {
				log.V(3).Infof("Notifying offer listener %s", listen.id)
				listen.notify <- empty{}
				return
			}
		}
		// no interesting offers found, re-queue the listener
		listen.age++
		if listen.age < 20 {
			// if the same listener has re-registered in the meantime we don't want to
			// destroy the newer listener channel. this is racy since a newer listener
			// can register between the Get() and Update(), but the consequences aren't
			// very dire - the listener merely has to wait their full backoff period.
			if _, ok := s.listeners.Get(listen.id); !ok {
				log.V(3).Infof("Re-registering offer listener %s", listen.id)
				s.listeners.Update(listen.id, listen)
			}
		} // else, you're gc'd
	}
}

func (s *offerStorage) Init() {
	go util.Forever(s.makeOfferHarvestFunc(), s.walkDelay)

	// periodically gc the listeners FIFO.
	// to avoid a rush on offers, we add a short delay between each registered
	// listener, so as to allow the most recently notified listener a bit of time
	// to act on the offer.
	go util.Forever(s.makeOfferListenerFunc(), s.listenerDelay)
}
<commit_msg>fix bug in acquire for timed offer that checked for expiration<commit_after>package scheduler

import (
	"sync/atomic"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/mesos"
)

type OfferFilter func(*mesos.Offer) bool

type OfferRegistry interface {
	// Initialize the instance, spawning necessary housekeeping go routines.
	Init()
	Add([]*mesos.Offer)
	AwaitNew(id string, f OfferFilter) <-chan empty

	// invoked when offers are rescinded or expired
	Delete(string)

	Get(offerId string) (PerishableOffer, bool)

	Walk(Walker) error

	// invalidate one or all (when offerId="") offers; offers are not declined,
	// but are simply flagged as expired in the offer history
	Invalidate(offerId string)
}

// callback that is invoked during a walk through a series of active offers,
// returning with stop=true when the walk should stop.
type Walker func(offer PerishableOffer) (stop bool, err error)

type OfferRegistryConfig struct {
	declineOffer  func(offerId string) error
	ttl           time.Duration // determines a perishable offer's expiration deadline: now+ttl
	lingerTtl     time.Duration // if zero, offers will not linger in the FIFO past their expiration deadline
	walkDelay     time.Duration // specifies the sleep time between expired offer harvests
	listenerDelay time.Duration // specifies the sleep time between offer listener notifications
}

type offerStorage struct {
	OfferRegistryConfig
	offers    *cache.FIFO // collection of timedOffer's
	listeners *cache.FIFO // collection of <-chan empty
}

type timedOffer struct {
	*mesos.Offer
	expiration time.Time
	acquired   int32 // 1 = acquired, 0 = free
}

type expiredOffer struct{}

type PerishableOffer interface {
	// returns true if this offer has expired
	hasExpired() bool
	// if not yet expired, return mesos offer details; otherwise nil
	details() *mesos.Offer
	// mark this offer as acquired, returning true if it was previously unacquired. thread-safe.
	acquire() bool
	// mark this offer as un-acquired. thread-safe.
	release()
}

func (e *expiredOffer) hasExpired() bool {
	return true
}

func (e *expiredOffer) details() *mesos.Offer {
	return nil
}

func (e *expiredOffer) acquire() bool {
	return false
}

func (e *expiredOffer) release() {}

func (to *timedOffer) hasExpired() bool {
	return time.Now().After(to.expiration)
}

func (to *timedOffer) details() *mesos.Offer {
	return to.Offer
}

func (to *timedOffer) acquire() bool {
	return atomic.CompareAndSwapInt32(&to.acquired, 0, 1)
}

func (to *timedOffer) release() {
	atomic.CompareAndSwapInt32(&to.acquired, 1, 0)
}

func CreateOfferRegistry(c OfferRegistryConfig) OfferRegistry {
	return &offerStorage{c, cache.NewFIFO(), cache.NewFIFO()}
}

func (s *offerStorage) Add(offers []*mesos.Offer) {
	now := time.Now()
	for _, offer := range offers {
		offerId := offer.Id.GetValue()
		log.V(3).Infof("Receiving offer %v", offerId)
		s.offers.Add(offerId, &timedOffer{offer, now.Add(s.ttl), 0})
	}
}

// delete an offer from storage, meaning that we expire it
func (s *offerStorage) Delete(offerId string) {
	if offer, ok := s.Get(offerId); ok {
		log.V(3).Infof("Deleting offer %v", offerId)
		// attempt to block others from consuming the offer. if it's already been
		// claimed and is not yet lingering then don't decline it - just mark it as
		// expired in the history: allow a prior claimant to attempt to launch with it
		myoffer := offer.acquire()
		if offer.details() != nil && myoffer {
			log.V(3).Infof("Declining offer %v", offerId)
			if err := s.declineOffer(offerId); err != nil {
				log.Warningf("Failed to decline offer %v: %v", offerId, err)
			}
		}
		s.expireOffer(offer)
	} // else, ignore offers not in the history
}

// expire all known, live offers
func (s *offerStorage) Invalidate(offerId string) {
	if offerId != "" {
		s.invalidateOne(offerId)
		return
	}
	obj := s.offers.List()
	for _, o := range obj {
		offer, ok := o.(PerishableOffer)
		if !ok {
			log.Errorf("Expected perishable offer, not %v", o)
			continue
		}
		offer.acquire() // attempt to block others from using it
		s.expireOffer(offer)
		// don't reject it from the controller, we already know that it's an invalid offer
	}
}

func (s *offerStorage) invalidateOne(offerId string) {
	if offer, ok := s.Get(offerId); ok {
		offer.acquire() // attempt to block others from using it
		s.expireOffer(offer)
		// don't reject it from the controller, we already know that it's an invalid offer
	}
}

// Walk the collection of offers, Delete()ing those that have expired,
// visiting those that haven't with the Walked. The walk stops either as
// indicated by the Walker or when the end of the offer list is reached.
func (s *offerStorage) Walk(w Walker) error {
	for offerId := range s.offers.Contains() {
		offer, ok := s.Get(offerId)
		if !ok {
			// offer disappeared...
			continue
		}
		// delete any non-lingering offers that have expired
		if offer.hasExpired() && offer.details() != nil {
			s.Delete(offerId)
			continue
		}
		if stop, err := w(offer); err != nil {
			return err
		} else if stop {
			return nil
		}
	}
	return nil
}

func (s *offerStorage) expireOffer(offer PerishableOffer) {
	// the offer may or may not be expired due to TTL so check for details
	// since that's a more reliable determinant of lingering status
	if details := offer.details(); details != nil {
		// recently expired, should linger
		offerId := details.Id.GetValue()
		log.V(3).Infof("Expiring offer %v", offerId)
		if s.lingerTtl > 0 {
			log.V(3).Infof("offer will linger: %v", offerId)
			s.offers.Update(offerId, &expiredOffer{})
			go func() {
				time.Sleep(s.lingerTtl)
				log.V(3).Infof("Delete lingering offer: %v", offerId)
				s.offers.Delete(offerId)
			}()
		} else {
			log.V(3).Infof("Permanently deleting offer %v", offerId)
			s.offers.Delete(offerId)
		}
	} // else, it's still lingering...
}

func (s *offerStorage) Get(id string) (PerishableOffer, bool) {
	if obj, ok := s.offers.Get(id); !ok {
		return nil, false
	} else {
		to, ok := obj.(PerishableOffer)
		if !ok {
			log.Errorf("invalid offer object in fifo '%v'", obj)
		}
		return to, ok
	}
}

type offerListener struct {
	id     string
	filter OfferFilter
	notify chan<- empty
	age    int
}

// register a listener for new offers, whom we'll notify upon receiving such.
func (s *offerStorage) AwaitNew(id string, f OfferFilter) <-chan empty {
	if f == nil {
		return nil
	}
	ch := make(chan empty, 1)
	listen := &offerListener{
		id:     id,
		filter: f,
		notify: ch,
		age:    0,
	}
	log.V(3).Infof("Registering offer listener %s", listen.id)
	s.listeners.Add(id, listen)
	return ch
}

func (s *offerStorage) makeOfferHarvestFunc() func() {
	return func() {
		s.Walk(func(PerishableOffer) (bool, error) {
			// noop; simply walking will expire offers past their TTL
			return false, nil
		})
	}
}

func (s *offerStorage) makeOfferListenerFunc() func() {
	return func() {
		var listen *offerListener
		var ok bool
		// get the next offer listener
		for {
			obj := s.listeners.Pop()
			if listen, ok = obj.(*offerListener); ok {
				break
			}
			log.Warningf("unexpected listener object %v", obj)
		}
		// notify if we find an acceptable offer
		for id := range s.offers.Contains() {
			var offer PerishableOffer
			if offer, ok = s.Get(id); !ok || offer.hasExpired() {
				continue
			}
			if listen.filter(offer.details()) {
				log.V(3).Infof("Notifying offer listener %s", listen.id)
				listen.notify <- empty{}
				return
			}
		}
		// no interesting offers found, re-queue the listener
		listen.age++
		if listen.age < 20 {
			// if the same listener has re-registered in the meantime we don't want to
			// destroy the newer listener channel. this is racy since a newer listener
			// can register between the Get() and Update(), but the consequences aren't
			// very dire - the listener merely has to wait their full backoff period.
			if _, ok := s.listeners.Get(listen.id); !ok {
				log.V(3).Infof("Re-registering offer listener %s", listen.id)
				s.listeners.Update(listen.id, listen)
			}
		} // else, you're gc'd
	}
}

func (s *offerStorage) Init() {
	go util.Forever(s.makeOfferHarvestFunc(), s.walkDelay)

	// periodically gc the listeners FIFO.
	// to avoid a rush on offers, we add a short delay between each registered
	// listener, so as to allow the most recently notified listener a bit of time
	// to act on the offer.
	go util.Forever(s.makeOfferListenerFunc(), s.listenerDelay)
}
<|endoftext|>