<commit_before>package app

import (
	"github.com/abustany/gollery/monitor"
	"github.com/abustany/gollery/thumbnailer"
	"github.com/robfig/revel"
	"github.com/robfig/revel/modules/jobs/app/jobs"
	"runtime"
)

const (
	GOLLERY_CONFIG_ROOT_DIR  = "gollery.root_dir"
	GOLLERY_CONFIG_CACHE_DIR = "gollery.cache_dir"
)

var Monitor *monitor.Monitor
var Thumbnailer *thumbnailer.Thumbnailer

func init() {
	// Filters is the default set of global filters.
	revel.Filters = []revel.Filter{
		revel.PanicFilter,             // Recover from panics and display an error page instead.
		revel.RouterFilter,            // Use the routing table to select the right Action
		revel.FilterConfiguringFilter, // A hook for adding or removing per-Action filters.
		revel.ParamsFilter,            // Parse parameters into Controller.Params.
		revel.SessionFilter,           // Restore and write the session cookie.
		revel.FlashFilter,             // Restore and write the flash cookie.
		revel.ValidationFilter,        // Restore kept validation errors and save new ones from cookie.
		revel.I18nFilter,              // Resolve the requested language
		revel.InterceptorFilter,       // Run interceptors around the action.
		revel.ActionInvoker,           // Invoke the action.
	}

	revel.OnAppStart(func() {
		initServices()
	})

	// Use as many threads as we have CPUs, helps for thumbnailing
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func initServices() {
	rootdir, found := revel.Config.String(GOLLERY_CONFIG_ROOT_DIR)

	if !found {
		panic("Missing configuration parameter: " + GOLLERY_CONFIG_ROOT_DIR)
	}

	cachedir, found := revel.Config.String(GOLLERY_CONFIG_CACHE_DIR)

	if !found {
		panic("Missing configuration parameter: " + GOLLERY_CONFIG_CACHE_DIR)
	}

	var err error

	Monitor, err = monitor.NewMonitor()

	if err != nil {
		panic("Cannot initalize file monitoring: " + err.Error())
	}

	Monitor.Watch(rootdir)

	Thumbnailer, err = thumbnailer.NewThumbnailer(rootdir, cachedir, Monitor)

	if err != nil {
		panic("Cannot initialize thumbnailer service: " + err.Error())
	}

	revel.INFO.Print("Started thumbnailing service")

	jobs.Now(jobs.Func(func() {
		err := Thumbnailer.CheckCache()

		if err != nil {
			revel.ERROR.Printf("Cannot check the thumbnail cache: %s", err)
		}
	}))
}
<commit_msg>app: Export RootDir to other modules<commit_after>package app

import (
	"github.com/abustany/gollery/monitor"
	"github.com/abustany/gollery/thumbnailer"
	"github.com/robfig/revel"
	"github.com/robfig/revel/modules/jobs/app/jobs"
	"runtime"
)

const (
	GOLLERY_CONFIG_ROOT_DIR  = "gollery.root_dir"
	GOLLERY_CONFIG_CACHE_DIR = "gollery.cache_dir"
)

var RootDir string
var Monitor *monitor.Monitor
var Thumbnailer *thumbnailer.Thumbnailer

func init() {
	// Filters is the default set of global filters.
	revel.Filters = []revel.Filter{
		revel.PanicFilter,             // Recover from panics and display an error page instead.
		revel.RouterFilter,            // Use the routing table to select the right Action
		revel.FilterConfiguringFilter, // A hook for adding or removing per-Action filters.
		revel.ParamsFilter,            // Parse parameters into Controller.Params.
		revel.SessionFilter,           // Restore and write the session cookie.
		revel.FlashFilter,             // Restore and write the flash cookie.
		revel.ValidationFilter,        // Restore kept validation errors and save new ones from cookie.
		revel.I18nFilter,              // Resolve the requested language
		revel.InterceptorFilter,       // Run interceptors around the action.
		revel.ActionInvoker,           // Invoke the action.
	}

	revel.OnAppStart(func() {
		initServices()
	})

	// Use as many threads as we have CPUs, helps for thumbnailing
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func initServices() {
	var found bool

	RootDir, found = revel.Config.String(GOLLERY_CONFIG_ROOT_DIR)

	if !found {
		panic("Missing configuration parameter: " + GOLLERY_CONFIG_ROOT_DIR)
	}

	cachedir, found := revel.Config.String(GOLLERY_CONFIG_CACHE_DIR)

	if !found {
		panic("Missing configuration parameter: " + GOLLERY_CONFIG_CACHE_DIR)
	}

	var err error

	Monitor, err = monitor.NewMonitor()

	if err != nil {
		panic("Cannot initalize file monitoring: " + err.Error())
	}

	Monitor.Watch(RootDir)

	Thumbnailer, err = thumbnailer.NewThumbnailer(RootDir, cachedir, Monitor)

	if err != nil {
		panic("Cannot initialize thumbnailer service: " + err.Error())
	}

	revel.INFO.Print("Started thumbnailing service")

	jobs.Now(jobs.Func(func() {
		err := Thumbnailer.CheckCache()

		if err != nil {
			revel.ERROR.Printf("Cannot check the thumbnail cache: %s", err)
		}
	}))
}
<|endoftext|>


<commit_before>package api_ws

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/deze333/vroom/auth"
	"github.com/deze333/vroom/reqres"
	"github.com/gorilla/websocket"
)

//------------------------------------------------------------
// WebSocket Implementation
//------------------------------------------------------------

// Handles WS Not Authd response.
func Handle_NotAuthd(w http.ResponseWriter, r *http.Request) {

	http.Error(w, "Not authorized", http.StatusUnauthorized)
}

// Creates WS connecttion and processes it in forever loop.
func Handle(w http.ResponseWriter, r *http.Request, router *reqres.WebSocket_Router, isAuthd bool) {

	var agentId string

	var upgrader = websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
		CheckOrigin: func(r *http.Request) bool {
			r.ParseForm()

			fmt.Printf("\n\n-------------------------------\nCheck WebSocket call: \nURL = %v \nOrigin = %v\nCaller ID = %v\n---------------------------------------\n\n", r.URL, r.Header["Origin"], r.Form["id"])

			// Check that ID is present
			if ids, ok := r.Form["id"]; ok {
				if len(ids) > 0 {
					agentId = ids[0]
				} else {
					return false
				}
			} else {
				return false
			}

			return true
		},
	}

	// Open websocket connection
	conn, err := upgrader.Upgrade(w, r, nil)

	// Process error
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", http.StatusBadRequest)
		return

	} else if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	// Zero time means unlimited connection
	err = conn.SetReadDeadline(time.Time{})
	if err != nil {
		_onPanic(
			fmt.Sprintf("Error setting WS deadline: %v", err),
			fmt.Sprintf("%v : %v", r.Host, r.URL),
			"Session", fmt.Sprint(auth.GetSessionValues(r)))
		return
	}

	// Create connection object
	ws := &Conn{
		w:                   w,
		r:                   r,
		conn:                conn,
		router:              router,
		agentId:             agentId,
		isAuthd:             isAuthd,
		chanIn:              make(chan []byte),
		chanOut:             make(chan []byte),
		chanProcClose:       make(chan int),
		chanProcWriterClose: make(chan int),
		isOpen:              true,
	}

	// Register this new connection
	RegisterConn(ws)

	// Retry failed messages, both in & out
	retryFailedMessages(agentId)

	// Close when done
	defer func() {
		DeregisterConn(ws)
		ws.conn.Close()
	}()

	fmt.Printf("++++ WebSocket %p opened\n", ws.conn)

	// Start async processors
	go proc(ws)
	go procWriter(ws)

	var msgType int
	var msg []byte

	// Forever loop listening for incoming messages
	for {

		// Read message
		msgType, msg, err = ws.conn.ReadMessage()
		if err != nil {
			break
		}

		switch msgType {

		// Text message
		case websocket.TextMessage:

			fmt.Println("O__________")
			// XXX TEMP TESTING
			// Choose processing logic depending on user type
			if isAuthd {

				vals, _ := auth.GetSessionValues(r)
				if q, ok := vals["qualities"]; ok {
					if strings.Contains(q, "tester") {
						_chanIn <- &Message{isAuthd: isAuthd, agentId: agentId, req: msg}
					} else {
						ws.chanIn <- msg
					}
				}
			}
			fmt.Println("__________X")

		// Binary not supported
		case websocket.BinaryMessage:
			// Do nothing

		case websocket.CloseMessage:
			break

		case websocket.PingMessage:
			// Do nothing

		case websocket.PongMessage:
			// Do nothing

		default:
		}
	}

	// Signal procs to finish
	ws.chanProcClose <- 1
	ws.chanProcWriterClose <- 1
	fmt.Printf("---X WebSocket %p closed\n", ws.conn)
}

// Processes incoming channel data.
func proc(ws *Conn) {
	for {
		select {

		case msg := <-ws.chanIn:
			processMessage(ws, msg)

		case <-ws.chanProcClose:
			// Exit
			fmt.Println("---X_R proc finished")
			return
		}
	}
}

// Processes outgoing channel responses.
func procWriter(ws *Conn) {
	for {
		select {

		case res := <-ws.chanOut:
			err := Respond(ws, res)

			// Exit on write error
			if err != nil {
				DeregisterConn(ws)
				ws.conn.Close()
				return
			}

		case <-ws.chanProcWriterClose:
			// Exit
			fmt.Println("---X_W procWriter finished")
			return
		}
	}
}
<commit_msg>testing ws<commit_after>package api_ws

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/deze333/vroom/auth"
	"github.com/deze333/vroom/reqres"
	"github.com/gorilla/websocket"
)

//------------------------------------------------------------
// WebSocket Implementation
//------------------------------------------------------------

// Handles WS Not Authd response.
func Handle_NotAuthd(w http.ResponseWriter, r *http.Request) {

	http.Error(w, "Not authorized", http.StatusUnauthorized)
}

// Creates WS connecttion and processes it in forever loop.
func Handle(w http.ResponseWriter, r *http.Request, router *reqres.WebSocket_Router, isAuthd bool) {

	var agentId string

	var upgrader = websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
		CheckOrigin: func(r *http.Request) bool {
			r.ParseForm()

			fmt.Printf("\n\n-------------------------------\nCheck WebSocket call: \nURL = %v \nOrigin = %v\nCaller ID = %v\n---------------------------------------\n\n", r.URL, r.Header["Origin"], r.Form["id"])

			// Check that ID is present
			if ids, ok := r.Form["id"]; ok {
				if len(ids) > 0 {
					agentId = ids[0]
				} else {
					return false
				}
			} else {
				return false
			}

			return true
		},
	}

	// Open websocket connection
	conn, err := upgrader.Upgrade(w, r, nil)

	// Process error
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", http.StatusBadRequest)
		return

	} else if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	// Zero time means unlimited connection
	err = conn.SetReadDeadline(time.Time{})
	if err != nil {
		_onPanic(
			fmt.Sprintf("Error setting WS deadline: %v", err),
			fmt.Sprintf("%v : %v", r.Host, r.URL),
			"Session", fmt.Sprint(auth.GetSessionValues(r)))
		return
	}

	// Create connection object
	ws := &Conn{
		w:                   w,
		r:                   r,
		conn:                conn,
		router:              router,
		agentId:             agentId,
		isAuthd:             isAuthd,
		chanIn:              make(chan []byte),
		chanOut:             make(chan []byte),
		chanProcClose:       make(chan int),
		chanProcWriterClose: make(chan int),
		isOpen:              true,
	}

	// Register this new connection
	RegisterConn(ws)

	// Retry failed messages, both in & out
	retryFailedMessages(agentId)

	// Close when done
	defer func() {
		DeregisterConn(ws)
		ws.conn.Close()
	}()

	fmt.Printf("++++ WebSocket %p opened\n", ws.conn)

	// Start async processors
	go proc(ws)
	go procWriter(ws)

	var msgType int
	var msg []byte

	// Forever loop listening for incoming messages
	for {

		// Read message
		msgType, msg, err = ws.conn.ReadMessage()
		if err != nil {
			break
		}

		switch msgType {

		// Text message
		case websocket.TextMessage:

			fmt.Println("O__________")
			// XXX TEMP TESTING
			// Choose processing logic depending on user type
			if isAuthd {

				vals, _ := auth.GetSessionValues(r)
				if q, ok := vals["qualities"]; ok {
					if strings.Contains(q, "tester") {
						fmt.Println("___TESTER____")
						_chanIn <- &Message{isAuthd: isAuthd, agentId: agentId, req: msg}
					} else {
						ws.chanIn <- msg
					}
				}
			}
			fmt.Println("__________X")

		// Binary not supported
		case websocket.BinaryMessage:
			// Do nothing

		case websocket.CloseMessage:
			break

		case websocket.PingMessage:
			// Do nothing

		case websocket.PongMessage:
			// Do nothing

		default:
		}
	}

	// Signal procs to finish
	ws.chanProcClose <- 1
	ws.chanProcWriterClose <- 1
	fmt.Printf("---X WebSocket %p closed\n", ws.conn)
}

// Processes incoming channel data.
func proc(ws *Conn) {
	for {
		select {

		case msg := <-ws.chanIn:
			processMessage(ws, msg)

		case <-ws.chanProcClose:
			// Exit
			fmt.Println("---X_R proc finished")
			return
		}
	}
}

// Processes outgoing channel responses.
func procWriter(ws *Conn) {
	for {
		select {

		case res := <-ws.chanOut:
			err := Respond(ws, res)

			// Exit on write error
			if err != nil {
				DeregisterConn(ws)
				ws.conn.Close()
				return
			}

		case <-ws.chanProcWriterClose:
			// Exit
			fmt.Println("---X_W procWriter finished")
			return
		}
	}
}
<|endoftext|>


<commit_before>package apigee

import (
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/zambien/go-apigee-edge"
	"reflect"
	"sort"
)

func flattenStringList(list []string) []interface{} {

	vs := make([]interface{}, 0, len(list))

	for _, v := range list {
		vs = append(vs, &v)
	}

	return vs
}

func getStringList(listName string, d *schema.ResourceData) []string {

	stringList := []string{}

	if attr, ok := d.GetOk(listName); ok {
		for _, s := range attr.([]interface{}) {
			if s != nil {
				stringList = append(stringList, s.(string))
			}
		}
	}

	return stringList
}

func apiProductsListFromCredentials(credentialApiProducts []apigee.CredentialApiProduct) []string {

	stringList := []string{}

	for _, apiProduct := range credentialApiProducts {
		stringList = append(stringList, apiProduct.ApiProduct)
	}

	return stringList
}

func attributesFromMap(attributes map[string]interface{}) []apigee.Attribute {

	result := make([]apigee.Attribute, 0, len(attributes))

	for k, v := range attributes {
		t := apigee.Attribute{
			Name:  k,
			Value: v.(string),
		}
		result = append(result, t)
	}

	return result
}

func arraySortedEqual(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	a_copy := make([]string, len(a))
	b_copy := make([]string, len(b))

	copy(a_copy, a)
	copy(b_copy, b)

	sort.Strings(a_copy)
	sort.Strings(b_copy)

	return reflect.DeepEqual(a_copy, b_copy)
}

func updateResourceOnSortedArrayChange(d *schema.ResourceData, key string, newValues []string) {
	currentValues := getStringList(key, d)
	if currentValues != nil && !arraySortedEqual(currentValues, newValues) {
		d.Set(key, newValues)
	}
}
<commit_msg>add back in creds helper code<commit_after>package apigee

import (
	"github.com/17media/structs"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/zambien/go-apigee-edge"
	"reflect"
	"sort"
)

func flattenStringList(list []string) []interface{} {

	vs := make([]interface{}, 0, len(list))

	for _, v := range list {
		vs = append(vs, &v)
	}

	return vs
}

func getStringList(listName string, d *schema.ResourceData) []string {

	stringList := []string{}

	if attr, ok := d.GetOk(listName); ok {
		for _, s := range attr.([]interface{}) {
			if s != nil {
				stringList = append(stringList, s.(string))
			}
		}
	}

	return stringList
}

func apiProductsListFromCredentials(credentialApiProducts []apigee.CredentialApiProduct) []string {

	stringList := []string{}

	for _, apiProduct := range credentialApiProducts {
		stringList = append(stringList, apiProduct.ApiProduct)
	}

	return stringList
}

func attributesFromMap(attributes map[string]interface{}) []apigee.Attribute {

	result := make([]apigee.Attribute, 0, len(attributes))

	for k, v := range attributes {
		t := apigee.Attribute{
			Name:  k,
			Value: v.(string),
		}
		result = append(result, t)
	}

	return result
}

func mapFromCredentials(credentials []apigee.Credential) []interface{} {

	result := make([]interface{}, 0, len(credentials))

	for _, elem := range credentials {
		credentialMap := structs.Map(elem)
		result = append(result, credentialMap)
	}

	return result
}

func arraySortedEqual(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	a_copy := make([]string, len(a))
	b_copy := make([]string, len(b))

	copy(a_copy, a)
	copy(b_copy, b)

	sort.Strings(a_copy)
	sort.Strings(b_copy)

	return reflect.DeepEqual(a_copy, b_copy)
}

func updateResourceOnSortedArrayChange(d *schema.ResourceData, key string, newValues []string) {
	currentValues := getStringList(key, d)
	if currentValues != nil && !arraySortedEqual(currentValues, newValues) {
		d.Set(key, newValues)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A janitor maintains feeds, creating and deleting as necessary.

func (mgr *Manager) JanitorKick(msg string) {
	resCh := make(chan error)
	mgr.janitorCh <- &WorkReq{msg: msg, resCh: resCh}
	<-resCh
}

func (mgr *Manager) JanitorLoop() {
	for m := range mgr.janitorCh {
		err := mgr.JanitorOnce(m.msg)
		if err != nil {
			log.Printf("error: JanitorOnce, err: %v", err)
			// Keep looping as perhaps it's a transient issue.
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}
}

func (mgr *Manager) JanitorOnce(reason string) error {
	log.Printf("janitor awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return fmt.Errorf("janitor skipped due to nil cfg")
	}

	planPIndexes, _, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return fmt.Errorf("janitor skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexes == nil {
		// Might happen if janitor wins a race.
		// TODO: Should we reschedule another future janitor kick?
		return fmt.Errorf("janitor skipped on nil planPIndexes")
	}

	currFeeds, currPIndexes := mgr.CurrentMaps()

	addPlanPIndexes, removePIndexes :=
		CalcPIndexesDelta(mgr.uuid, currPIndexes, planPIndexes)

	log.Printf("janitor pindexes to add:")
	for _, ppi := range addPlanPIndexes {
		log.Printf("  %+v", ppi)
	}
	log.Printf("janitor pindexes to remove:")
	for _, pi := range removePIndexes {
		log.Printf("  %+v", pi)
	}

	// First, teardown pindexes that need to be removed.
	for _, removePIndex := range removePIndexes {
		log.Printf("janitor removing pindex: %s", removePIndex.Name)
		err = mgr.StopPIndex(removePIndex)
		if err != nil {
			return fmt.Errorf("error: janitor removing pindex: %s, err: %v",
				removePIndex.Name, err)
		}
	}
	// Then, (re-)create pindexes that we're missing.
	for _, addPlanPIndex := range addPlanPIndexes {
		log.Printf("janitor adding pindex: %s", addPlanPIndex.Name)
		err = mgr.StartPIndex(addPlanPIndex)
		if err != nil {
			return fmt.Errorf("error: janitor adding pindex: %s, err: %v",
				addPlanPIndex.Name, err)
		}
	}

	currFeeds, currPIndexes = mgr.CurrentMaps()

	addFeeds, removeFeeds := CalcFeedsDelta(currFeeds, currPIndexes)
	log.Printf("janitor feeds add: %+v, remove: %+v",
		addFeeds, removeFeeds)

	// First, teardown feeds that need to be removed.
	for _, removeFeed := range removeFeeds {
		err = mgr.StopFeed(removeFeed)
		if err != nil {
			return fmt.Errorf("error: janitor removing feed: %s, err: %v",
				removeFeed.Name, err)
		}
	}
	// Then, (re-)create feeds that we're missing.
	for _, targePIndexes := range addFeeds {
		err = mgr.StartFeed(targePIndexes)
		if err != nil {
			return fmt.Errorf("error: janitor adding feed, err: %v", err)
		}
	}

	return nil
}

// Functionally determine the delta of which pindexes need creation
// and which should be shut down on our local node (mgrUUID).
func CalcPIndexesDelta(mgrUUID string,
	currPIndexes map[string]*PIndex,
	wantedPlanPIndexes *PlanPIndexes) (
	addPlanPIndexes []*PlanPIndex,
	removePIndexes []*PIndex) {
	// Allocate our return arrays.
	addPlanPIndexes = make([]*PlanPIndex, 0)
	removePIndexes = make([]*PIndex, 0)

	// Just for fast transient lookups.
	mapWantedPlanPIndex := make(map[string]*PlanPIndex)
	mapRemovePIndex := make(map[string]*PIndex)

	// For each wanted plan pindex, if a pindex does not exist or is
	// different, then schedule to add.
	for _, wantedPlanPIndex := range wantedPlanPIndexes.PlanPIndexes {
	nodeUUIDs:
		for nodeUUID, nodeState := range wantedPlanPIndex.NodeUUIDs {
			if nodeUUID == mgrUUID && nodeState == "active" {
				mapWantedPlanPIndex[wantedPlanPIndex.Name] = wantedPlanPIndex

				currPIndex, exists := currPIndexes[wantedPlanPIndex.Name]
				if !exists {
					addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
				} else if PIndexMatchesPlan(currPIndex, wantedPlanPIndex) == false {
					addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
					removePIndexes = append(removePIndexes, currPIndex)
					mapRemovePIndex[currPIndex.Name] = currPIndex
				}

				break nodeUUIDs
			}
		}
	}

	// For each existing pindex, if not part of wanted plan pindex,
	// then schedule for removal.
	for _, currPIndex := range currPIndexes {
		if _, exists := mapWantedPlanPIndex[currPIndex.Name]; !exists {
			if _, exists = mapRemovePIndex[currPIndex.Name]; !exists {
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}
		}
	}

	return addPlanPIndexes, removePIndexes
}

// Functionally determine the delta of which feeds need creation and
// which should be shut down.
func CalcFeedsDelta(currFeeds map[string]Feed, pindexes map[string]*PIndex) (
	addFeeds [][]*PIndex, removeFeeds []Feed) {
	addFeeds = make([][]*PIndex, 0)
	removeFeeds = make([]Feed, 0)

	for _, pindex := range pindexes {
		addFeedName := FeedName("default", pindex.SourceName, "")
		if _, ok := currFeeds[addFeedName]; !ok {
			addFeeds = append(addFeeds, []*PIndex{pindex})

			// TODO: this doesn't seem to do delta calc right.
		}
	}

	return addFeeds, removeFeeds
}

// --------------------------------------------------------

func (mgr *Manager) StartPIndex(planPIndex *PlanPIndex) error {
	pindex, err := NewPIndex(planPIndex.Name, NewUUID(),
		planPIndex.IndexName,
		planPIndex.IndexUUID,
		planPIndex.IndexMapping,
		planPIndex.SourceType,
		planPIndex.SourceName,
		planPIndex.SourceUUID,
		planPIndex.SourcePartitions,
		mgr.PIndexPath(planPIndex.Name))
	if err != nil {
		return fmt.Errorf("error: NewPIndex, name: %s, err: %v",
			planPIndex.Name, err)
	}

	if err = mgr.RegisterPIndex(pindex); err != nil {
		close(pindex.Stream)
		return err
	}

	return nil
}

func (mgr *Manager) StopPIndex(pindex *PIndex) error {
	feeds, _ := mgr.CurrentMaps()
	for _, feed := range feeds {
		for _, stream := range feed.Streams() {
			if stream == pindex.Stream {
				feedUnreg := mgr.UnregisterFeed(feed.Name())
				if feedUnreg != feed {
					panic("error: unregistered feed isn't the one we're closing")
				}

				// NOTE: We're depending on feed to synchronously
				// close, where we know it will no longer be writing
				// any pindex streams anymore.
				if err := feed.Close(); err != nil {
					panic(fmt.Sprintf("error: could not close feed, err: %v", err))
				}
			}
		}
	}

	pindexUnreg := mgr.UnregisterPIndex(pindex.Name)
	if pindexUnreg != nil && pindexUnreg != pindex {
		panic("unregistered pindex isn't the one we're stopping")
	}

	close(pindex.Stream)

	return nil
}

// --------------------------------------------------------

func (mgr *Manager) StartFeed(pindexes []*PIndex) error {
	// TODO: Need to create a fan-out feed.
	for _, pindex := range pindexes {
		// TODO: Need bucket UUID.
		err := mgr.StartTAPFeed(pindex) // TODO: err handling.
		if err != nil {
			return err
		}
	}
	return nil
}

func (mgr *Manager) StopFeed(feed Feed) error {
	// TODO.
	return nil
}

// --------------------------------------------------------

func (mgr *Manager) StartTAPFeed(pindex *PIndex) error {
	// TODO: bad assumption of 1-to-1 pindex.name to indexName
	indexName := pindex.IndexName

	// TODO: do more with SourceType, SourceName, SourceUUID.
	bucketName := pindex.SourceName
	bucketUUID := pindex.SourceUUID

	// TODO: utiilzed SourcePartitions
	streams := map[string]Stream{
		"": pindex.Stream,
	}
	feed, err := NewTAPFeed(mgr.server, "default", bucketName, bucketUUID, streams)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	if err = mgr.RegisterFeed(feed); err != nil {
		// TODO: cleanup?
		return err
	}

	return nil
}

// --------------------------------------------------------

func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexMapping == planPIndex.IndexMapping &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	if !same {
		log.Printf("PIndexMatchesPlan false, pindex: %#v, planPIndex: %#v",
			pindex, planPIndex)
	}
	return same
}
<commit_msg>moved PIndexMatchesPlan higher nearer to usage<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A janitor maintains feeds, creating and deleting as necessary.

func (mgr *Manager) JanitorKick(msg string) {
	resCh := make(chan error)
	mgr.janitorCh <- &WorkReq{msg: msg, resCh: resCh}
	<-resCh
}

func (mgr *Manager) JanitorLoop() {
	for m := range mgr.janitorCh {
		err := mgr.JanitorOnce(m.msg)
		if err != nil {
			log.Printf("error: JanitorOnce, err: %v", err)
			// Keep looping as perhaps it's a transient issue.
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}
}

func (mgr *Manager) JanitorOnce(reason string) error {
	log.Printf("janitor awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return fmt.Errorf("janitor skipped due to nil cfg")
	}

	planPIndexes, _, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return fmt.Errorf("janitor skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexes == nil {
		// Might happen if janitor wins a race.
		// TODO: Should we reschedule another future janitor kick?
		return fmt.Errorf("janitor skipped on nil planPIndexes")
	}

	currFeeds, currPIndexes := mgr.CurrentMaps()

	addPlanPIndexes, removePIndexes :=
		CalcPIndexesDelta(mgr.uuid, currPIndexes, planPIndexes)

	log.Printf("janitor pindexes to add:")
	for _, ppi := range addPlanPIndexes {
		log.Printf("  %+v", ppi)
	}
	log.Printf("janitor pindexes to remove:")
	for _, pi := range removePIndexes {
		log.Printf("  %+v", pi)
	}

	// First, teardown pindexes that need to be removed.
	for _, removePIndex := range removePIndexes {
		log.Printf("janitor removing pindex: %s", removePIndex.Name)
		err = mgr.StopPIndex(removePIndex)
		if err != nil {
			return fmt.Errorf("error: janitor removing pindex: %s, err: %v",
				removePIndex.Name, err)
		}
	}
	// Then, (re-)create pindexes that we're missing.
	for _, addPlanPIndex := range addPlanPIndexes {
		log.Printf("janitor adding pindex: %s", addPlanPIndex.Name)
		err = mgr.StartPIndex(addPlanPIndex)
		if err != nil {
			return fmt.Errorf("error: janitor adding pindex: %s, err: %v",
				addPlanPIndex.Name, err)
		}
	}

	currFeeds, currPIndexes = mgr.CurrentMaps()

	addFeeds, removeFeeds := CalcFeedsDelta(currFeeds, currPIndexes)
	log.Printf("janitor feeds add: %+v, remove: %+v",
		addFeeds, removeFeeds)

	// First, teardown feeds that need to be removed.
	for _, removeFeed := range removeFeeds {
		err = mgr.StopFeed(removeFeed)
		if err != nil {
			return fmt.Errorf("error: janitor removing feed: %s, err: %v",
				removeFeed.Name, err)
		}
	}
	// Then, (re-)create feeds that we're missing.
	for _, targePIndexes := range addFeeds {
		err = mgr.StartFeed(targePIndexes)
		if err != nil {
			return fmt.Errorf("error: janitor adding feed, err: %v", err)
		}
	}

	return nil
}

// --------------------------------------------------------

// Functionally determine the delta of which pindexes need creation
// and which should be shut down on our local node (mgrUUID).
func CalcPIndexesDelta(mgrUUID string,
	currPIndexes map[string]*PIndex,
	wantedPlanPIndexes *PlanPIndexes) (
	addPlanPIndexes []*PlanPIndex,
	removePIndexes []*PIndex) {
	// Allocate our return arrays.
	addPlanPIndexes = make([]*PlanPIndex, 0)
	removePIndexes = make([]*PIndex, 0)

	// Just for fast transient lookups.
	mapWantedPlanPIndex := make(map[string]*PlanPIndex)
	mapRemovePIndex := make(map[string]*PIndex)

	// For each wanted plan pindex, if a pindex does not exist or is
	// different, then schedule to add.
	for _, wantedPlanPIndex := range wantedPlanPIndexes.PlanPIndexes {
	nodeUUIDs:
		for nodeUUID, nodeState := range wantedPlanPIndex.NodeUUIDs {
			if nodeUUID != mgrUUID || nodeState != "active" {
				continue nodeUUIDs
			}

			mapWantedPlanPIndex[wantedPlanPIndex.Name] = wantedPlanPIndex

			currPIndex, exists := currPIndexes[wantedPlanPIndex.Name]
			if !exists {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
			} else if PIndexMatchesPlan(currPIndex, wantedPlanPIndex) == false {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}

			break nodeUUIDs
		}
	}

	// For each existing pindex, if not part of wanted plan pindex,
	// then schedule for removal.
	for _, currPIndex := range currPIndexes {
		if _, exists := mapWantedPlanPIndex[currPIndex.Name]; !exists {
			if _, exists = mapRemovePIndex[currPIndex.Name]; !exists {
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}
		}
	}

	return addPlanPIndexes, removePIndexes
}

func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexMapping == planPIndex.IndexMapping &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	if !same {
		log.Printf("PIndexMatchesPlan false, pindex: %#v, planPIndex: %#v",
			pindex, planPIndex)
	}
	return same
}

// --------------------------------------------------------

// Functionally determine the delta of which feeds need creation and
// which should be shut down.
func CalcFeedsDelta(currFeeds map[string]Feed, pindexes map[string]*PIndex) (
	addFeeds [][]*PIndex, removeFeeds []Feed) {
	addFeeds = make([][]*PIndex, 0)
	removeFeeds = make([]Feed, 0)

	// TODO: Group the pindexes by SourceType/Name/UUID/SourcePartition?
	for _, pindex := range pindexes {
		addFeedName := FeedName("default", pindex.SourceName, "")
		if _, ok := currFeeds[addFeedName]; !ok {
			addFeeds = append(addFeeds, []*PIndex{pindex})

			// TODO: this doesn't seem to do delta calc right.
		}
	}

	return addFeeds, removeFeeds
}

// --------------------------------------------------------

func (mgr *Manager) StartPIndex(planPIndex *PlanPIndex) error {
	pindex, err := NewPIndex(planPIndex.Name, NewUUID(),
		planPIndex.IndexName,
		planPIndex.IndexUUID,
		planPIndex.IndexMapping,
		planPIndex.SourceType,
		planPIndex.SourceName,
		planPIndex.SourceUUID,
		planPIndex.SourcePartitions,
		mgr.PIndexPath(planPIndex.Name))
	if err != nil {
		return fmt.Errorf("error: NewPIndex, name: %s, err: %v",
			planPIndex.Name, err)
	}

	if err = mgr.RegisterPIndex(pindex); err != nil {
		close(pindex.Stream)
		return err
	}

	return nil
}

func (mgr *Manager) StopPIndex(pindex *PIndex) error {
	feeds, _ := mgr.CurrentMaps()
	for _, feed := range feeds {
		for _, stream := range feed.Streams() {
			if stream == pindex.Stream {
				feedUnreg := mgr.UnregisterFeed(feed.Name())
				if feedUnreg != feed {
					panic("error: unregistered feed isn't the one we're closing")
				}

				// NOTE: We're depending on feed to synchronously
				// close, where we know it will no longer be writing
				// any pindex streams anymore.
				if err := feed.Close(); err != nil {
					panic(fmt.Sprintf("error: could not close feed, err: %v", err))
				}
			}
		}
	}

	pindexUnreg := mgr.UnregisterPIndex(pindex.Name)
	if pindexUnreg != nil && pindexUnreg != pindex {
		panic("unregistered pindex isn't the one we're stopping")
	}

	close(pindex.Stream)

	return nil
}

// --------------------------------------------------------

func (mgr *Manager) StartFeed(pindexes []*PIndex) error {
	// TODO: Need to create a fan-out feed.
	for _, pindex := range pindexes {
		// TODO: Need bucket UUID.
		err := mgr.StartTAPFeed(pindex) // TODO: err handling.
		if err != nil {
			return err
		}
	}
	return nil
}

func (mgr *Manager) StopFeed(feed Feed) error {
	// TODO.
	return nil
}

// --------------------------------------------------------

func (mgr *Manager) StartTAPFeed(pindex *PIndex) error {
	// TODO: bad assumption of 1-to-1 pindex.name to indexName
	indexName := pindex.IndexName

	// TODO: do more with SourceType, SourceName, SourceUUID.
	bucketName := pindex.SourceName
	bucketUUID := pindex.SourceUUID

	// TODO: utiilzed SourcePartitions
	streams := map[string]Stream{
		"": pindex.Stream,
	}
	feed, err := NewTAPFeed(mgr.server, "default", bucketName, bucketUUID, streams)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	if err = mgr.RegisterFeed(feed); err != nil {
		// TODO: cleanup?
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package textmate

import (
	"code.google.com/p/log4go"
	"encoding/json"
	"fmt"
	"github.com/moovweb/rubex"
	"github.com/quarnster/parser"
	"strconv"
	"strings"
)

const maxiter = 10000

type (
	Regex struct {
		re *rubex.Regexp
	}

	Language struct {
		UnpatchedLanguage
	}

	LanguageProvider interface {
		GetLanguage(id string) (*Language, error)
	}

	UnpatchedLanguage struct {
		FileTypes      []string
		FirstLineMatch string
		RootPattern    RootPattern `json:"patterns"`
		Repository     map[string]*Pattern
		ScopeName      string
	}

	Named struct {
		Name string
	}

	Captures map[string]Named

	MatchObject []int

	Pattern struct {
		Named
		Include        string
		Match          Regex
		Captures       Captures
		Begin          Regex
		BeginCaptures  Captures
		End            Regex
		EndCaptures    Captures
		Patterns       []*Pattern
		owner          *Language // needed for include directives
		cachedData     string
		cachedPat      *Pattern
		cachedPatterns []*Pattern
		cachedMatch    MatchObject
		hits           int
		misses         int
	}
	RootPattern struct {
		Pattern
	}

	LanguageParser struct {
		Language *Language
		root     parser.Node
	}
)

var (
	Provider LanguageProvider
	failed   = make(map[string]bool)
)

func (p Pattern) String() (ret string) {
	ret = fmt.Sprintf(`---------------------------------------
Name:    %s
Match:   %s
Begin:   %s
End:     %s
Include: %s
`, p.Name, p.Match, p.Begin, p.End, p.Include)
	ret += fmt.Sprintf("<Sub-Patterns>\n")
	for i := range p.Patterns {
		inner := fmt.Sprintf("%s", p.Patterns[i])
		ret += fmt.Sprintf("\t%s\n", strings.Replace(strings.Replace(inner, "\t", "\t\t", -1), "\n", "\n\t", -1))
	}
	ret += fmt.Sprintf("</Sub-Patterns>\n---------------------------------------")
	return
}

func (r *Regex) String() string {
	return r.re.String()
}

func (r *RootPattern) String() (ret string) {
	for i := range r.Patterns {
		ret += fmt.Sprintf("\t%s\n", r.Patterns[i])
	}
	return
}

func (s *Language) String() string {
	return fmt.Sprintf("%s\n%s", s.ScopeName, s.RootPattern)
}

func (p *Pattern) setOwner(l *Language) {
	p.owner = l
	for i := range p.Patterns {
		p.Patterns[i].setOwner(l)
	}
}

func (l *Language) UnmarshalJSON(data []byte) error {
	if err := json.Unmarshal(data, &l.UnpatchedLanguage); err != nil {
		return err
	}
	l.RootPattern.setOwner(l)
	for k := range l.Repository {
		l.Repository[k].setOwner(l)
	}
	return nil
}

func (r *RootPattern) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &r.Patterns)
}

func (r *Regex) UnmarshalJSON(data []byte) error {
	str := string(data[1 : len(data)-1])
	str = strings.Replace(str, "\\\\", "\\", -1)
	if re, err := rubex.Compile(str); err != nil {
		log4go.Warn("Couldn't compile language pattern %s: %s", str, err)
	} else {
		r.re = re
	}
	return nil
}

func (m MatchObject) fix(add int) {
	for i := range m {
		m[i] += add
	}
}

func (p *Pattern) FirstMatch(data string, pos int) (pat *Pattern, ret MatchObject) {
	startIdx := -1
	for i := 0; i < len(p.cachedPatterns); {
		ip, im := p.cachedPatterns[i].Cache(data, pos)
		if im != nil {
			if startIdx < 0 || startIdx > im[0] {
				startIdx, pat, ret = im[0], ip, im
				// This match is right at the start, we're not going to find a better pattern than this,
				// so stop the search
				if im[0] == pos {
					break
				}
			}
			i++
		} else {
			// If it wasn't found now, it'll never be found, so the pattern can be popped from the cache
			copy(p.cachedPatterns[i:], p.cachedPatterns[i+1:])
			p.cachedPatterns = p.cachedPatterns[:len(p.cachedPatterns)-1]
		}
	}
	return
}

func (p *Pattern) Cache(data string, pos int) (pat *Pattern, ret MatchObject) {
	if p.cachedData == data {
		if p.cachedMatch == nil {
			return nil, nil
		}
		if p.cachedMatch[0] >= pos {
			p.hits++
			return p.cachedPat, p.cachedMatch
		}
	} else {
		p.cachedPatterns = nil
	}
	if p.cachedPatterns == nil {
		p.cachedPatterns = make([]*Pattern, len(p.Patterns))
		copy(p.cachedPatterns, p.Patterns)
	}
	p.misses++

	if p.Match.re != nil {
		pat, ret = p, p.Match.re.FindStringSubmatchIndex(data[pos:])
		ret.fix(pos)
	} else if p.Begin.re != nil {
		pat, ret = p, p.Begin.re.FindStringSubmatchIndex(data[pos:])
		ret.fix(pos)
	} else if p.Include != "" {
		if z := p.Include[0]; z == '#' {
			key := p.Include[1:]
			if p2, ok := p.owner.Repository[key]; ok {
				pat, ret = p2.Cache(data, pos)
			} else {
				log4go.Error("Not found in repository: %s", p.Include)
			}
		} else if z == '$' {
			// TODO(q): Implement tmLanguage $ include directives
			log4go.Warn("Unhandled include directive: %s", p.Include)
		} else if Provider == nil {
			log4go.Warn("Include directive %s couldn't be handled as there's no Provider set", p.Include)
		} else if l, err := Provider.GetLanguage(p.Include); err != nil {
			if !failed[p.Include] {
				log4go.Error("Include directive %s failed: %s", p.Include, err)
			}
			failed[p.Include] = true
		} else {
			return l.RootPattern.Cache(data, pos)
		}
	} else {
		pat, ret = p.FirstMatch(data, pos)
	}
	p.cachedData = data
	p.cachedMatch = ret
	p.cachedPat = pat

	return
}

func (p *Pattern) CreateCaptureNodes(data string, pos int, d parser.DataSource, mo MatchObject, parent *parser.Node, cap Captures) {
	ranges := make([]parser.Range, len(mo)/2)
	parentIndex := make([]int, len(ranges))
	parents := make([]*parser.Node, len(parentIndex))
	for i := range ranges {
		ranges[i] = parser.Range{mo[i*2+0], mo[i*2+1]}
		if i < 2 {
			parents[i] = parent
			continue
		}
		r := ranges[i]
		for j := i - 1; j >= 0; j-- {
			if ranges[j].Contains(r) {
				parentIndex[i] = j
				break
			}
		}
	}

	for k, v := range cap {
		i64, err := strconv.ParseInt(k, 10, 32)
		if i := int(i64); err == nil && i < len(parents) {
			child := &parser.Node{Name: v.Name, Range: ranges[i], P: d}
			parents[i] = child
			var p *parser.Node
			for p == nil {
				i = parentIndex[i]
				p = parents[i]
			}
			p.Append(child)
		}
	}
}

func (p *Pattern) CreateNode(data string, pos int, d parser.DataSource, mo MatchObject) *parser.Node {
	fmt.Println("skipping: ", data[pos:mo[0]], "consuming:", data[mo[0]:mo[1]])
	ret := parser.Node{Name: p.Name, Range: parser.Range{mo[0], mo[1]}, P: d}
	if p.Match.re != nil {
		p.CreateCaptureNodes(data, pos, d, mo, &ret, p.Captures)
	} else if p.Begin.re != nil {
		if len(p.BeginCaptures) > 0 {
			p.CreateCaptureNodes(data, pos, d, mo, &ret, p.BeginCaptures)
		} else {
			p.CreateCaptureNodes(data, pos, d, mo, &ret, p.Captures)
		}

		if p.End.re != nil {
			var (
				found  = false
				i, end int
			)
			for i, end = ret.Range.End, len(data); i < len(data); {
				endmatch := MatchObject(p.End.re.FindStringSubmatchIndex(data[i:]))
				endmatch.fix(i)
				//fmt.Println(i, end, endmatch, p.End.re)
				if endmatch != nil {
					end = endmatch[1]
				} else {
					if !found {
						// oops.. no end found at all, set it to the next line
						if e2 := strings.IndexRune(data[i:], '\n'); e2 != -1 {
							end = i + e2
						}
						//	fmt.Println("break 1")
						break
					} else {
						end = i
						//fmt.Println("break 2")
						break
					}
				}
				if (endmatch == nil || (endmatch != nil && endmatch[0] != i+1)) && len(p.cachedPatterns) > 0 {
					// Might be more recursive patterns to apply BEFORE the end is reached
					pattern2, match2 := p.FirstMatch(data, i)
					if match2 != nil &&
						((endmatch == nil && match2[0] < end) ||
							(endmatch != nil && match2[0] < endmatch[0])) {

						found = true
						r := pattern2.CreateNode(data, i, d, match2)
						ret.Append(r)
						i = r.Range.End
						continue
					}
				}
				if endmatch != nil {
					p.CreateCaptureNodes(data, i, d, endmatch, &ret, p.EndCaptures)
				}
				//fmt.Println("break 3")
				break
			}
			ret.Range.End = end
		}
	}
	ret.UpdateRange()
	return &ret
}

type dp struct {
	data string
}

func (d *dp) Data(a, b int) string {
	return d.data[a:b]
}

func (lp *LanguageParser) Parse(data string) bool {
	d := &dp{data}
	lp.root = parser.Node{P: d}
	iter := maxiter
	for i := 0; i < len(data) && iter > 0; iter-- {
		fmt.Println(i, len(data))
		pat, ret := lp.Language.RootPattern.Cache(data, i)
		if ret == nil {
			break
		} else {
			n := pat.CreateNode(data, i, d, ret)
			lp.root.Append(n)

			i = n.Range.End
		}
	}
	lp.root.UpdateRange()
	fmt.Println(lp.root)
	return true
}
<commit_msg>The simple plist TmLanguage test appears to work. More testing to come<commit_after>package textmate

import (
	"code.google.com/p/log4go"
	"encoding/json"
	"fmt"
	"github.com/moovweb/rubex"
	"github.com/quarnster/parser"
	"strconv"
	"strings"
)

const maxiter = 10000

type (
	Regex struct {
		re        *rubex.Regexp
		lastIndex int
		lastFound int
		lastData  string
	}

	Language struct {
		UnpatchedLanguage
	}

	LanguageProvider interface {
		GetLanguage(id string) (*Language, error)
	}

	UnpatchedLanguage struct {
		FileTypes      []string
		FirstLineMatch string
		RootPattern    RootPattern `json:"patterns"`
		Repository     map[string]*Pattern
		ScopeName      string
	}

	Named struct {
		Name string
	}

	Captures map[string]Named

	MatchObject []int

	Pattern struct {
		Named
		Include        string
		Match          Regex
		Captures       Captures
		Begin          Regex
		BeginCaptures  Captures
		End            Regex
		EndCaptures    Captures
		Patterns       []*Pattern
		owner          *Language // needed for include directives
		cachedData     string
		cachedPat      *Pattern
		cachedPatterns []*Pattern
		cachedMatch    MatchObject
		hits           int
		misses         int
	}
	RootPattern struct {
		Pattern
	}

	LanguageParser struct {
		Language *Language
		root     parser.Node
	}
)

var (
	Provider LanguageProvider
	failed   = make(map[string]bool)
)

func (p Pattern) String() (ret string) {
	ret = fmt.Sprintf(`---------------------------------------
Name:    %s
Match:   %s
Begin:   %s
End:     %s
Include: %s
`, p.Name, p.Match, p.Begin, p.End, p.Include)
	ret += fmt.Sprintf("<Sub-Patterns>\n")
	for i := range p.Patterns {
		inner := fmt.Sprintf("%s", p.Patterns[i])
		ret += fmt.Sprintf("\t%s\n", strings.Replace(strings.Replace(inner, "\t", "\t\t", -1), "\n", "\n\t", -1))
	}
	ret += fmt.Sprintf("</Sub-Patterns>\n---------------------------------------")
	return
}

func (r *Regex) String() string {
	return r.re.String()
}

func (r *RootPattern) String() (ret string) {
	for i := range r.Patterns {
		ret += fmt.Sprintf("\t%s\n", r.Patterns[i])
	}
	return
}

func (s *Language) String() string {
	return fmt.Sprintf("%s\n%s", s.ScopeName, s.RootPattern)
}

func (p *Pattern) setOwner(l *Language) {
	p.owner = l
	for i := range p.Patterns {
		p.Patterns[i].setOwner(l)
	}
}

func (l *Language) UnmarshalJSON(data []byte) error {
	if err := json.Unmarshal(data, &l.UnpatchedLanguage); err != nil {
		return err
	}
	l.RootPattern.setOwner(l)
	for k := range l.Repository {
		l.Repository[k].setOwner(l)
	}
	return nil
}

func (r *RootPattern) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &r.Patterns)
}

func (r *Regex) UnmarshalJSON(data []byte) error {
	str := string(data[1 : len(data)-1])
	str = strings.Replace(str, "\\\\", "\\", -1)
	if re, err := rubex.Compile(str); err != nil {
		log4go.Warn("Couldn't compile language pattern %s: %s", str, err)
	} else {
		r.re = re
	}
	return nil
}

func (m MatchObject) fix(add int) {
	for i := range m {
		m[i] += add
	}
}

func (r *Regex) Find(data string, pos int) MatchObject {
	if r.lastData != data || r.lastIndex > pos {
		r.lastData = data
		r.lastFound = 0
	}
	r.lastIndex = pos
	for r.lastFound < len(data) {
		ret := r.re.FindStringSubmatchIndex(data[r.lastFound:])
		if ret == nil {
			break
		} else if (ret[0] + r.lastFound) < pos {
			if ret[0] == 0 {
				r.lastFound++
			} else {
				r.lastFound += ret[0]
			}
			continue
		}
		mo := MatchObject(ret)
		mo.fix(r.lastFound)
		return mo
	}
	return nil
}

func (p *Pattern) FirstMatch(data string, pos int) (pat *Pattern, ret MatchObject) {
	startIdx := -1
	for i := 0; i < len(p.cachedPatterns); {
		ip, im := p.cachedPatterns[i].Cache(data, pos)
		if im != nil {
			if startIdx < 0 || startIdx > im[0] {
				startIdx, pat, ret = im[0], ip, im
				// This match is right at the start, we're not going to find a better pattern than this,
				// so stop the search
				if im[0] == pos {
					break
				}
			}
			i++
		} else {
			// If it wasn't found now, it'll never be found, so the pattern can be popped from the cache
			copy(p.cachedPatterns[i:], p.cachedPatterns[i+1:])
			p.cachedPatterns = p.cachedPatterns[:len(p.cachedPatterns)-1]
		}
	}
	return
}

func (p *Pattern) Cache(data string, pos int) (pat *Pattern, ret MatchObject) {
	if p.cachedData == data {
		if p.cachedMatch == nil {
			return nil, nil
		}
		if p.cachedMatch[0] >= pos {
			p.hits++
			return p.cachedPat, p.cachedMatch
		}
	} else {
		p.cachedPatterns = nil
	}
	if p.cachedPatterns == nil {
		p.cachedPatterns = make([]*Pattern, len(p.Patterns))
		copy(p.cachedPatterns, p.Patterns)
	}
	p.misses++

	if p.Match.re != nil {
		pat, ret = p, p.Match.Find(data, pos)
	} else if p.Begin.re != nil {
		pat, ret = p, p.Begin.Find(data, pos)
	} else if p.Include != "" {
		if z := p.Include[0]; z == '#' {
			key := p.Include[1:]
			if p2, ok := p.owner.Repository[key]; ok {
				pat, ret = p2.Cache(data, pos)
			} else {
				log4go.Error("Not found in repository: %s", p.Include)
			}
		} else if z == '$' {
			// TODO(q): Implement tmLanguage $ include directives
			log4go.Warn("Unhandled include directive: %s", p.Include)
		} else if Provider == nil {
			log4go.Warn("Include directive %s couldn't be handled as there's no Provider set", p.Include)
		} else if l, err := Provider.GetLanguage(p.Include); err != nil {
			if !failed[p.Include] {
				log4go.Error("Include directive %s failed: %s", p.Include, err)
			}
			failed[p.Include] = true
		} else {
			return l.RootPattern.Cache(data, pos)
		}
	} else {
		pat, ret = p.FirstMatch(data, pos)
	}
	p.cachedData = data
	p.cachedMatch = ret
	p.cachedPat = pat

	return
}

func (p *Pattern) CreateCaptureNodes(data string, pos int, d parser.DataSource, mo MatchObject, parent *parser.Node, cap Captures) {
	ranges := make([]parser.Range, len(mo)/2)
	parentIndex := make([]int, len(ranges))
	parents := make([]*parser.Node, len(parentIndex))
	for i := range ranges {
		ranges[i] = parser.Range{mo[i*2+0], mo[i*2+1]}
		if i < 2 {
			parents[i] = parent
			continue
		}
		r := ranges[i]
		for j := i - 1; j >= 0; j-- {
			if ranges[j].Contains(r) {
				parentIndex[i] = j
				break
			}
		}
	}

	for k, v := range cap {
		i64, err := strconv.ParseInt(k, 10, 32)
		if i := int(i64); err == nil && i < len(parents) {
			if ranges[i].Start == -1 {
				continue
			}
			child := &parser.Node{Name: v.Name, Range: ranges[i], P: d}
			parents[i] = child
			var p *parser.Node
			for p == nil {
				i = parentIndex[i]
				p = parents[i]
			}
			p.Append(child)
		}
	}
}

func (p *Pattern) CreateNode(data string, pos int, d parser.DataSource, mo MatchObject) *parser.Node {
	ret := parser.Node{Name: p.Name, Range: parser.Range{mo[0], mo[1]}, P: d}
	if p.Match.re != nil {
		p.CreateCaptureNodes(data, pos, d, mo, &ret, p.Captures)
	} else if p.Begin.re != nil {
		if len(p.BeginCaptures) > 0 {
			p.CreateCaptureNodes(data, pos, d, mo, &ret, p.BeginCaptures)
		} else {
			p.CreateCaptureNodes(data, pos, d, mo, &ret, p.Captures)
		}

		if p.End.re != nil {
			var (
				found  = false
				i, end int
			)
			for i, end = ret.Range.End, len(data); i < len(data); {
				endmatch := p.End.Find(data, i)
				if endmatch != nil {
					end = endmatch[1]
				} else {
					if !found {
						// oops.. no end found at all, set it to the next line
						if e2 := strings.IndexRune(data[i:], '\n'); e2 != -1 {
							end = i + e2
						}
						break
					} else {
						end = i
						break
					}
				}
				if (endmatch == nil || (endmatch != nil && endmatch[0] != i)) && len(p.cachedPatterns) > 0 {
					// Might be more recursive patterns to apply BEFORE the end is reached
					pattern2, match2 := p.FirstMatch(data, i)
					if match2 != nil &&
						((endmatch == nil && match2[0] < end) ||
							(endmatch != nil && match2[0] < endmatch[0])) {

						found = true
						r := pattern2.CreateNode(data, i, d, match2)
						ret.Append(r)
						i = r.Range.End
						continue
					}
				}
				if endmatch != nil {
					if len(p.EndCaptures) > 0 {
						p.CreateCaptureNodes(data, i, d, endmatch, &ret, p.EndCaptures)
					} else {
						p.CreateCaptureNodes(data, i, d, endmatch, &ret, p.Captures)
					}
				}
				break
			}
			ret.Range.End = end
		}
	}
	ret.UpdateRange()
	return &ret
}

type dp struct {
	data string
}

func (d *dp) Data(a, b int) string {
	return d.data[a:b]
}

func (lp *LanguageParser) Parse(data string) bool {
	d := &dp{data}
	lp.root = parser.Node{P: d}
	iter := maxiter
	for i := 0; i < len(data) && iter > 0; iter-- {
		pat, ret := lp.Language.RootPattern.Cache(data, i)
		if ret == nil {
			break
		} else {
			n := pat.CreateNode(data, i, d, ret)
			lp.root.Append(n)

			i = n.Range.End
		}
	}
	lp.root.UpdateRange()
	fmt.Println(lp.root)
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	storage "github.com/MSOpenTech/azure-sdk-for-go/storage"
	fb "github.com/huandu/facebook"
	"code.google.com/p/gorest"
)

func getBlobClient() (*storage.BlobStorageClient, error) {
	name := os.Getenv("STORAGE_ACCOUNT_NAME")
	key := os.Getenv("STORAGE_KEY")
	cli, err := storage.NewBasicClient(name, key)

	if err != nil {
		return nil, err
	}

	return cli.GetBlobService(), nil
}

func main() {

    gorest.RegisterService(new(DropletServer))
    http.Handle("/",gorest.Handle())
    http.ListenAndServe(":8080",nil)

    res, _ := fb.Get("/4", fb.Params{
        "fields": "username",
    })
    fmt.Println("here is my facebook username:", res["username"])

    cli, err := getBlobClient()

   	if err != nil {
		fmt.Printf("Error trying to check if container exists!\n")
		return
	}

	cnt := "testcontainer"
	ok, err := cli.ContainerExists(cnt)
	if err != nil {
		fmt.Printf("Error trying to check if container exists!\n")
	}

	if ok {
		fmt.Printf("Found container!\n")
	} else {
		fmt.Printf("Containr not found");
	}
}

type Droplet struct{
    IntId             int
    StringName        string
    BoolValue         bool
}

type DropletServer struct {
    gorest.RestService `root:"/" consumes:"application/json" produces:"application/json"`

	item    gorest.EndPoint `method:"GET" path:"/item/{Id:int}" output:"Droplet"`
    items   gorest.EndPoint `method:"GET" path:"/items/" output:"[]Droplet"`
    insert  gorest.EndPoint `method:"POST" path:"/insert/" postdata:"[]Droplet"`
}

func(serv DropletServer) Item(Id int) Droplet {
    serv.ResponseBuilder().SetResponseCode(200)
    item := Droplet {IntId:Id, StringName:"Name with id returned", BoolValue:true}
    return item
}

func(serv DropletServer) Items() []Droplet{
    serv.ResponseBuilder().SetResponseCode(200)
    slice := []Droplet{
      Droplet {IntId:0, StringName:"Name 0", BoolValue:true},
      Droplet {IntId:1, StringName:"Name 1", BoolValue:true},
    }

    item := Droplet {IntId:200, StringName:"Name 4", BoolValue:true}
    slice = append(slice, item)

    return slice
}

func(serv DropletServer) Insert(items []Droplet) {
    fmt.Println("Got a request to insert items")
    fmt.Println("Item Count", len(items))
    serv.ResponseBuilder().SetResponseCode(200)
}<commit_msg>droplet server that retrieves data from storage<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"io/ioutil"
	storage "github.com/MSOpenTech/azure-sdk-for-go/storage"
	fb "github.com/huandu/facebook"
	"code.google.com/p/gorest"
)

func main() {

    gorest.RegisterService(new(DropletServer))
    http.Handle("/",gorest.Handle())
    http.ListenAndServe(":8080",nil)

    res, _ := fb.Get("/4", fb.Params{
        "fields": "username",
    })
    fmt.Println("here is my facebook username:", res["username"])

}

type Droplet struct{
    IntId             int
    StringName        string
    BoolValue         bool
}

type DropletServer struct {
    gorest.RestService `root:"/" consumes:"application/json" produces:"application/json"`

	item    gorest.EndPoint `method:"GET" path:"/item/{Id:int}" output:"Droplet"`
    items   gorest.EndPoint `method:"GET" path:"/items/" output:"[]Droplet"`
    insert  gorest.EndPoint `method:"POST" path:"/insert/" postdata:"[]Droplet"`
    getDroplet    gorest.EndPoint `method:"GET" path:"/d/{userName:string}/{dropletName:string}" output:"Droplet"`
}

func(serv DropletServer) Item(Id int) Droplet {
    serv.ResponseBuilder().SetResponseCode(200)
    item := Droplet {IntId:Id, StringName:"Name with id returned", BoolValue:true}
    return item
}

func(serv DropletServer) Items() []Droplet{
    serv.ResponseBuilder().SetResponseCode(200)
    slice := []Droplet{
      Droplet {IntId:0, StringName:"Name 0", BoolValue:true},
      Droplet {IntId:1, StringName:"Name 1", BoolValue:true},
    }

    item := Droplet {IntId:200, StringName:"Name 4", BoolValue:true}
    slice = append(slice, item)

    return slice
}

func(serv DropletServer) Insert(items []Droplet) {
    fmt.Println("Got a request to insert items")
    fmt.Println("Item Count", len(items))
    serv.ResponseBuilder().SetResponseCode(200)
}

var blobClient *storage.BlobStorageClient

func(serv DropletServer) GetDroplet(userName string, dropletName string) Droplet {
    serv.ResponseBuilder().SetResponseCode(200)

    if blobClient == nil {
    	err := initializeBlobClient()
    	 if err != nil {
    		return Droplet {0,"Blob Client was null",false}
    	}
    }

    resp, err := blobClient.GetBlob(userName, dropletName)
	if err != nil {
		fmt.Printf("Error trying to check blob contents!\n")
	}

	// Verify contents
	respBody, err := ioutil.ReadAll(resp)
	defer resp.Close()
	if err != nil {
		fmt.Printf("Error trying to get blob contents!\n")
	}

	contents := string(respBody[:len(respBody)])

    item := Droplet {IntId:100, StringName:contents, BoolValue:true}
    return item
}

func initializeBlobClient() (error) {
	name := os.Getenv("STORAGE_ACCOUNT_NAME")
	key := os.Getenv("STORAGE_KEY")
	cli, err := storage.NewBasicClient(name, key)

	if err != nil {
		return err
	}

	blobClient = cli.GetBlobService()
	return nil
}<|endoftext|>


<commit_before>package getput

import (
	"context"
	"crypto/sha1"
	"errors"
	"log"
	"math"
	"sync"

	"github.com/anacrolix/dht/v2"
	"github.com/anacrolix/dht/v2/bep44"
	k_nearest_nodes "github.com/anacrolix/dht/v2/k-nearest-nodes"
	"github.com/anacrolix/dht/v2/krpc"
	"github.com/anacrolix/dht/v2/traversal"
	"github.com/anacrolix/torrent/bencode"
)

type PutGetResult struct {
	GetV           interface{}
	GetBytes       []byte
	TraversalStats *traversal.Stats
	SuccessfulPuts []krpc.NodeAddr
}

type GetResult struct {
	Seq     int64
	V       bencode.Bytes
	Mutable bool
}

func Get(
	ctx context.Context, target bep44.Target, s *dht.Server, seq *int64, salt []byte,
) (
	ret GetResult, stats *traversal.Stats, err error,
) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	vChan := make(chan GetResult)
	op := traversal.Start(traversal.OperationInput{
		Alpha:  15,
		Target: target,
		DoQuery: func(ctx context.Context, addr krpc.NodeAddr) traversal.QueryResult {
			res := s.Get(ctx, dht.NewAddr(addr.UDP()), target, seq, dht.QueryRateLimiting{})
			err := res.ToError()
			if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, dht.TransactionTimeout) {
				// log.Printf("error querying %v: %v", addr, err)
			}
			if r := res.Reply.R; r != nil {
				rv := r.V
				bv := rv
				if sha1.Sum(bv) == target {
					select {
					case vChan <- GetResult{
						V:       rv,
						Mutable: false,
					}:
					case <-ctx.Done():
					}
				} else if sha1.Sum(append(r.K[:], salt...)) == target && bep44.Verify(r.K[:], salt, *r.Seq, bv, r.Sig[:]) {
					select {
					case vChan <- GetResult{
						Seq:     *r.Seq,
						V:       rv,
						Mutable: true,
					}:
					case <-ctx.Done():
					}
				} else if rv != nil {
					log.Printf("get response item hash didn't match target: %q", rv)
				}
			}
			return res.TraversalQueryResult(addr)
		},
		NodeFilter: s.TraversalNodeFilter,
	})
	nodes, err := s.TraversalStartingNodes()
	if err != nil {
		return
	}
	op.AddNodes(nodes)
	ret.Seq = math.MinInt64
	gotValue := false
receiveResults:
	select {
	case <-op.Stalled():
		if !gotValue {
			err = errors.New("value not found")
		}
	case v := <-vChan:
		log.Printf("received %#v", v)
		gotValue = true
		if !v.Mutable {
			ret = v
			break
		}
		if v.Seq >= ret.Seq {
			ret = v
		}
		goto receiveResults
	case <-ctx.Done():
		err = ctx.Err()
	}
	op.Stop()
	stats = op.Stats()
	return
}

func Put(
	ctx context.Context, target krpc.ID, s *dht.Server, put bep44.Put,
) (
	stats *traversal.Stats, err error,
) {
	op := traversal.Start(traversal.OperationInput{
		Alpha:  15,
		Target: target,
		DoQuery: func(ctx context.Context, addr krpc.NodeAddr) traversal.QueryResult {
			res := s.Get(ctx, dht.NewAddr(addr.UDP()), target, nil, dht.QueryRateLimiting{})
			err := res.ToError()
			if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, dht.TransactionTimeout) {
				// log.Printf("error querying %v: %v", addr, err)
			}
			tqr := res.TraversalQueryResult(addr)
			if tqr.ClosestData == nil {
				tqr.ResponseFrom = nil
			}
			return tqr
		},
		NodeFilter: s.TraversalNodeFilter,
	})
	nodes, err := s.TraversalStartingNodes()
	if err != nil {
		return
	}
	op.AddNodes(nodes)
	select {
	case <-op.Stalled():
	case <-ctx.Done():
		err = ctx.Err()
	}
	op.Stop()
	var wg sync.WaitGroup
	op.Closest().Range(func(elem k_nearest_nodes.Elem) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			token := elem.Data.(string)
			res := s.Put(ctx, dht.NewAddr(elem.Addr.UDP()), put, token, dht.QueryRateLimiting{})
			err := res.ToError()
			if err != nil {
				log.Printf("error putting to %v [token=%q]: %v", elem.Addr, token, err)
			} else {
				log.Printf("put to %v [token=%q]", elem.Addr, token)
			}
		}()
	})
	wg.Wait()
	stats = op.Stats()
	return
}
<commit_msg>Remove unused struct<commit_after>package getput

import (
	"context"
	"crypto/sha1"
	"errors"
	"log"
	"math"
	"sync"

	"github.com/anacrolix/dht/v2"
	"github.com/anacrolix/dht/v2/bep44"
	k_nearest_nodes "github.com/anacrolix/dht/v2/k-nearest-nodes"
	"github.com/anacrolix/dht/v2/krpc"
	"github.com/anacrolix/dht/v2/traversal"
	"github.com/anacrolix/torrent/bencode"
)

type GetResult struct {
	Seq     int64
	V       bencode.Bytes
	Mutable bool
}

func Get(
	ctx context.Context, target bep44.Target, s *dht.Server, seq *int64, salt []byte,
) (
	ret GetResult, stats *traversal.Stats, err error,
) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	vChan := make(chan GetResult)
	op := traversal.Start(traversal.OperationInput{
		Alpha:  15,
		Target: target,
		DoQuery: func(ctx context.Context, addr krpc.NodeAddr) traversal.QueryResult {
			res := s.Get(ctx, dht.NewAddr(addr.UDP()), target, seq, dht.QueryRateLimiting{})
			err := res.ToError()
			if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, dht.TransactionTimeout) {
				// log.Printf("error querying %v: %v", addr, err)
			}
			if r := res.Reply.R; r != nil {
				rv := r.V
				bv := rv
				if sha1.Sum(bv) == target {
					select {
					case vChan <- GetResult{
						V:       rv,
						Mutable: false,
					}:
					case <-ctx.Done():
					}
				} else if sha1.Sum(append(r.K[:], salt...)) == target && bep44.Verify(r.K[:], salt, *r.Seq, bv, r.Sig[:]) {
					select {
					case vChan <- GetResult{
						Seq:     *r.Seq,
						V:       rv,
						Mutable: true,
					}:
					case <-ctx.Done():
					}
				} else if rv != nil {
					log.Printf("get response item hash didn't match target: %q", rv)
				}
			}
			return res.TraversalQueryResult(addr)
		},
		NodeFilter: s.TraversalNodeFilter,
	})
	nodes, err := s.TraversalStartingNodes()
	if err != nil {
		return
	}
	op.AddNodes(nodes)
	ret.Seq = math.MinInt64
	gotValue := false
receiveResults:
	select {
	case <-op.Stalled():
		if !gotValue {
			err = errors.New("value not found")
		}
	case v := <-vChan:
		log.Printf("received %#v", v)
		gotValue = true
		if !v.Mutable {
			ret = v
			break
		}
		if v.Seq >= ret.Seq {
			ret = v
		}
		goto receiveResults
	case <-ctx.Done():
		err = ctx.Err()
	}
	op.Stop()
	stats = op.Stats()
	return
}

func Put(
	ctx context.Context, target krpc.ID, s *dht.Server, put bep44.Put,
) (
	stats *traversal.Stats, err error,
) {
	op := traversal.Start(traversal.OperationInput{
		Alpha:  15,
		Target: target,
		DoQuery: func(ctx context.Context, addr krpc.NodeAddr) traversal.QueryResult {
			res := s.Get(ctx, dht.NewAddr(addr.UDP()), target, nil, dht.QueryRateLimiting{})
			err := res.ToError()
			if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, dht.TransactionTimeout) {
				// log.Printf("error querying %v: %v", addr, err)
			}
			tqr := res.TraversalQueryResult(addr)
			if tqr.ClosestData == nil {
				tqr.ResponseFrom = nil
			}
			return tqr
		},
		NodeFilter: s.TraversalNodeFilter,
	})
	nodes, err := s.TraversalStartingNodes()
	if err != nil {
		return
	}
	op.AddNodes(nodes)
	select {
	case <-op.Stalled():
	case <-ctx.Done():
		err = ctx.Err()
	}
	op.Stop()
	var wg sync.WaitGroup
	op.Closest().Range(func(elem k_nearest_nodes.Elem) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			token := elem.Data.(string)
			res := s.Put(ctx, dht.NewAddr(elem.Addr.UDP()), put, token, dht.QueryRateLimiting{})
			err := res.ToError()
			if err != nil {
				log.Printf("error putting to %v [token=%q]: %v", elem.Addr, token, err)
			} else {
				log.Printf("put to %v [token=%q]", elem.Addr, token)
			}
		}()
	})
	wg.Wait()
	stats = op.Stats()
	return
}
<|endoftext|>


<commit_before>package proxycfg

import (
	"context"
	"fmt"

	"github.com/hashicorp/consul/agent/cache"
	cachetype "github.com/hashicorp/consul/agent/cache-types"
	"github.com/hashicorp/consul/agent/structs"
)

type handlerIngressGateway struct {
	handlerState
}

func (s *handlerIngressGateway) initialize(ctx context.Context) (ConfigSnapshot, error) {
	snap := newConfigSnapshotFromServiceInstance(s.serviceInstance, s.stateConfig)
	// Watch for root changes
	err := s.cache.Notify(ctx, cachetype.ConnectCARootName, &structs.DCSpecificRequest{
		Datacenter:   s.source.Datacenter,
		QueryOptions: structs.QueryOptions{Token: s.token},
		Source:       *s.source,
	}, rootsWatchID, s.ch)
	if err != nil {
		return snap, err
	}

	// Watch this ingress gateway's config entry
	err = s.cache.Notify(ctx, cachetype.ConfigEntryName, &structs.ConfigEntryQuery{
		Kind:           structs.IngressGateway,
		Name:           s.service,
		Datacenter:     s.source.Datacenter,
		QueryOptions:   structs.QueryOptions{Token: s.token},
		EnterpriseMeta: s.proxyID.EnterpriseMeta,
	}, gatewayConfigWatchID, s.ch)
	if err != nil {
		return snap, err
	}

	// Watch the ingress-gateway's list of upstreams
	err = s.cache.Notify(ctx, cachetype.GatewayServicesName, &structs.ServiceSpecificRequest{
		Datacenter:     s.source.Datacenter,
		QueryOptions:   structs.QueryOptions{Token: s.token},
		ServiceName:    s.service,
		EnterpriseMeta: s.proxyID.EnterpriseMeta,
	}, gatewayServicesWatchID, s.ch)
	if err != nil {
		return snap, err
	}

	snap.IngressGateway.WatchedDiscoveryChains = make(map[string]context.CancelFunc)
	snap.IngressGateway.DiscoveryChain = make(map[string]*structs.CompiledDiscoveryChain)
	snap.IngressGateway.WatchedUpstreams = make(map[string]map[string]context.CancelFunc)
	snap.IngressGateway.WatchedUpstreamEndpoints = make(map[string]map[string]structs.CheckServiceNodes)
	snap.IngressGateway.WatchedGateways = make(map[string]map[string]context.CancelFunc)
	snap.IngressGateway.WatchedGatewayEndpoints = make(map[string]map[string]structs.CheckServiceNodes)
	snap.IngressGateway.Listeners = make(map[IngressListenerKey]structs.IngressListener)
	return snap, nil
}

func (s *handlerIngressGateway) handleUpdate(ctx context.Context, u cache.UpdateEvent, snap *ConfigSnapshot) error {
	if u.Err != nil {
		return fmt.Errorf("error filling agent cache: %v", u.Err)
	}

	switch {
	case u.CorrelationID == rootsWatchID:
		roots, ok := u.Result.(*structs.IndexedCARoots)
		if !ok {
			return fmt.Errorf("invalid type for response: %T", u.Result)
		}
		snap.Roots = roots
	case u.CorrelationID == gatewayConfigWatchID:
		resp, ok := u.Result.(*structs.ConfigEntryResponse)
		if !ok {
			return fmt.Errorf("invalid type for response: %T", u.Result)
		}
		gatewayConf, ok := resp.Entry.(*structs.IngressGatewayConfigEntry)
		if !ok {
			return fmt.Errorf("invalid type for config entry: %T", resp.Entry)
		}

		snap.IngressGateway.GatewayConfigLoaded = true
		snap.IngressGateway.TLSConfig = gatewayConf.TLS

		// Load each listener's config from the config entry so we don't have to
		// pass listener config through "upstreams" types as that grows.
		for _, l := range gatewayConf.Listeners {
			key := IngressListenerKeyFromListener(l)
			snap.IngressGateway.Listeners[key] = l
		}

		if err := s.watchIngressLeafCert(ctx, snap); err != nil {
			return err
		}

	case u.CorrelationID == gatewayServicesWatchID:
		services, ok := u.Result.(*structs.IndexedGatewayServices)
		if !ok {
			return fmt.Errorf("invalid type for response: %T", u.Result)
		}

		// Update our upstreams and watches.
		var hosts []string
		watchedSvcs := make(map[string]struct{})
		upstreamsMap := make(map[IngressListenerKey]structs.Upstreams)
		for _, service := range services.Services {
			u := makeUpstream(service)

			watchOpts := discoveryChainWatchOpts{
				id:         u.Identifier(),
				name:       u.DestinationName,
				namespace:  u.DestinationNamespace,
				partition:  u.DestinationPartition,
				datacenter: s.source.Datacenter,
			}
			up := &handlerUpstreams{handlerState: s.handlerState}
			err := up.watchDiscoveryChain(ctx, snap, watchOpts)
			if err != nil {
				return fmt.Errorf("failed to watch discovery chain for %s: %v", u.Identifier(), err)
			}
			watchedSvcs[u.Identifier()] = struct{}{}

			hosts = append(hosts, service.Hosts...)

			id := IngressListenerKeyFromGWService(*service)
			upstreamsMap[id] = append(upstreamsMap[id], u)
		}

		snap.IngressGateway.Upstreams = upstreamsMap
		snap.IngressGateway.Hosts = hosts
		snap.IngressGateway.HostsSet = true

		for id, cancelFn := range snap.IngressGateway.WatchedDiscoveryChains {
			if _, ok := watchedSvcs[id]; !ok {
				cancelFn()
				delete(snap.IngressGateway.WatchedDiscoveryChains, id)
			}
		}

		if err := s.watchIngressLeafCert(ctx, snap); err != nil {
			return err
		}

	default:
		return (*handlerUpstreams)(s).handleUpdateUpstreams(ctx, u, snap)
	}

	return nil
}

// Note: Ingress gateways are always bound to ports and never unix sockets.
// This means LocalBindPort is the only possibility
func makeUpstream(g *structs.GatewayService) structs.Upstream {
	upstream := structs.Upstream{
		DestinationName:      g.Service.Name,
		DestinationNamespace: g.Service.NamespaceOrEmpty(),
		DestinationPartition: g.Gateway.PartitionOrEmpty(),
		LocalBindPort:        g.Port,
		IngressHosts:         g.Hosts,
		// Pass the protocol that was configured on the ingress listener in order
		// to force that protocol on the Envoy listener.
		Config: map[string]interface{}{
			"protocol": g.Protocol,
		},
	}

	return upstream
}

func (s *handlerIngressGateway) watchIngressLeafCert(ctx context.Context, snap *ConfigSnapshot) error {
	// Note that we DON'T test for TLS.Enabled because we need a leaf cert for the
	// gateway even without TLS to use as a client cert.
	if !snap.IngressGateway.GatewayConfigLoaded || !snap.IngressGateway.HostsSet {
		return nil
	}

	// Watch the leaf cert
	if snap.IngressGateway.LeafCertWatchCancel != nil {
		snap.IngressGateway.LeafCertWatchCancel()
	}
	ctx, cancel := context.WithCancel(ctx)
	err := s.cache.Notify(ctx, cachetype.ConnectCALeafName, &cachetype.ConnectCALeafRequest{
		Datacenter:     s.source.Datacenter,
		Token:          s.token,
		Service:        s.service,
		DNSSAN:         s.generateIngressDNSSANs(snap),
		EnterpriseMeta: s.proxyID.EnterpriseMeta,
	}, leafWatchID, s.ch)
	if err != nil {
		cancel()
		return err
	}
	snap.IngressGateway.LeafCertWatchCancel = cancel

	return nil
}

func (s *handlerIngressGateway) generateIngressDNSSANs(snap *ConfigSnapshot) []string {
	// Update our leaf cert watch with wildcard entries for our DNS domains as well as any
	// configured custom hostnames from the service.
	if !snap.IngressGateway.TLSConfig.Enabled {
		return nil
	}

	var dnsNames []string
	namespaces := make(map[string]struct{})
	for _, upstreams := range snap.IngressGateway.Upstreams {
		for _, u := range upstreams {
			namespaces[u.DestinationNamespace] = struct{}{}
		}
	}

	for ns := range namespaces {
		// The default namespace is special cased in DNS resolution, so special
		// case it here.
		if ns == structs.IntentionDefaultNamespace {
			ns = ""
		} else {
			ns = ns + "."
		}

		dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s", ns, s.dnsConfig.Domain))
		dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s.%s", ns, s.source.Datacenter, s.dnsConfig.Domain))
		if s.dnsConfig.AltDomain != "" {
			dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s", ns, s.dnsConfig.AltDomain))
			dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s.%s", ns, s.source.Datacenter, s.dnsConfig.AltDomain))
		}
	}

	dnsNames = append(dnsNames, snap.IngressGateway.Hosts...)

	return dnsNames
}
<commit_msg>Revert abandonned changes to proxycfg for Ent test consistency<commit_after>package proxycfg

import (
	"context"
	"fmt"

	"github.com/hashicorp/consul/agent/cache"
	cachetype "github.com/hashicorp/consul/agent/cache-types"
	"github.com/hashicorp/consul/agent/structs"
)

type handlerIngressGateway struct {
	handlerState
}

func (s *handlerIngressGateway) initialize(ctx context.Context) (ConfigSnapshot, error) {
	snap := newConfigSnapshotFromServiceInstance(s.serviceInstance, s.stateConfig)
	// Watch for root changes
	err := s.cache.Notify(ctx, cachetype.ConnectCARootName, &structs.DCSpecificRequest{
		Datacenter:   s.source.Datacenter,
		QueryOptions: structs.QueryOptions{Token: s.token},
		Source:       *s.source,
	}, rootsWatchID, s.ch)
	if err != nil {
		return snap, err
	}

	// Watch this ingress gateway's config entry
	err = s.cache.Notify(ctx, cachetype.ConfigEntryName, &structs.ConfigEntryQuery{
		Kind:           structs.IngressGateway,
		Name:           s.service,
		Datacenter:     s.source.Datacenter,
		QueryOptions:   structs.QueryOptions{Token: s.token},
		EnterpriseMeta: s.proxyID.EnterpriseMeta,
	}, gatewayConfigWatchID, s.ch)
	if err != nil {
		return snap, err
	}

	// Watch the ingress-gateway's list of upstreams
	err = s.cache.Notify(ctx, cachetype.GatewayServicesName, &structs.ServiceSpecificRequest{
		Datacenter:     s.source.Datacenter,
		QueryOptions:   structs.QueryOptions{Token: s.token},
		ServiceName:    s.service,
		EnterpriseMeta: s.proxyID.EnterpriseMeta,
	}, gatewayServicesWatchID, s.ch)
	if err != nil {
		return snap, err
	}

	snap.IngressGateway.WatchedDiscoveryChains = make(map[string]context.CancelFunc)
	snap.IngressGateway.DiscoveryChain = make(map[string]*structs.CompiledDiscoveryChain)
	snap.IngressGateway.WatchedUpstreams = make(map[string]map[string]context.CancelFunc)
	snap.IngressGateway.WatchedUpstreamEndpoints = make(map[string]map[string]structs.CheckServiceNodes)
	snap.IngressGateway.WatchedGateways = make(map[string]map[string]context.CancelFunc)
	snap.IngressGateway.WatchedGatewayEndpoints = make(map[string]map[string]structs.CheckServiceNodes)
	snap.IngressGateway.Listeners = make(map[IngressListenerKey]structs.IngressListener)
	return snap, nil
}

func (s *handlerIngressGateway) handleUpdate(ctx context.Context, u cache.UpdateEvent, snap *ConfigSnapshot) error {
	if u.Err != nil {
		return fmt.Errorf("error filling agent cache: %v", u.Err)
	}

	switch {
	case u.CorrelationID == rootsWatchID:
		roots, ok := u.Result.(*structs.IndexedCARoots)
		if !ok {
			return fmt.Errorf("invalid type for response: %T", u.Result)
		}
		snap.Roots = roots
	case u.CorrelationID == gatewayConfigWatchID:
		resp, ok := u.Result.(*structs.ConfigEntryResponse)
		if !ok {
			return fmt.Errorf("invalid type for response: %T", u.Result)
		}
		gatewayConf, ok := resp.Entry.(*structs.IngressGatewayConfigEntry)
		if !ok {
			return fmt.Errorf("invalid type for config entry: %T", resp.Entry)
		}

		snap.IngressGateway.GatewayConfigLoaded = true
		snap.IngressGateway.TLSConfig = gatewayConf.TLS

		// Load each listener's config from the config entry so we don't have to
		// pass listener config through "upstreams" types as that grows.
		for _, l := range gatewayConf.Listeners {
			key := IngressListenerKeyFromListener(l)
			snap.IngressGateway.Listeners[key] = l
		}

		if err := s.watchIngressLeafCert(ctx, snap); err != nil {
			return err
		}

	case u.CorrelationID == gatewayServicesWatchID:
		services, ok := u.Result.(*structs.IndexedGatewayServices)
		if !ok {
			return fmt.Errorf("invalid type for response: %T", u.Result)
		}

		// Update our upstreams and watches.
		var hosts []string
		watchedSvcs := make(map[string]struct{})
		upstreamsMap := make(map[IngressListenerKey]structs.Upstreams)
		for _, service := range services.Services {
			u := makeUpstream(service)

			watchOpts := discoveryChainWatchOpts{
				id:         u.Identifier(),
				name:       u.DestinationName,
				namespace:  u.DestinationNamespace,
				partition:  u.DestinationPartition,
				datacenter: s.source.Datacenter,
			}
			up := &handlerUpstreams{handlerState: s.handlerState}
			err := up.watchDiscoveryChain(ctx, snap, watchOpts)
			if err != nil {
				return fmt.Errorf("failed to watch discovery chain for %s: %v", u.Identifier(), err)
			}
			watchedSvcs[u.Identifier()] = struct{}{}

			hosts = append(hosts, service.Hosts...)

			id := IngressListenerKeyFromGWService(*service)
			upstreamsMap[id] = append(upstreamsMap[id], u)
		}

		snap.IngressGateway.Upstreams = upstreamsMap
		snap.IngressGateway.Hosts = hosts
		snap.IngressGateway.HostsSet = true

		for id, cancelFn := range snap.IngressGateway.WatchedDiscoveryChains {
			if _, ok := watchedSvcs[id]; !ok {
				cancelFn()
				delete(snap.IngressGateway.WatchedDiscoveryChains, id)
			}
		}

		if err := s.watchIngressLeafCert(ctx, snap); err != nil {
			return err
		}

	default:
		return (*handlerUpstreams)(s).handleUpdateUpstreams(ctx, u, snap)
	}

	return nil
}

// Note: Ingress gateways are always bound to ports and never unix sockets.
// This means LocalBindPort is the only possibility
func makeUpstream(g *structs.GatewayService) structs.Upstream {
	upstream := structs.Upstream{
		DestinationName:      g.Service.Name,
		DestinationNamespace: g.Service.NamespaceOrDefault(),
		DestinationPartition: g.Gateway.PartitionOrDefault(),
		LocalBindPort:        g.Port,
		IngressHosts:         g.Hosts,
		// Pass the protocol that was configured on the ingress listener in order
		// to force that protocol on the Envoy listener.
		Config: map[string]interface{}{
			"protocol": g.Protocol,
		},
	}

	return upstream
}

func (s *handlerIngressGateway) watchIngressLeafCert(ctx context.Context, snap *ConfigSnapshot) error {
	// Note that we DON'T test for TLS.Enabled because we need a leaf cert for the
	// gateway even without TLS to use as a client cert.
	if !snap.IngressGateway.GatewayConfigLoaded || !snap.IngressGateway.HostsSet {
		return nil
	}

	// Watch the leaf cert
	if snap.IngressGateway.LeafCertWatchCancel != nil {
		snap.IngressGateway.LeafCertWatchCancel()
	}
	ctx, cancel := context.WithCancel(ctx)
	err := s.cache.Notify(ctx, cachetype.ConnectCALeafName, &cachetype.ConnectCALeafRequest{
		Datacenter:     s.source.Datacenter,
		Token:          s.token,
		Service:        s.service,
		DNSSAN:         s.generateIngressDNSSANs(snap),
		EnterpriseMeta: s.proxyID.EnterpriseMeta,
	}, leafWatchID, s.ch)
	if err != nil {
		cancel()
		return err
	}
	snap.IngressGateway.LeafCertWatchCancel = cancel

	return nil
}

func (s *handlerIngressGateway) generateIngressDNSSANs(snap *ConfigSnapshot) []string {
	// Update our leaf cert watch with wildcard entries for our DNS domains as well as any
	// configured custom hostnames from the service.
	if !snap.IngressGateway.TLSConfig.Enabled {
		return nil
	}

	var dnsNames []string
	namespaces := make(map[string]struct{})
	for _, upstreams := range snap.IngressGateway.Upstreams {
		for _, u := range upstreams {
			namespaces[u.DestinationNamespace] = struct{}{}
		}
	}

	for ns := range namespaces {
		// The default namespace is special cased in DNS resolution, so special
		// case it here.
		if ns == structs.IntentionDefaultNamespace {
			ns = ""
		} else {
			ns = ns + "."
		}

		dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s", ns, s.dnsConfig.Domain))
		dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s.%s", ns, s.source.Datacenter, s.dnsConfig.Domain))
		if s.dnsConfig.AltDomain != "" {
			dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s", ns, s.dnsConfig.AltDomain))
			dnsNames = append(dnsNames, fmt.Sprintf("*.ingress.%s%s.%s", ns, s.source.Datacenter, s.dnsConfig.AltDomain))
		}
	}

	dnsNames = append(dnsNames, snap.IngressGateway.Hosts...)

	return dnsNames
}
<|endoftext|>


<commit_before>package matcher

import (
	"fmt"
	"regexp"
	"strings"
)

func fuzzify(input string) string {
	chars := strings.Split(input, "")
	fuzzy := ".*" + strings.Join(chars, ".*")
	fmt.Println(fuzzy)
	return fuzzy
}

func Match(input string, list []string) []string {
	pattern := regexp.MustCompile("(?i)" + fuzzify(input))
	matches := []string{}

	for _, dir := range list {
		match := pattern.MatchString(dir)
		if match {
			matches = append(matches, dir)
		}
	}

	return matches
}
<commit_msg>instead of empty string array initializer just use new()<commit_after>package matcher

import (
	"fmt"
	"regexp"
	"strings"
)

func fuzzify(input string) string {
	chars := strings.Split(input, "")
	fuzzy := ".*" + strings.Join(chars, ".*")
	fmt.Println(fuzzy)
	return fuzzy
}

func Match(input string, list []string) []string {
	pattern := regexp.MustCompile("(?i)" + fuzzify(input))
	matches := new([]string)

	for _, dir := range list {
		match := pattern.MatchString(dir)
		if match {
			matches = append(matches, dir)
		}
	}

	return matches
}
<|endoftext|>