<commit_before>package main

import (
	"io"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/auth"
	"github.com/taskcluster/webhooktunnel/util"
	"github.com/taskcluster/webhooktunnel/whclient"
)

// whclient <clientID> <accessToken> <targetPort>
const usage = `whcli <clientID> <accessToken> <targetPort>`

func main() {
	log.SetOutput(os.Stdout)
	log.SetFormatter(&log.JSONFormatter{})

	if len(os.Args) != 4 {
		log.Fatal(usage)
	}

	clientID, accessToken := os.Args[1], os.Args[2]
	targetPort, err := strconv.Atoi(os.Args[3])
	if err != nil || targetPort < 0 || targetPort > 65535 {
		log.Fatal(usage)
	}

	configurer := func() (whclient.Config, error) {
		creds := &tcclient.Credentials{
			ClientID:    clientID,
			AccessToken: accessToken,
		}
		myAuth := auth.New(creds)
		whtResponse, err := myAuth.WebhooktunnelToken()
		if err != nil {
			return whclient.Config{}, err
		}
		proxyURL := util.MakeWsURL(whtResponse.ProxyURL)
		return whclient.Config{
			ID:        whtResponse.TunnelID,
			Token:     whtResponse.Token,
			ProxyAddr: proxyURL,
			UseDomain: true,
		}, nil
	}

	client, err := whclient.New(configurer)
	if err != nil {
		log.Fatal(err)
	}

	log.WithFields(log.Fields{"url": client.URL()}).Info("connected to proxy")

	waitTime := 5 * time.Millisecond
	for {
		stream, err := client.Accept()
		if err != nil {
			ne, ok := err.(net.Error)
			if !ok || !ne.Temporary() {
				log.Fatal(err)
			}
			// wait if temporary
			time.Sleep(waitTime)
			waitTime = 2 * waitTime
			continue
		}
		// reset wait time
		waitTime = 5 * time.Millisecond
		go forward(stream, targetPort)
	}
}

// forwards multiplexed stream to port
func forward(stream net.Conn, port int) {
	strPort := strconv.Itoa(port)
	conn, err := net.Dial("tcp", ":"+strPort)
	if err != nil {
		return
	}
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		_, _ = io.Copy(stream, conn)
	}()
	go func() {
		defer wg.Done()
		_, _ = io.Copy(conn, stream)
	}()
	wg.Wait()
}
<commit_msg>use new whclient<commit_after>package main

import (
	"io"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/auth"
	"github.com/taskcluster/webhooktunnel/whclient"
)

// whclient <clientID> <accessToken> <targetPort>
const usage = `whclient <clientID> <accessToken> <targetPort>`

func main() {
	log.SetOutput(os.Stdout)
	log.SetFormatter(&log.JSONFormatter{})

	if len(os.Args) != 4 {
		log.Fatal(usage)
	}

	clientID, accessToken := os.Args[1], os.Args[2]
	targetPort, err := strconv.Atoi(os.Args[3])
	if err != nil || targetPort < 0 || targetPort > 65535 {
		log.Fatal(usage)
	}

	configurer := func() (whclient.Config, error) {
		creds := &tcclient.Credentials{
			ClientID:    clientID,
			AccessToken: accessToken,
		}
		myAuth := auth.New(creds)
		whtResponse, err := myAuth.WebhooktunnelToken()
		if err != nil {
			return whclient.Config{}, err
		}
		return whclient.Config{
			ID:        whtResponse.TunnelID,
			Token:     whtResponse.Token,
			ProxyAddr: whtResponse.ProxyURL,
		}, nil
	}

	client, err := whclient.New(configurer)
	if err != nil {
		log.Fatal(err)
	}

	log.WithFields(log.Fields{"url": client.URL()}).Info("connected to proxy")

	waitTime := 5 * time.Millisecond
	for {
		stream, err := client.Accept()
		if err != nil {
			ne, ok := err.(net.Error)
			if !ok || !ne.Temporary() {
				log.Fatal(err)
			}
			// wait if temporary
			time.Sleep(waitTime)
			waitTime = 2 * waitTime
			continue
		}
		// reset wait time
		waitTime = 5 * time.Millisecond
		go forward(stream, targetPort)
	}
}

// forwards multiplexed stream to port
func forward(stream net.Conn, port int) {
	strPort := strconv.Itoa(port)
	conn, err := net.Dial("tcp", ":"+strPort)
	if err != nil {
		return
	}
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		_, _ = io.Copy(stream, conn)
	}()
	go func() {
		defer wg.Done()
		_, _ = io.Copy(conn, stream)
	}()
	wg.Wait()
}
<|endoftext|>


<commit_before>package resp

import (
	"embed"
	"encoding/xml"
	"fmt"
	"io/fs"

	"github.com/GeoNet/delta/internal/stationxml/v1.1"
)

//TODO: add embed when populated
var locations = []string{"files", "auto"}

//go:embed files/*.xml
//go:embed auto/*.xml
var files embed.FS

// Resp returns a pointer to an embeded stationxml Response if present.
func Lookup(response string) (*stationxml.ResponseType, error) {

	for _, l := range locations {
		names, err := fs.Glob(files, fmt.Sprintf("%s/%s.xml", l, response))
		if err != nil {
			return nil, err
		}

		for _, name := range names {
			data, err := fs.ReadFile(files, name)
			if err != nil {
				return nil, err
			}

			var resp stationxml.ResponseType
			if err := xml.Unmarshal(data, &resp); err != nil {
				return nil, err
			}
			return &resp, nil
		}
	}

	return nil, nil
}
<commit_msg>expand resp to allow a base call<commit_after>package resp

import (
	"embed"
	"encoding/xml"
	"fmt"
	"io/fs"
	"os"

	"github.com/GeoNet/delta/internal/stationxml/v1.1"
)

//TODO: add embed when populated
var locations = []string{"files", "auto"}

//go:embed files/*.xml
//go:embed auto/*.xml
var files embed.FS

// LookupFS returns a pointer to an stationxml Response if present in the given file system.
func LookupFS(fsys fs.FS, response string) (*stationxml.ResponseType, error) {
	for _, l := range locations {
		names, err := fs.Glob(fsys, fmt.Sprintf("%s/%s.xml", l, response))
		if err != nil {
			return nil, err
		}

		for _, name := range names {
			data, err := fs.ReadFile(files, name)
			if err != nil {
				return nil, err
			}

			var resp stationxml.ResponseType
			if err := xml.Unmarshal(data, &resp); err != nil {
				return nil, err
			}
			return &resp, nil
		}
	}

	return nil, nil
}

// Lookup returns a pointer to an embeded stationxml Response if present.
func Lookup(response string) (*stationxml.ResponseType, error) {
	return LookupFS(files, response)
}

// LookupDir returns a pointer to an stationxml Response if stored in the given directory.
func LookupDir(path string, response string) (*stationxml.ResponseType, error) {
	return LookupFS(os.DirFS(path), response)
}

// LookupBase returns a pointer to an stationxml Response either stored in a given directory or
// in the embedded files if no base directory given.
func LookupBase(base string, response string) (*stationxml.ResponseType, error) {
	if base != "" {
		return LookupDir(base, response)
	}
	return Lookup(response)
}
<|endoftext|>


<commit_before>package util

import (
	"testing"

	"github.com/davecgh/go-spew/spew"
	"github.com/omakoto/go-common/src/fileutils"
)

func Ar(a ...string) []string {
	return a
}

func SlicesEqual(a []string, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i, v := range a {
		if b[i] != v {
			return false
		}
	}
	return true
}

func AssertStringsEqual(t *testing.T, input string, expected string, actual string) {
	if expected != actual {
		t.Errorf("input=%s expected=%s actual=%s", input, expected, actual)
	}
}

func AssertStringSlicesEqual(t *testing.T, input string, expected []string, actual []string) {
	if !SlicesEqual(expected, actual) {
		t.Errorf("input=%s expected=%s actual=%s", input, spew.Sdump(expected), spew.Sdump(actual))
	}
}

func AssertFileExist(t *testing.T, file string) {
	if !fileutils.FileExists(file) {
		t.Errorf("File %s not createtd.", file)
	}
}
<commit_msg>latest<commit_after>package util

import (
	"testing"

	"github.com/davecgh/go-spew/spew"
	"github.com/omakoto/go-common/src/fileutils"
)

func Ar(a ...string) []string {
	if a == nil {
		return make([]string, 0)
	}
	return a
}

func SlicesEqual(a []string, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i, v := range a {
		if b[i] != v {
			return false
		}
	}
	return true
}

func AssertStringsEqual(t *testing.T, input string, expected string, actual string) {
	if expected != actual {
		t.Errorf("input=%s expected=%s actual=%s", input, expected, actual)
	}
}

func AssertStringSlicesEqual(t *testing.T, input string, expected []string, actual []string) {
	if !SlicesEqual(expected, actual) {
		t.Errorf("input=%s expected=%s actual=%s", input, spew.Sdump(expected), spew.Sdump(actual))
	}
}

func AssertFileExist(t *testing.T, file string) {
	if !fileutils.FileExists(file) {
		t.Errorf("File %s not createtd.", file)
	}
}
<|endoftext|>


<commit_before>// Package manager provides interface between handler and lower level implementation
// such as geoloader.
package manager

// The implementation is currently rather naive.  Eviction is done based only on whether
// there is a pending request, and there are already the max number of datasets loaded.
// A later implementation will use LRU and dead time to make this determination.
//
// Behavior:
//   If a legacy dataset is requests, return the CurrentAnnotator instead.
//   If the requested dataset is loaded, return it.
//   If the requested dataset is loading, return ErrPendingAnnotatorLoad
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply with ErrPendingAnnotatorLoad (even though this isn't true)
//      B: If there is room to load it?
//       YES: start loading it, and return ErrPendingAnnotatorLoad
//        NO: kick out an existing dataset and return ErrPendingAnnotatorLoad.
//
// Please modify with extreme caution.  The lock MUST be held when ACCESSING any field
// of AnnotatorMap.

// Note that the system may evict up to the number of pending loads, so at any given time,
// there may only be MaxDatasetInMemory = MaxPending actually loaded.

// Also note that anyone holding an annotator will prevent it from being collected by the
// GC, so simply evicting it is not a guarantee that the memory will be reclaimed.

import (
	"errors"
	"log"
	"sync"
	"time"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geoloader"
	"github.com/m-lab/annotation-service/metrics"
)

var (
	// These are vars instead of consts to facilitate testing.
	MaxDatasetInMemory = 12 // Limit on number of loaded datasets
	MaxPending         = 2  // Limit on number of concurrently loading datasets.

	// ErrNilDataset is returned when CurrentAnnotator is nil.
	ErrNilDataset = errors.New("CurrentAnnotator is nil")

	// ErrPendingAnnotatorLoad is returned when a new annotator is requested, but not yet loaded.
	ErrPendingAnnotatorLoad = errors.New("annotator is loading")

	// ErrAnnotatorLoadFailed is returned when a requested annotator has failed to load.
	ErrAnnotatorLoadFailed = errors.New("unable to load annoator")

	// These are UNEXPECTED errors!!
	ErrGoroutineNotOwner  = errors.New("Goroutine not owner")
	ErrMapEntryAlreadySet = errors.New("Map entry already set")

	// A mutex to make sure that we are not reading from the CurrentAnnotator
	// pointer while trying to update it
	currentDataMutex = &sync.RWMutex{}

	// CurrentAnnotator points to a GeoDataset struct containing the absolute
	// latest data for the annotator to search and reply with
	CurrentAnnotator api.Annotator

	// ArchivedLoader points to a AnnotatorMap struct containing the archived
	// Geolite2 and legacy dataset in memory.
	archivedAnnotator = NewAnnotatorMap(geoloader.ArchivedLoader)
)

// AnnotatorMap manages all loading and fetching of Annotators.
// TODO - should we call this AnnotatorCache?
// TODO - should this be a generic cache of interface{}?
//
// Synchronization:
//  All accesses must hold the mutex.  If an element is not found, the
//  goroutine may attempt to take responsibility for loading it by obtaining
//  the write lock, and writing an entry with a nil pointer.
// TODO - still need a strategy for dealing with persistent errors.
type AnnotatorMap struct {
	// Keys are filename of the datasets.
	annotators map[string]api.Annotator
	// Lock to be held when reading or writing the map.
	mutex      sync.RWMutex
	numPending int
	loader     func(string) (api.Annotator, error)
}

// NewAnnotatorMap creates a new map that will use the provided loader for loading new Annotators.
func NewAnnotatorMap(loader func(string) (api.Annotator, error)) *AnnotatorMap {
	return &AnnotatorMap{annotators: make(map[string]api.Annotator), loader: loader}
}

// NOTE: Should only be called by checkAndLoadAnnotator.
// The calling goroutine should "own" the responsibility for
// setting the annotator.
func (am *AnnotatorMap) setAnnotatorIfNil(key string, ann api.Annotator) error {
	am.mutex.Lock()
	defer am.mutex.Unlock()

	old, ok := am.annotators[key]
	if !ok {
		log.Println("This should never happen", ErrGoroutineNotOwner)
		metrics.ErrorTotal.WithLabelValues("WrongOwner").Inc()
		return ErrGoroutineNotOwner
	}
	if old != nil {
		log.Println("This should never happen", ErrMapEntryAlreadySet)
		metrics.ErrorTotal.WithLabelValues("MapEntryAlreadySet").Inc()
		return ErrMapEntryAlreadySet
	}

	am.annotators[key] = ann
	metrics.LoadCount.Inc()
	metrics.PendingLoads.Dec()
	metrics.DatasetCount.Inc()
	am.numPending--
	log.Println("Loaded", key)
	return nil
}

// This creates a reservation for loading a dataset, IFF map entry is empty (not nil or populated)
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply false
//      B: If there is room to load it?
//       YES: make the reservation (by setting entry to nil) and return true.
//        NO: kick out an existing dataset and return false.
func (am *AnnotatorMap) maybeSetNil(key string) bool {
	am.mutex.Lock()
	defer am.mutex.Unlock()
	_, ok := am.annotators[key]
	if ok {
		// Another goroutine is already responsible for loading.
		return false
	}

	if am.numPending >= MaxPending {
		log.Println("Too many pending", key)
		return false
	}
	// Check the number of datasets in memory. Given the memory
	// limit, some dataset may be removed from memory if needed.
	if len(am.annotators) >= MaxDatasetInMemory {
		for fileKey := range am.annotators {
			if am.annotators[fileKey] != nil {
				log.Println("removing Geolite2 dataset " + fileKey)
				delete(am.annotators, fileKey)
				metrics.EvictionCount.Inc()
				metrics.DatasetCount.Dec()
				break
			}
		}
		return false
	}

	// Place marker so that other requesters know it is loading.
	am.annotators[key] = nil
	metrics.PendingLoads.Inc()
	am.numPending++
	return true
}

// This synchronously attempts to set map entry to nil, and
// if successful, proceeds to asynchronously load the new dataset.
func (am *AnnotatorMap) checkAndLoadAnnotator(key string) {
	reserved := am.maybeSetNil(key)
	log.Println(key)
	if reserved {
		// This goroutine now has exclusive ownership of the
		// map entry, and the responsibility for loading the annotator.
		go func(key string) {
			newAnn, err := am.loader(key)
			if err != nil {
				// TODO add a metric
				log.Println(err)
				return
			}
			// Set the new annotator value.  Entry should be nil.
			err = am.setAnnotatorIfNil(key, newAnn)
			if err != nil {
				// TODO add a metric
				log.Println(err)
			}
		}(key)
	}
}

// GetAnnotator gets the named annotator, if already in the map.
// If not already loaded, this will trigger loading, and return ErrPendingAnnotatorLoad
func (am *AnnotatorMap) GetAnnotator(key string) (api.Annotator, error) {
	am.mutex.RLock()
	ann, ok := am.annotators[key]
	am.mutex.RUnlock()

	if !ok {
		log.Println("There is not yet any entry for this date.  Try to load " + key)
		am.checkAndLoadAnnotator(key)
		metrics.RejectionCount.WithLabelValues("New Dataset").Inc()
		return nil, ErrPendingAnnotatorLoad
	}

	if ann == nil {
		// Another goroutine is already loading this entry.  Return error.
		metrics.RejectionCount.WithLabelValues("Dataset Pending").Inc()
		return nil, ErrPendingAnnotatorLoad
	}
	return ann, nil
}

// GetAnnotator returns the correct annotator to use for a given timestamp.
// TODO: Update to properly handle legacy datasets.
func GetAnnotator(date time.Time) (api.Annotator, error) {
	// key := strconv.FormatInt(date.Unix(), encodingBase)
	if date.After(geoloader.Latest()) {
		currentDataMutex.RLock()
		ann := CurrentAnnotator
		currentDataMutex.RUnlock()
		return ann, nil
	}

	filename, err := geoloader.SelectArchivedDataset(date)

	if err != nil {
		metrics.RejectionCount.WithLabelValues("Selection Error").Inc()
		return nil, err
	}

	return archivedAnnotator.GetAnnotator(filename)
}

// InitDataset will update the filename list of archived dataset in memory
// and load the latest Geolite2 dataset in memory.
func InitDataset() {
	geoloader.UpdateArchivedFilenames()

	ann := geoloader.GetLatestData()
	currentDataMutex.Lock()
	CurrentAnnotator = ann
	currentDataMutex.Unlock()
}
<commit_msg>improve errors and logs<commit_after>// Package manager provides interface between handler and lower level implementation
// such as geoloader.
package manager

// The implementation is currently rather naive.  Eviction is done based only on whether
// there is a pending request, and there are already the max number of datasets loaded.
// A later implementation will use LRU and dead time to make this determination.
//
// Behavior:
//   If a legacy dataset is requests, return the CurrentAnnotator instead.
//   If the requested dataset is loaded, return it.
//   If the requested dataset is loading, return ErrPendingAnnotatorLoad
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply with ErrPendingAnnotatorLoad (even though this isn't true)
//      B: If there is room to load it?
//       YES: start loading it, and return ErrPendingAnnotatorLoad
//        NO: kick out an existing dataset and return ErrPendingAnnotatorLoad.
//
// Please modify with extreme caution.  The lock MUST be held when ACCESSING any field
// of AnnotatorMap.

// Note that the system may evict up to the number of pending loads, so at any given time,
// there may only be MaxDatasetInMemory = MaxPending actually loaded.

// Also note that anyone holding an annotator will prevent it from being collected by the
// GC, so simply evicting it is not a guarantee that the memory will be reclaimed.

import (
	"errors"
	"log"
	"sync"
	"time"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geoloader"
	"github.com/m-lab/annotation-service/metrics"
)

var (
	// These are vars instead of consts to facilitate testing.
	MaxDatasetInMemory = 12 // Limit on number of loaded datasets
	MaxPending         = 2  // Limit on number of concurrently loading datasets.

	// ErrNilDataset is returned when CurrentAnnotator is nil.
	ErrNilDataset = errors.New("CurrentAnnotator is nil")

	// ErrPendingAnnotatorLoad is returned when a new annotator is requested, but not yet loaded.
	ErrPendingAnnotatorLoad = errors.New("annotator is loading")

	// ErrAnnotatorLoadFailed is returned when a requested annotator has failed to load.
	ErrAnnotatorLoadFailed = errors.New("unable to load annoator")

	// These are UNEXPECTED errors!!
	ErrGoroutineNotOwner  = errors.New("Goroutine not owner")
	ErrMapEntryAlreadySet = errors.New("Map entry already set")

	// A mutex to make sure that we are not reading from the CurrentAnnotator
	// pointer while trying to update it
	currentDataMutex = &sync.RWMutex{}

	// CurrentAnnotator points to a GeoDataset struct containing the absolute
	// latest data for the annotator to search and reply with
	CurrentAnnotator api.Annotator

	// ArchivedLoader points to a AnnotatorMap struct containing the archived
	// Geolite2 and legacy dataset in memory.
	archivedAnnotator = NewAnnotatorMap(geoloader.ArchivedLoader)
)

// AnnotatorMap manages all loading and fetching of Annotators.
// TODO - should we call this AnnotatorCache?
// TODO - should this be a generic cache of interface{}?
//
// Synchronization:
//  All accesses must hold the mutex.  If an element is not found, the
//  goroutine may attempt to take responsibility for loading it by obtaining
//  the write lock, and writing an entry with a nil pointer.
// TODO - still need a strategy for dealing with persistent errors.
type AnnotatorMap struct {
	// Keys are filename of the datasets.
	annotators map[string]api.Annotator
	// Lock to be held when reading or writing the map.
	mutex      sync.RWMutex
	numPending int
	loader     func(string) (api.Annotator, error)
}

// NewAnnotatorMap creates a new map that will use the provided loader for loading new Annotators.
func NewAnnotatorMap(loader func(string) (api.Annotator, error)) *AnnotatorMap {
	return &AnnotatorMap{annotators: make(map[string]api.Annotator), loader: loader}
}

// NOTE: Should only be called by checkAndLoadAnnotator.
// The calling goroutine should "own" the responsibility for
// setting the annotator.
func (am *AnnotatorMap) setAnnotatorIfNil(key string, ann api.Annotator) error {
	am.mutex.Lock()
	defer am.mutex.Unlock()

	old, ok := am.annotators[key]
	if !ok {
		log.Println("This should never happen", ErrGoroutineNotOwner)
		metrics.ErrorTotal.WithLabelValues("WrongOwner").Inc()
		return ErrGoroutineNotOwner
	}
	if old != nil {
		log.Println("This should never happen", ErrMapEntryAlreadySet)
		metrics.ErrorTotal.WithLabelValues("MapEntryAlreadySet").Inc()
		return ErrMapEntryAlreadySet
	}

	am.annotators[key] = ann
	metrics.LoadCount.Inc()
	metrics.PendingLoads.Dec()
	metrics.DatasetCount.Inc()
	am.numPending--
	log.Println("Loaded", key)
	return nil
}

// This creates a reservation for loading a dataset, IFF map entry is empty (not nil or populated)
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply false
//      B: If there is room to load it?
//       YES: make the reservation (by setting entry to nil) and return true.
//        NO: kick out an existing dataset and return false.
func (am *AnnotatorMap) maybeSetNil(key string) bool {
	am.mutex.Lock()
	defer am.mutex.Unlock()
	_, ok := am.annotators[key]
	if ok {
		// Another goroutine is already responsible for loading.
		return false
	}

	if am.numPending >= MaxPending {
		return false
	}
	// Check the number of datasets in memory. Given the memory
	// limit, some dataset may be removed from memory if needed.
	if len(am.annotators) >= MaxDatasetInMemory {
		for fileKey := range am.annotators {
			if am.annotators[fileKey] != nil {
				log.Println("removing Geolite2 dataset " + fileKey)
				delete(am.annotators, fileKey)
				metrics.EvictionCount.Inc()
				metrics.DatasetCount.Dec()
				break
			}
		}
		return false
	}

	// Place marker so that other requesters know it is loading.
	am.annotators[key] = nil
	metrics.PendingLoads.Inc()
	am.numPending++
	return true
}

// This synchronously attempts to set map entry to nil, and
// if successful, proceeds to asynchronously load the new dataset.
func (am *AnnotatorMap) checkAndLoadAnnotator(key string) {
	reserved := am.maybeSetNil(key)
	if reserved {
		// This goroutine now has exclusive ownership of the
		// map entry, and the responsibility for loading the annotator.
		go func(key string) {
			newAnn, err := am.loader(key)
			if err != nil {
				metrics.ErrorTotal.WithLabelValues(err.Error()).Inc()
				log.Println(err)
				return
			}
			// Set the new annotator value.  Entry should be nil.
			err = am.setAnnotatorIfNil(key, newAnn)
			if err != nil {
				metrics.ErrorTotal.WithLabelValues(err.Error()).Inc()
				log.Println(err)
			}
		}(key)
	}
}

// GetAnnotator gets the named annotator, if already in the map.
// If not already loaded, this will trigger loading, and return ErrPendingAnnotatorLoad
func (am *AnnotatorMap) GetAnnotator(key string) (api.Annotator, error) {
	am.mutex.RLock()
	ann, ok := am.annotators[key]
	am.mutex.RUnlock()

	if !ok {
		am.checkAndLoadAnnotator(key)
		metrics.RejectionCount.WithLabelValues("New Dataset").Inc()
		return nil, ErrPendingAnnotatorLoad
	}

	if ann == nil {
		// Another goroutine is already loading this entry.  Return error.
		metrics.RejectionCount.WithLabelValues("Dataset Pending").Inc()
		return nil, ErrPendingAnnotatorLoad
	}
	return ann, nil
}

// GetAnnotator returns the correct annotator to use for a given timestamp.
// TODO: Update to properly handle legacy datasets.
func GetAnnotator(date time.Time) (api.Annotator, error) {
	// key := strconv.FormatInt(date.Unix(), encodingBase)
	if date.After(geoloader.Latest()) {
		currentDataMutex.RLock()
		ann := CurrentAnnotator
		currentDataMutex.RUnlock()
		return ann, nil
	}

	filename, err := geoloader.SelectArchivedDataset(date)

	if err != nil {
		metrics.ErrorTotal.WithLabelValues(err.Error()).Inc()
		return nil, err
	}

	return archivedAnnotator.GetAnnotator(filename)
}

// InitDataset will update the filename list of archived dataset in memory
// and load the latest Geolite2 dataset in memory.
func InitDataset() {
	geoloader.UpdateArchivedFilenames()

	ann := geoloader.GetLatestData()
	currentDataMutex.Lock()
	CurrentAnnotator = ann
	currentDataMutex.Unlock()
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	log "github.com/couchbaselabs/clog"
)

// A planner assigns partitions to cbft's and to PIndexes on each cbft.
// NOTE: You *must* update PLANNER_VERSION if these planning algorithm
// or schema changes, following semver rules.

func (mgr *Manager) PlannerLoop() {
	for reason := range mgr.plannerCh {
		log.Printf("planner awakes, reason: %s", reason)

		if mgr.cfg == nil { // Can occur during testing.
			log.Printf("planner skipped due to nil cfg")
			continue
		}
		ok, err := CheckVersion(mgr.cfg, mgr.version)
		if err != nil {
			log.Printf("planner skipped due to CheckVersion err: %v", err)
			continue
		}
		if !ok {
			log.Printf("planner skipped because version is too low: %v",
				mgr.version)
			continue
		}

		// TODO: What about downgrades?

		indexDefs, _, err := CfgGetIndexDefs(mgr.cfg)
		if err != nil {
			log.Printf("planner skipped due to CfgGetIndexDefs err: %v", err)
			continue
		}
		if indexDefs == nil {
			log.Printf("planner ended since no IndexDefs")
			continue
		}
		if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
			log.Printf("planner ended since indexDefs.ImplVersion: %s"+
				" > mgr.version: %s", indexDefs.ImplVersion, mgr.version)
			continue
		}

		nodeDefs, _, err := CfgGetNodeDefs(mgr.cfg, NODE_DEFS_WANTED)
		if err != nil {
			log.Printf("planner skipped due to CfgGetNodeDefs err: %v", err)
			continue
		}
		if nodeDefs == nil {
			log.Printf("planner ended since no NodeDefs")
			continue
		}
		if VersionGTE(mgr.version, nodeDefs.ImplVersion) == false {
			log.Printf("planner ended since nodeDefs.ImplVersion: %s"+
				" > mgr.version: %s", nodeDefs.ImplVersion, mgr.version)
			continue
		}

		planPIndexesPrev, cas, err := CfgGetPlanPIndexes(mgr.cfg)
		if err != nil {
			log.Printf("planner skipped due to CfgGetPlanPIndexes err: %v", err)
			continue
		}
		if planPIndexesPrev == nil {
			planPIndexesPrev = NewPlanPIndexes(mgr.version)
		}
		if VersionGTE(mgr.version, planPIndexesPrev.ImplVersion) == false {
			log.Printf("planner ended since planPIndexesPrev.ImplVersion: %s"+
				" > mgr.version: %s", planPIndexesPrev.ImplVersion, mgr.version)
			continue
		}

		planPIndexes, err := CalcPlan(indexDefs, nodeDefs, planPIndexesPrev, mgr.version)
		if err != nil {
			log.Printf("error: CalcPlan, err: %v", err)
		}
		if planPIndexes != nil {
			if SamePlanPIndexes(planPIndexes, planPIndexesPrev) {
				log.Printf("planner found no changes")
				continue
			}

			_, err = CfgSetPlanPIndexes(mgr.cfg, planPIndexes, cas)
			if err != nil {
				log.Printf("planner could not save new plan,"+
					" perhaps a concurrent planner won, cas: %d, err: %v",
					cas, err)
				continue
			}

			mgr.janitorCh <- "the plans have changed"

			// TODO: need some distributed notify/event facility,
			// perhaps in the Cfg, to kick any remote janitors.
		}
	}
}

// Split logical indexes into Pindexes and assign PIndexes to nodes.
func CalcPlan(indexDefs *IndexDefs, nodeDefs *NodeDefs, planPIndexesPrev *PlanPIndexes,
	version string) (
	*PlanPIndexes, error) {
	// First planner attempt here is naive & simple, where every
	// single Index is "split" into just a single PIndex (so all the
	// datasource partitions (vbuckets) will feed into that single
	// PIndex).  And, every single node has a replica of that PIndex.
	// This takes care of the "single node" requirement of a developer
	// preview.
	//
	// TODO: Implement more advanced planners another day,
	// - multiple PIndexes per Index.
	// - different fan-out/fan-in topologies from datasource to
	//   Feed to PIndex.
	//
	if indexDefs == nil || nodeDefs == nil {
		return nil, nil
	}

	planPIndexes := NewPlanPIndexes(version)

	for _, indexDef := range indexDefs.IndexDefs {
		_, exists := planPIndexes.PlanPIndexes[indexDef.UUID]
		if !exists {
			planPIndex := &PlanPIndex{
				// This name only works for simple 1-to-1.
				Name:             indexDef.Name + "/" + indexDef.UUID,
				UUID:             NewUUID(),
				IndexName:        indexDef.Name,
				IndexUUID:        indexDef.UUID,
				IndexMapping:     indexDef.Mapping,
				SourceType:       indexDef.SourceType,
				SourceName:       indexDef.SourceName,
				SourceUUID:       indexDef.SourceUUID,
				SourcePartitions: "", // Simple version is get all partitions.
				NodeUUIDs:        make(map[string]string),
			}
			for _, nodeDef := range nodeDefs.NodeDefs {
				planPIndex.NodeUUIDs[nodeDef.UUID] = "active" // TODO: better val needed.
			}

			planPIndexes.PlanPIndexes[planPIndex.Name] = planPIndex
		}
		// TODO: what if UUID's don't match?
	}

	return planPIndexes, nil
}
<commit_msg>more simple planner<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	log "github.com/couchbaselabs/clog"
)

// A planner assigns partitions to cbft's and to PIndexes on each cbft.
// NOTE: You *must* update PLANNER_VERSION if these planning algorithm
// or schema changes, following semver rules.

func (mgr *Manager) PlannerLoop() {
	for reason := range mgr.plannerCh {
		log.Printf("planner awakes, reason: %s", reason)

		if mgr.cfg == nil { // Can occur during testing.
			log.Printf("planner skipped due to nil cfg")
			continue
		}
		ok, err := CheckVersion(mgr.cfg, mgr.version)
		if err != nil {
			log.Printf("planner skipped due to CheckVersion err: %v", err)
			continue
		}
		if !ok {
			log.Printf("planner skipped because version is too low: %v",
				mgr.version)
			continue
		}

		// TODO: What about downgrades?

		indexDefs, _, err := CfgGetIndexDefs(mgr.cfg)
		if err != nil {
			log.Printf("planner skipped due to CfgGetIndexDefs err: %v", err)
			continue
		}
		if indexDefs == nil {
			log.Printf("planner ended since no IndexDefs")
			continue
		}
		if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
			log.Printf("planner ended since indexDefs.ImplVersion: %s"+
				" > mgr.version: %s", indexDefs.ImplVersion, mgr.version)
			continue
		}

		nodeDefs, _, err := CfgGetNodeDefs(mgr.cfg, NODE_DEFS_WANTED)
		if err != nil {
			log.Printf("planner skipped due to CfgGetNodeDefs err: %v", err)
			continue
		}
		if nodeDefs == nil {
			log.Printf("planner ended since no NodeDefs")
			continue
		}
		if VersionGTE(mgr.version, nodeDefs.ImplVersion) == false {
			log.Printf("planner ended since nodeDefs.ImplVersion: %s"+
				" > mgr.version: %s", nodeDefs.ImplVersion, mgr.version)
			continue
		}

		planPIndexesPrev, cas, err := CfgGetPlanPIndexes(mgr.cfg)
		if err != nil {
			log.Printf("planner skipped due to CfgGetPlanPIndexes err: %v", err)
			continue
		}
		if planPIndexesPrev == nil {
			planPIndexesPrev = NewPlanPIndexes(mgr.version)
		}
		if VersionGTE(mgr.version, planPIndexesPrev.ImplVersion) == false {
			log.Printf("planner ended since planPIndexesPrev.ImplVersion: %s"+
				" > mgr.version: %s", planPIndexesPrev.ImplVersion, mgr.version)
			continue
		}

		planPIndexes, err := CalcPlan(indexDefs, nodeDefs, planPIndexesPrev, mgr.version)
		if err != nil {
			log.Printf("error: CalcPlan, err: %v", err)
		}
		if planPIndexes != nil {
			if SamePlanPIndexes(planPIndexes, planPIndexesPrev) {
				log.Printf("planner found no changes")
				continue
			}

			_, err = CfgSetPlanPIndexes(mgr.cfg, planPIndexes, cas)
			if err != nil {
				log.Printf("planner could not save new plan,"+
					" perhaps a concurrent planner won, cas: %d, err: %v",
					cas, err)
				continue
			}

			mgr.janitorCh <- "the plans have changed"

			// TODO: need some distributed notify/event facility,
			// perhaps in the Cfg, to kick any remote janitors.
		}
	}
}

// Split logical indexes into Pindexes and assign PIndexes to nodes.
func CalcPlan(indexDefs *IndexDefs, nodeDefs *NodeDefs, planPIndexesPrev *PlanPIndexes,
	version string) (
	*PlanPIndexes, error) {
	// First planner attempt here is naive & simple, where every
	// single Index is "split" into just a single PIndex (so all the
	// datasource partitions (vbuckets) will feed into that single
	// PIndex).  And, every single node has a replica of that PIndex.
	// This takes care of the "single node" requirement of a developer
	// preview.
	//
	// TODO: Implement more advanced planners another day,
	// - multiple PIndexes per Index.
	// - different fan-out/fan-in topologies from datasource to
	//   Feed to PIndex.
	//
	if indexDefs == nil || nodeDefs == nil {
		return nil, nil
	}

	planPIndexes := NewPlanPIndexes(version)

	for _, indexDef := range indexDefs.IndexDefs {
		planPIndex := &PlanPIndex{
			// This simple PlanPIndex.Name here only works for simple
			// 1-to-1 case.  NOTE: PlanPIndex.Name must be unique
			// across the cluster and functionally based off of the
			// indexDef.
			//
			// TODO: More advanced planners will probably have to
			// incorporate SourcePartitions info into the
			// PlanPIndex.Name.
			Name:             indexDef.Name + "/" + indexDef.UUID,
			UUID:             NewUUID(),
			IndexName:        indexDef.Name,
			IndexUUID:        indexDef.UUID,
			IndexMapping:     indexDef.Mapping,
			SourceType:       indexDef.SourceType,
			SourceName:       indexDef.SourceName,
			SourceUUID:       indexDef.SourceUUID,
			SourcePartitions: "", // Simple version is get all partitions.
			NodeUUIDs:        make(map[string]string),
		}
		for _, nodeDef := range nodeDefs.NodeDefs {
			// TODO: better val needed.
			planPIndex.NodeUUIDs[nodeDef.UUID] = "active"
		}
		planPIndexes.PlanPIndexes[planPIndex.Name] = planPIndex
	}

	return planPIndexes, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A planner assigns partitions to cbft's and to PIndexes on each cbft.
// NOTE: You *must* update PLANNER_VERSION if the planning algorithm
// or schema changes, following semver rules.

func (mgr *Manager) PlannerNOOP(msg string) {
	SyncWorkReq(mgr.plannerCh, WORK_NOOP, msg, nil)
}

func (mgr *Manager) PlannerKick(msg string) {
	SyncWorkReq(mgr.plannerCh, WORK_KICK, msg, nil)
}

func (mgr *Manager) PlannerLoop() {
	for m := range mgr.plannerCh {
		if m.op == WORK_KICK {
			changed, err := mgr.PlannerOnce(m.msg)
			if err != nil {
				log.Printf("error: PlannerOnce, err: %v", err)
				// Keep looping as perhaps it's a transient issue.
			} else if changed {
				// TODO: need some distributed notify/event facility,
				// perhaps in the Cfg, to kick any remote janitors.
				//
				mgr.JanitorKick("the plans have changed")
			}
		} else if m.op == WORK_NOOP {
			// NOOP.
		} else {
			log.Printf("error: unknown planner op: %s, m: %#v", m.op, m)
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}

	close(mgr.janitorCh)
}

func (mgr *Manager) PlannerOnce(reason string) (bool, error) {
	log.Printf("planner awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return false, fmt.Errorf("planner skipped due to nil cfg")
	}

	ok, err := CheckVersion(mgr.cfg, mgr.version)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CheckVersion err: %v", err)
	}
	if !ok {
		return false, fmt.Errorf("planner skipped with version too low: %v",
			mgr.version)
	}

	// TODO: What about downgrades?

	indexDefs, _, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CfgGetIndexDefs err: %v", err)
	}
	if indexDefs == nil {
		return false, fmt.Errorf("planner ended since no IndexDefs")
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return false, fmt.Errorf("planner ended since indexDefs.ImplVersion: %s"+
			" > mgr.version: %s", indexDefs.ImplVersion, mgr.version)
	}

	nodeDefs, _, err := CfgGetNodeDefs(mgr.cfg, NODE_DEFS_WANTED)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CfgGetNodeDefs err: %v", err)
	}
	if nodeDefs == nil {
		return false, fmt.Errorf("planner ended since no NodeDefs")
	}
	if VersionGTE(mgr.version, nodeDefs.ImplVersion) == false {
		return false, fmt.Errorf("planner ended since nodeDefs.ImplVersion: %s"+
			" > mgr.version: %s", nodeDefs.ImplVersion, mgr.version)
	}

	planPIndexesPrev, cas, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexesPrev == nil {
		planPIndexesPrev = NewPlanPIndexes(mgr.version)
	}
	if VersionGTE(mgr.version, planPIndexesPrev.ImplVersion) == false {
		return false, fmt.Errorf("planner ended on planPIndexesPrev.ImplVersion: %s"+
			" > mgr.version: %s", planPIndexesPrev.ImplVersion, mgr.version)
	}

	planPIndexes, err := CalcPlan(indexDefs, nodeDefs, planPIndexesPrev, mgr.version)
	if err != nil {
		return false, fmt.Errorf("planner ended on CalcPlan, err: %v", err)
	}
	if planPIndexes == nil {
		return false, fmt.Errorf("planner found no plans from CalcPlan")
	}
	if SamePlanPIndexes(planPIndexes, planPIndexesPrev) {
		return false, nil
	}
	_, err = CfgSetPlanPIndexes(mgr.cfg, planPIndexes, cas)
	if err != nil {
		return false, fmt.Errorf("planner could not save new plan,"+
			" perhaps a concurrent planner won, cas: %d, err: %v",
			cas, err)
	}

	return true, nil
}

// Split logical indexes into PIndexes and assign PIndexes to nodes.
func CalcPlan(indexDefs *IndexDefs, nodeDefs *NodeDefs, planPIndexesPrev *PlanPIndexes,
	version string) (
	*PlanPIndexes, error) {
	// First planner attempt here is naive & simple, where every
	// single Index is "split" into just a single PIndex (so all the
	// datasource partitions (vbuckets) will feed into that single
	// PIndex).  And, every single node has a replica of that PIndex.
	// This takes care of the "single node" requirement of a developer
	// preview.
	//
	// TODO: Implement more advanced planners another day,
	// - multiple PIndexes per Index.
	// - different fan-out/fan-in topologies from datasource to
	//   Feed to PIndex.
	//
	if indexDefs == nil || nodeDefs == nil {
		return nil, nil
	}

	planPIndexes := NewPlanPIndexes(version)

	for _, indexDef := range indexDefs.IndexDefs {
		// This simple planner puts every PIndex on every node, so
		// there is not "real" partitioning.  This is good enough for
		// developer preview level requirement.
		sourcePartitions := ""

		// This simple PlanPIndex.Name here only works for simple
		// 1-to-1 case, which is developer preview level requirement.
		// NOTE: PlanPIndex.Name must be unique across the cluster and
		// functionally based off of the indexDef.
		name := indexDef.Name + "_" + indexDef.UUID + "_" + sourcePartitions

		planPIndex := &PlanPIndex{
			// TODO: More advanced planners will probably have to
			// incorporate SourcePartitions info into the
			// PlanPIndex.Name.
			Name:             name,
			UUID:             NewUUID(),
			IndexType:        indexDef.Type,
			IndexName:        indexDef.Name,
			IndexUUID:        indexDef.UUID,
			IndexMapping:     indexDef.Mapping,
			SourceType:       indexDef.SourceType,
			SourceName:       indexDef.SourceName,
			SourceUUID:       indexDef.SourceUUID,
			SourcePartitions: sourcePartitions,
			NodeUUIDs:        make(map[string]string),
		}
		for _, nodeDef := range nodeDefs.NodeDefs {
			// TODO: better val needed.
			planPIndex.NodeUUIDs[nodeDef.UUID] = "active"
		}

		planPIndexes.PlanPIndexes[planPIndex.Name] = planPIndex
	}

	return planPIndexes, nil
}
<commit_msg>can't use SourcePartitions in PlanPIndex.Name<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A planner assigns partitions to cbft's and to PIndexes on each cbft.
// NOTE: You *must* update PLANNER_VERSION if the planning algorithm
// or schema changes, following semver rules.

func (mgr *Manager) PlannerNOOP(msg string) {
	SyncWorkReq(mgr.plannerCh, WORK_NOOP, msg, nil)
}

func (mgr *Manager) PlannerKick(msg string) {
	SyncWorkReq(mgr.plannerCh, WORK_KICK, msg, nil)
}

func (mgr *Manager) PlannerLoop() {
	for m := range mgr.plannerCh {
		if m.op == WORK_KICK {
			changed, err := mgr.PlannerOnce(m.msg)
			if err != nil {
				log.Printf("error: PlannerOnce, err: %v", err)
				// Keep looping as perhaps it's a transient issue.
			} else if changed {
				// TODO: need some distributed notify/event facility,
				// perhaps in the Cfg, to kick any remote janitors.
				//
				mgr.JanitorKick("the plans have changed")
			}
		} else if m.op == WORK_NOOP {
			// NOOP.
		} else {
			log.Printf("error: unknown planner op: %s, m: %#v", m.op, m)
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}

	close(mgr.janitorCh)
}

func (mgr *Manager) PlannerOnce(reason string) (bool, error) {
	log.Printf("planner awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return false, fmt.Errorf("planner skipped due to nil cfg")
	}

	ok, err := CheckVersion(mgr.cfg, mgr.version)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CheckVersion err: %v", err)
	}
	if !ok {
		return false, fmt.Errorf("planner skipped with version too low: %v",
			mgr.version)
	}

	// TODO: What about downgrades?

	indexDefs, _, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CfgGetIndexDefs err: %v", err)
	}
	if indexDefs == nil {
		return false, fmt.Errorf("planner ended since no IndexDefs")
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return false, fmt.Errorf("planner ended since indexDefs.ImplVersion: %s"+
			" > mgr.version: %s", indexDefs.ImplVersion, mgr.version)
	}

	nodeDefs, _, err := CfgGetNodeDefs(mgr.cfg, NODE_DEFS_WANTED)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CfgGetNodeDefs err: %v", err)
	}
	if nodeDefs == nil {
		return false, fmt.Errorf("planner ended since no NodeDefs")
	}
	if VersionGTE(mgr.version, nodeDefs.ImplVersion) == false {
		return false, fmt.Errorf("planner ended since nodeDefs.ImplVersion: %s"+
			" > mgr.version: %s", nodeDefs.ImplVersion, mgr.version)
	}

	planPIndexesPrev, cas, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return false, fmt.Errorf("planner skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexesPrev == nil {
		planPIndexesPrev = NewPlanPIndexes(mgr.version)
	}
	if VersionGTE(mgr.version, planPIndexesPrev.ImplVersion) == false {
		return false, fmt.Errorf("planner ended on planPIndexesPrev.ImplVersion: %s"+
			" > mgr.version: %s", planPIndexesPrev.ImplVersion, mgr.version)
	}

	planPIndexes, err := CalcPlan(indexDefs, nodeDefs, planPIndexesPrev, mgr.version)
	if err != nil {
		return false, fmt.Errorf("planner ended on CalcPlan, err: %v", err)
	}
	if planPIndexes == nil {
		return false, fmt.Errorf("planner found no plans from CalcPlan")
	}
	if SamePlanPIndexes(planPIndexes, planPIndexesPrev) {
		return false, nil
	}
	_, err = CfgSetPlanPIndexes(mgr.cfg, planPIndexes, cas)
	if err != nil {
		return false, fmt.Errorf("planner could not save new plan,"+
			" perhaps a concurrent planner won, cas: %d, err: %v",
			cas, err)
	}

	return true, nil
}

// Split logical indexes into PIndexes and assign PIndexes to nodes.
func CalcPlan(indexDefs *IndexDefs, nodeDefs *NodeDefs, planPIndexesPrev *PlanPIndexes,
	version string) (
	*PlanPIndexes, error) {
	// First planner attempt here is naive & simple, where every
	// single Index is "split" into just a single PIndex (so all the
	// datasource partitions (vbuckets) will feed into that single
	// PIndex).  And, every single node has a replica of that PIndex.
	// This takes care of the "single node" requirement of a developer
	// preview.
	//
	// TODO: Implement more advanced planners another day,
	// - multiple PIndexes per Index.
	// - different fan-out/fan-in topologies from datasource to
	//   Feed to PIndex.
	//
	if indexDefs == nil || nodeDefs == nil {
		return nil, nil
	}

	planPIndexes := NewPlanPIndexes(version)

	for _, indexDef := range indexDefs.IndexDefs {
		// This simple planner puts every PIndex on every node, so
		// there is not "real" partitioning.  This is good enough for
		// developer preview level requirement.
		sourcePartitions := ""

		// This simple PlanPIndex.Name here only works for simple
		// 1-to-1 case, which is developer preview level requirement.
		// NOTE: PlanPIndex.Name must be unique across the cluster and
		// functionally based off of the indexDef.
		// NOTE: Can't use SourcePartitions as part of PlanPIndex.Name
		// because in vbucket/hash partitioning, the string would be
		// too large where PIndexes would use encode it into paths.
		name := indexDef.Name + "_" + indexDef.UUID

		planPIndex := &PlanPIndex{
			// TODO: More advanced planners will probably have to
			// incorporate SourcePartitions info into the
			// PlanPIndex.Name.
			Name:             name,
			UUID:             NewUUID(),
			IndexType:        indexDef.Type,
			IndexName:        indexDef.Name,
			IndexUUID:        indexDef.UUID,
			IndexMapping:     indexDef.Mapping,
			SourceType:       indexDef.SourceType,
			SourceName:       indexDef.SourceName,
			SourceUUID:       indexDef.SourceUUID,
			SourcePartitions: sourcePartitions,
			NodeUUIDs:        make(map[string]string),
		}
		for _, nodeDef := range nodeDefs.NodeDefs {
			// TODO: rules around val needed; perhaps "paused"
			// could be used to encode index ingest pausing.
			planPIndex.NodeUUIDs[nodeDef.UUID] = "active"
		}

		planPIndexes.PlanPIndexes[planPIndex.Name] = planPIndex
	}

	return planPIndexes, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Alexander Palaistras. All rights reserved.
// Use of this source code is governed by the MIT license that can be found in
// the LICENSE file.

// Package engine provides methods allowing for the initialization and teardown
// of PHP engine bindings, off which execution contexts can be launched.
package engine

// #cgo CFLAGS: -I/usr/include/php -I/usr/include/php/main -I/usr/include/php/TSRM
// #cgo CFLAGS: -I/usr/include/php/Zend -I../value -I../context
// #cgo LDFLAGS: -L${SRCDIR}/value -L${SRCDIR}/context -lphp5
//
// #include <stdlib.h>
// #include <main/php.h>
//
// #include "context.h"
// #include "engine.h"
// #include "class.h"
import "C"

import (
	"fmt"
	"reflect"
	"unsafe"

	"github.com/deuill/go-php/context"
)

// Receiver represents a method receiver.
type Receiver struct {
	values  map[string]reflect.Value
	methods map[string]reflect.Value
}

// Engine represents the core PHP engine bindings.
type Engine struct {
	engine   *C.struct__php_engine
	contexts []*context.Context
}

// New initializes a PHP engine instance on which contexts can be executed. It
// corresponds to PHP's MINIT (module init) phase.
func New() (*Engine, error) {
	ptr, err := C.engine_init()
	if err != nil {
		return nil, fmt.Errorf("PHP engine failed to initialize")
	}

	return &Engine{engine: ptr, contexts: make([]*context.Context, 0)}, nil
}

// NewContext creates a new execution context on which scripts can be executed
// and variables can be binded. It corresponds to PHP's RINIT (request init)
// phase.
func (e *Engine) NewContext() (*context.Context, error) {
	c, err := context.New()
	if err != nil {
		return nil, err
	}

	e.contexts = append(e.contexts, c)

	return c, nil
}

func (e *Engine) Define(rcvr interface{}) error {
	v := reflect.ValueOf(rcvr)
	name := reflect.Indirect(v).Type().Name()

	if name == "" {
		return fmt.Errorf("Cannot define anonymous method receiver")
	} else if v.Type().NumMethod() == 0 {
		return fmt.Errorf("Cannot define receiver with no embedded methods")
	}

	n := C.CString(name)
	defer C.free(unsafe.Pointer(n))

	r := unsafe.Pointer(newReceiver(rcvr))

	C.engine_class_define(r, n)
	return nil
}

// Destroy shuts down and frees any resources related to the PHP engine bindings.
func (e *Engine) Destroy() {
	for _, c := range e.contexts {
		c.Destroy()
	}

	e.contexts = nil

	if e.engine != nil {
		C.engine_shutdown(e.engine)
		e.engine = nil
	}
}

func newReceiver(rcvr interface{}) *Receiver {
	v := reflect.ValueOf(rcvr)
	vi := reflect.Indirect(v)

	methods := make(map[string]reflect.Value)
	values := make(map[string]reflect.Value)

	for i := 0; i < v.NumMethod(); i++ {
		methods[v.Type().Method(i).Name] = v.Method(i)
	}

	if vi.Kind() == reflect.Struct {
		for i := 0; i < vi.NumField(); i++ {
			values[vi.Type().Field(i).Name] = vi.Field(i)
		}
	}

	return &Receiver{values: values, methods: methods}
}
<commit_msg>engine: Add documentation for Define method<commit_after>// Copyright 2015 Alexander Palaistras. All rights reserved.
// Use of this source code is governed by the MIT license that can be found in
// the LICENSE file.

// Package engine provides methods allowing for the initialization and teardown
// of PHP engine bindings, off which execution contexts can be launched.
package engine

// #cgo CFLAGS: -I/usr/include/php -I/usr/include/php/main -I/usr/include/php/TSRM
// #cgo CFLAGS: -I/usr/include/php/Zend -I../value -I../context
// #cgo LDFLAGS: -L${SRCDIR}/value -L${SRCDIR}/context -lphp5
//
// #include <stdlib.h>
// #include <main/php.h>
//
// #include "context.h"
// #include "engine.h"
// #include "class.h"
import "C"

import (
	"fmt"
	"reflect"
	"unsafe"

	"github.com/deuill/go-php/context"
)

// Receiver represents a method receiver.
type Receiver struct {
	values  map[string]reflect.Value
	methods map[string]reflect.Value
}

// Engine represents the core PHP engine bindings.
type Engine struct {
	engine   *C.struct__php_engine
	contexts []*context.Context
}

// New initializes a PHP engine instance on which contexts can be executed. It
// corresponds to PHP's MINIT (module init) phase.
func New() (*Engine, error) {
	ptr, err := C.engine_init()
	if err != nil {
		return nil, fmt.Errorf("PHP engine failed to initialize")
	}

	return &Engine{engine: ptr, contexts: make([]*context.Context, 0)}, nil
}

// NewContext creates a new execution context on which scripts can be executed
// and variables can be binded. It corresponds to PHP's RINIT (request init)
// phase.
func (e *Engine) NewContext() (*context.Context, error) {
	c, err := context.New()
	if err != nil {
		return nil, err
	}

	e.contexts = append(e.contexts, c)

	return c, nil
}

// Define registers a Go method receiver as a PHP class, allowing for method
// calls, as well as internal property access (for struct values), from PHP
// contexts.
func (e *Engine) Define(rcvr interface{}) error {
	v := reflect.ValueOf(rcvr)
	name := reflect.Indirect(v).Type().Name()

	if name == "" {
		return fmt.Errorf("Cannot define anonymous method receiver")
	} else if v.Type().NumMethod() == 0 {
		return fmt.Errorf("Cannot define receiver with no embedded methods")
	}

	n := C.CString(name)
	defer C.free(unsafe.Pointer(n))

	r := unsafe.Pointer(newReceiver(rcvr))

	C.engine_class_define(r, n)
	return nil
}

// Destroy shuts down and frees any resources related to the PHP engine bindings.
func (e *Engine) Destroy() {
	for _, c := range e.contexts {
		c.Destroy()
	}

	e.contexts = nil

	if e.engine != nil {
		C.engine_shutdown(e.engine)
		e.engine = nil
	}
}

func newReceiver(rcvr interface{}) *Receiver {
	v := reflect.ValueOf(rcvr)
	vi := reflect.Indirect(v)

	methods := make(map[string]reflect.Value)
	values := make(map[string]reflect.Value)

	for i := 0; i < v.NumMethod(); i++ {
		methods[v.Type().Method(i).Name] = v.Method(i)
	}

	if vi.Kind() == reflect.Struct {
		for i := 0; i < vi.NumField(); i++ {
			values[vi.Type().Field(i).Name] = vi.Field(i)
		}
	}

	return &Receiver{values: values, methods: methods}
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"io"
	"os"
	"sort"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})
var _ = fs.NodeGetxattrer(&File{})
var _ = fs.NodeSetxattrer(&File{})
var _ = fs.NodeRemovexattrer(&File{})
var _ = fs.NodeListxattrer(&File{})
var _ = fs.NodeForgetter(&File{})

type File struct {
	Name           string
	dir            *Dir
	wfs            *WFS
	entry          *filer_pb.Entry
	entryViewCache []filer2.VisibleInterval
	isOpen         int
	reader         io.ReaderAt
	dirtyMetadata  bool
}

func (file *File) fullpath() util.FullPath {
	return util.NewFullPath(file.dir.FullPath(), file.Name)
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {

	glog.V(5).Infof("file Attr %s, open:%v, existing attr: %+v", file.fullpath(), file.isOpen, attr)

	if file.isOpen <= 0 {
		if err := file.maybeLoadEntry(ctx); err != nil {
			return err
		}
	}

	attr.Inode = file.fullpath().AsInode()
	attr.Valid = time.Second
	attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
	attr.Size = filer2.FileSize(file.entry)
	if file.isOpen > 0 {
		attr.Size = file.entry.Attributes.FileSize
		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
	}
	attr.Crtime = time.Unix(file.entry.Attributes.Crtime, 0)
	attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0)
	attr.Gid = file.entry.Attributes.Gid
	attr.Uid = file.entry.Attributes.Uid
	attr.Blocks = attr.Size/blockSize + 1
	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)

	return nil

}

func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {

	glog.V(4).Infof("file Getxattr %s", file.fullpath())

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	return getxattr(file.entry, req, resp)
}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	glog.V(5).Infof("%v file setattr %+v", file.fullpath(), req)

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}
	if file.isOpen > 0 {
		file.wfs.handlesLock.Lock()
		fileHandle := file.wfs.handles[file.fullpath().AsInode()]
		file.wfs.handlesLock.Unlock()

		if fileHandle != nil {
			fileHandle.Lock()
			defer fileHandle.Unlock()
		}
	}

	if req.Valid.Size() {

		glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(file.entry.Chunks))
		if req.Size < filer2.FileSize(file.entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range file.entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(req.Size) {
					// this chunk is truncated
					int64Size = int64(req.Size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			file.wfs.deleteFileChunks(truncatedChunks)
			file.entry.Chunks = chunks
			file.entryViewCache = nil
			file.reader = nil
		}
		file.entry.Attributes.FileSize = req.Size
		file.dirtyMetadata = true
	}

	if req.Valid.Mode() {
		file.entry.Attributes.FileMode = uint32(req.Mode)
		file.dirtyMetadata = true
	}

	if req.Valid.Uid() {
		file.entry.Attributes.Uid = req.Uid
		file.dirtyMetadata = true
	}

	if req.Valid.Gid() {
		file.entry.Attributes.Gid = req.Gid
		file.dirtyMetadata = true
	}

	if req.Valid.Crtime() {
		file.entry.Attributes.Crtime = req.Crtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Mtime() {
		file.entry.Attributes.Mtime = req.Mtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Handle() {
		// fmt.Printf("file handle => %d\n", req.Handle)
	}

	if file.isOpen > 0 {
		return nil
	}

	if !file.dirtyMetadata {
		return nil
	}

	return file.saveEntry()

}

func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	if err := setxattr(file.entry, req); err != nil {
		return err
	}

	return file.saveEntry()

}

func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	if err := removexattr(file.entry, req); err != nil {
		return err
	}

	return file.saveEntry()

}

func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {

	glog.V(4).Infof("file Listxattr %s", file.fullpath())

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	if err := listxattr(file.entry, req, resp); err != nil {
		return err
	}

	return nil

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	// fsync works at OS level
	// write the file chunks to the filerGrpcAddress
	glog.V(4).Infof("%s/%s fsync file %+v", file.dir.FullPath(), file.Name, req)

	return nil
}

func (file *File) Forget() {
	t := util.NewFullPath(file.dir.FullPath(), file.Name)
	glog.V(5).Infof("Forget file %s", t)
	file.wfs.fsNodeCache.DeleteFsNode(t)
}

func (file *File) maybeLoadEntry(ctx context.Context) error {
	if file.entry == nil && file.isOpen <= 0 {
		entry, err := file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name)
		if err != nil {
			glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return err
		}
		if entry != nil {
			file.setEntry(entry)
		}
	}
	return nil
}

func (file *File) addChunks(chunks []*filer_pb.FileChunk) {

	sort.Slice(chunks, func(i, j int) bool {
		if chunks[i].Mtime == chunks[j].Mtime {
			return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey
		}
		return chunks[i].Mtime < chunks[j].Mtime
	})

	for _, chunk := range chunks {
		file.entryViewCache = filer2.MergeIntoVisibles(file.entryViewCache, chunk)
	}

	file.reader = nil

	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(file.entry.Chunks), len(chunks))

	file.entry.Chunks = append(file.entry.Chunks, chunks...)
}

func (file *File) setEntry(entry *filer_pb.Entry) {
	file.entry = entry
	file.entryViewCache, _ = filer2.NonOverlappingVisibleIntervals(filer2.LookupFn(file.wfs), file.entry.Chunks)
	file.reader = nil
}

func (file *File) saveEntry() error {
	return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.UpdateEntryRequest{
			Directory: file.dir.FullPath(),
			Entry:     file.entry,
		}

		glog.V(4).Infof("save file entry: %v", request)
		_, err := client.UpdateEntry(context.Background(), request)
		if err != nil {
			glog.Errorf("UpdateEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		file.wfs.metaCache.UpdateEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry))

		return nil
	})
}
<commit_msg>adjust deletion ordering<commit_after>package filesys

import (
	"context"
	"io"
	"os"
	"sort"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})
var _ = fs.NodeGetxattrer(&File{})
var _ = fs.NodeSetxattrer(&File{})
var _ = fs.NodeRemovexattrer(&File{})
var _ = fs.NodeListxattrer(&File{})
var _ = fs.NodeForgetter(&File{})

type File struct {
	Name           string
	dir            *Dir
	wfs            *WFS
	entry          *filer_pb.Entry
	entryViewCache []filer2.VisibleInterval
	isOpen         int
	reader         io.ReaderAt
	dirtyMetadata  bool
}

func (file *File) fullpath() util.FullPath {
	return util.NewFullPath(file.dir.FullPath(), file.Name)
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {

	glog.V(5).Infof("file Attr %s, open:%v, existing attr: %+v", file.fullpath(), file.isOpen, attr)

	if file.isOpen <= 0 {
		if err := file.maybeLoadEntry(ctx); err != nil {
			return err
		}
	}

	attr.Inode = file.fullpath().AsInode()
	attr.Valid = time.Second
	attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
	attr.Size = filer2.FileSize(file.entry)
	if file.isOpen > 0 {
		attr.Size = file.entry.Attributes.FileSize
		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
	}
	attr.Crtime = time.Unix(file.entry.Attributes.Crtime, 0)
	attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0)
	attr.Gid = file.entry.Attributes.Gid
	attr.Uid = file.entry.Attributes.Uid
	attr.Blocks = attr.Size/blockSize + 1
	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)

	return nil

}

func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {

	glog.V(4).Infof("file Getxattr %s", file.fullpath())

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	return getxattr(file.entry, req, resp)
}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	glog.V(5).Infof("%v file setattr %+v", file.fullpath(), req)

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}
	if file.isOpen > 0 {
		file.wfs.handlesLock.Lock()
		fileHandle := file.wfs.handles[file.fullpath().AsInode()]
		file.wfs.handlesLock.Unlock()

		if fileHandle != nil {
			fileHandle.Lock()
			defer fileHandle.Unlock()
		}
	}

	if req.Valid.Size() {

		glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(file.entry.Chunks))
		if req.Size < filer2.FileSize(file.entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range file.entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(req.Size) {
					// this chunk is truncated
					int64Size = int64(req.Size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			file.entry.Chunks = chunks
			file.entryViewCache = nil
			file.reader = nil
			file.wfs.deleteFileChunks(truncatedChunks)
		}
		file.entry.Attributes.FileSize = req.Size
		file.dirtyMetadata = true
	}

	if req.Valid.Mode() {
		file.entry.Attributes.FileMode = uint32(req.Mode)
		file.dirtyMetadata = true
	}

	if req.Valid.Uid() {
		file.entry.Attributes.Uid = req.Uid
		file.dirtyMetadata = true
	}

	if req.Valid.Gid() {
		file.entry.Attributes.Gid = req.Gid
		file.dirtyMetadata = true
	}

	if req.Valid.Crtime() {
		file.entry.Attributes.Crtime = req.Crtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Mtime() {
		file.entry.Attributes.Mtime = req.Mtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Handle() {
		// fmt.Printf("file handle => %d\n", req.Handle)
	}

	if file.isOpen > 0 {
		return nil
	}

	if !file.dirtyMetadata {
		return nil
	}

	return file.saveEntry()

}

func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	if err := setxattr(file.entry, req); err != nil {
		return err
	}

	return file.saveEntry()

}

func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	if err := removexattr(file.entry, req); err != nil {
		return err
	}

	return file.saveEntry()

}

func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {

	glog.V(4).Infof("file Listxattr %s", file.fullpath())

	if err := file.maybeLoadEntry(ctx); err != nil {
		return err
	}

	if err := listxattr(file.entry, req, resp); err != nil {
		return err
	}

	return nil

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	// fsync works at OS level
	// write the file chunks to the filerGrpcAddress
	glog.V(4).Infof("%s/%s fsync file %+v", file.dir.FullPath(), file.Name, req)

	return nil
}

func (file *File) Forget() {
	t := util.NewFullPath(file.dir.FullPath(), file.Name)
	glog.V(5).Infof("Forget file %s", t)
	file.wfs.fsNodeCache.DeleteFsNode(t)
}

func (file *File) maybeLoadEntry(ctx context.Context) error {
	if file.entry == nil && file.isOpen <= 0 {
		entry, err := file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name)
		if err != nil {
			glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return err
		}
		if entry != nil {
			file.setEntry(entry)
		}
	}
	return nil
}

func (file *File) addChunks(chunks []*filer_pb.FileChunk) {

	sort.Slice(chunks, func(i, j int) bool {
		if chunks[i].Mtime == chunks[j].Mtime {
			return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey
		}
		return chunks[i].Mtime < chunks[j].Mtime
	})

	for _, chunk := range chunks {
		file.entryViewCache = filer2.MergeIntoVisibles(file.entryViewCache, chunk)
	}

	file.reader = nil

	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(file.entry.Chunks), len(chunks))

	file.entry.Chunks = append(file.entry.Chunks, chunks...)
}

func (file *File) setEntry(entry *filer_pb.Entry) {
	file.entry = entry
	file.entryViewCache, _ = filer2.NonOverlappingVisibleIntervals(filer2.LookupFn(file.wfs), file.entry.Chunks)
	file.reader = nil
}

func (file *File) saveEntry() error {
	return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.UpdateEntryRequest{
			Directory: file.dir.FullPath(),
			Entry:     file.entry,
		}

		glog.V(4).Infof("save file entry: %v", request)
		_, err := client.UpdateEntry(context.Background(), request)
		if err != nil {
			glog.Errorf("UpdateEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		file.wfs.metaCache.UpdateEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry))

		return nil
	})
}
<|endoftext|>


<commit_before>package errors

import (
	"fmt"
	"os"
)

type Err string

func (e Err) Error() string {
	return string(e)
}

func Assert(val bool, err error) {
	if !val {
		panic(err)
	}
}

func New(v ...interface{}) error {
	return Err(fmt.Sprint(v...))
}

func Newln(v ...interface{}) error {
	return Err(fmt.Sprintln(v...))
}

func Newf(format string, v ...interface{}) error {
	return Err(fmt.Sprintf(format, v...))
}

func Panic(err error) {
	if err != nil {
		panic(err)
	}
}

func Panicln(err error) {
	if err != nil {
		panic(fmt.Sprintln(err))
	}
}

func Panicf(format string, err error) {
	if err != nil {
		panic(fmt.Sprintf(format, err))
	}
}

func Print(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
	}
}

func Println(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func Printf(format string, err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, format, err)
	}
}

func Exit(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		os.Exit(0)
	}
}

func Exitln(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(0)
	}
}

func Exitf(format string, err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, format, err)
		os.Exit(0)
	}
}

func Fatal(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		os.Exit(-1)
	}
}

func Fatalln(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}
}

func Fatalf(format string, err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, format, err)
		os.Exit(-1)
	}
}

func ExitAny(err error) {
	fmt.Fprint(os.Stderr, err)
	os.Exit(0)
}

func ExitAnyf(format string, err error) {
	fmt.Fprintln(os.Stderr, format, err)
	os.Exit(0)
}

func ExitAnyln(err error) {
	fmt.Fprintln(os.Stderr, err)
	os.Exit(0)
}

func FatalAny(err error) {
	fmt.Fprint(os.Stderr, err)
	os.Exit(-1)
}

func FatalAnyf(format string, err error) {
	fmt.Fprint(os.Stderr, format, err)
	os.Exit(-1)
}

func FatalAnyln(err error) {
	fmt.Fprintln(os.Stderr, err)
	os.Exit(-1)
}

func Do(err error, fn func(err error)) {
	if err != nil {
		fn(err)
	}
}

func CondDo(val bool, err error, fn func(error)) {
	if val {
		fn(err)
	}
}
<commit_msg>errors: add Nil function<commit_after>package errors

import (
	"fmt"
	"os"
)

type Err string

func (e Err) Error() string {
	return string(e)
}

func Assert(val bool, err error) {
	if !val {
		panic(err)
	}
}

func New(v ...interface{}) error {
	return Err(fmt.Sprint(v...))
}

func Newln(v ...interface{}) error {
	return Err(fmt.Sprintln(v...))
}

func Newf(format string, v ...interface{}) error {
	return Err(fmt.Sprintf(format, v...))
}

func Panic(err error) {
	if err != nil {
		panic(err)
	}
}

func Panicln(err error) {
	if err != nil {
		panic(fmt.Sprintln(err))
	}
}

func Panicf(format string, err error) {
	if err != nil {
		panic(fmt.Sprintf(format, err))
	}
}

func Print(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
	}
}

func Println(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func Printf(format string, err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, format, err)
	}
}

func Exit(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		os.Exit(0)
	}
}

func Exitln(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(0)
	}
}

func Exitf(format string, err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, format, err)
		os.Exit(0)
	}
}

func Fatal(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		os.Exit(-1)
	}
}

func Fatalln(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}
}

func Fatalf(format string, err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, format, err)
		os.Exit(-1)
	}
}

func ExitAny(err error) {
	fmt.Fprint(os.Stderr, err)
	os.Exit(0)
}

func ExitAnyf(format string, err error) {
	fmt.Fprintln(os.Stderr, format, err)
	os.Exit(0)
}

func ExitAnyln(err error) {
	fmt.Fprintln(os.Stderr, err)
	os.Exit(0)
}

func FatalAny(err error) {
	fmt.Fprint(os.Stderr, err)
	os.Exit(-1)
}

func FatalAnyf(format string, err error) {
	fmt.Fprint(os.Stderr, format, err)
	os.Exit(-1)
}

func FatalAnyln(err error) {
	fmt.Fprintln(os.Stderr, err)
	os.Exit(-1)
}

func Do(err error, fn func(err error)) {
	if err != nil {
		fn(err)
	}
}

func CondDo(val bool, err error, fn func(error)) {
	if val {
		fn(err)
	}
}

func Nil(err error, errs ...error) error {
	for _, e := range errs {
		if err == e {
			return nil
		}
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ethereum/eth-go/ethlog"
	"github.com/ethereum/go-ethereum/utils"
	"runtime"
)

var logger = ethlog.NewLogger("CLI")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	utils.HandleInterrupt()

	// precedence: code-internal flag default < config file < environment variables < command line
	Init() // parsing command line
	utils.InitConfig(ConfigFile, Datadir, Identifier, "ETH")

	utils.InitDataDir(Datadir)

	utils.InitLogging(Datadir, LogFile, LogLevel, DebugFile)

	ethereum := utils.NewEthereum(UseUPnP, OutboundPort, MaxPeer)

	// create, import, export keys
	utils.KeyTasks(GenAddr, ImportKey, ExportKey, NonInteractive)

	if ShowGenesis {
		utils.ShowGenesis(ethereum)
	}

	if StartMining {
		utils.StartMining(ethereum)
	}

	// better reworked as cases
	if StartJsConsole {
		InitJsConsole(ethereum)
	} else if len(InputFile) > 0 {
		ExecJsFile(ethereum, InputFile)
	}

	if StartRpc {
		utils.StartRpc(ethereum, RpcPort)
	}

	utils.StartEthereum(ethereum, UseSeed)

	// this blocks the thread
	ethereum.WaitForShutdown()
	ethlog.Flush()
}
<commit_msg>main loop uses new flags and common methods in util: db, keymanager set up<commit_after>package main

import (
	"github.com/ethereum/eth-go/ethlog"
	"github.com/ethereum/go-ethereum/utils"
	"runtime"
)

var logger = ethlog.NewLogger("CLI")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	utils.HandleInterrupt()

	// precedence: code-internal flag default < config file < environment variables < command line
	Init() // parsing command line
	utils.InitConfig(ConfigFile, Datadir, Identifier, "ETH")

	utils.InitDataDir(Datadir)

	utils.InitLogging(Datadir, LogFile, LogLevel, DebugFile)

	db := utils.NewDatabase()

	keyManager := utils.NewKeyManager(KeyStore, Datadir, db)

	// create, import, export keys
	utils.KeyTasks(keyManager, KeyRing, GenAddr, SecretFile, ExportDir, NonInteractive)

	ethereum := utils.NewEthereum(db, keyManager, UseUPnP, OutboundPort, MaxPeer)

	if ShowGenesis {
		utils.ShowGenesis(ethereum)
	}

	if StartMining {
		utils.StartMining(ethereum)
	}

	// better reworked as cases
	if StartJsConsole {
		InitJsConsole(ethereum)
	} else if len(InputFile) > 0 {
		ExecJsFile(ethereum, InputFile)
	}

	if StartRpc {
		utils.StartRpc(ethereum, RpcPort)
	}

	utils.StartEthereum(ethereum, UseSeed)

	// this blocks the thread
	ethereum.WaitForShutdown()
	ethlog.Flush()
}
<|endoftext|>


<commit_before>/*
 * ZGrab Copyright 2015 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package telnet

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"net"
)

// RFC 854 - https://tools.ietf.org/html/rfc854
const (
	IAC                = byte(255) //Interpret as command
	DONT               = byte(254)
	DO                 = byte(253)
	WONT               = byte(252)
	WILL               = byte(251)
	GO_AHEAD           = byte(249) // Special go ahead command
	IAC_CMD_LENGTH     = 3         // IAC commands take 3 bytes (inclusive)
	READ_BUFFER_LENGTH = 8192
)

func GetTelnetBanner(logStruct *TelnetLog, conn net.Conn, maxReadSize int) (err error) {
	if err = NegotiateOptions(logStruct, conn); err != nil {
		return err
	}

	//grab banner
	buffer := make([]byte, READ_BUFFER_LENGTH)

	numBytes := len(buffer)
	rounds := int(math.Ceil(float64(maxReadSize) / READ_BUFFER_LENGTH))
	count := 0
	for numBytes != 0 && count < rounds && numBytes == READ_BUFFER_LENGTH {

		numBytes, err = conn.Read(buffer)
		if err != nil && err != io.EOF {
			return err
		}
		if count == rounds-1 {
			logStruct.Banner = logStruct.Banner + string(buffer[0:maxReadSize%READ_BUFFER_LENGTH])
		} else {
			logStruct.Banner = logStruct.Banner + string(buffer[0:numBytes])
		}
		count += 1
	}

	return nil
}

func NegotiateOptions(logStruct *TelnetLog, conn net.Conn) error {
	var readBuffer, retBuffer []byte
	var option, optionType, returnOptionType byte
	var iacIndex, firstUnreadIndex, numBytes, numDataBytes int
	var err error

	for finishedNegotiating := false; finishedNegotiating == false; {
		readBuffer = make([]byte, READ_BUFFER_LENGTH)
		retBuffer = nil
		numBytes, err = conn.Read(readBuffer)
		numDataBytes = numBytes

		if err != nil {
			return err
		}

		if numBytes == len(readBuffer) {
			return errors.New("Not enough buffer space for telnet options")
		}

		// Negotiate options

		firstUnreadIndex = 0
		for iacIndex = bytes.IndexByte(readBuffer, IAC); iacIndex != -1; iacIndex = bytes.IndexByte(readBuffer, IAC) {
			optionType = readBuffer[iacIndex+1]
			option = readBuffer[iacIndex+2]

			// ignore go ahead
			if optionType == GO_AHEAD {
				readBuffer = readBuffer[0:iacIndex]
				numBytes = iacIndex
				firstUnreadIndex = 0
				break
			}

			// record all offered options
			if optionType == WILL || optionType == DO {
				logStruct.SupportedOpts = append(logStruct.SupportedOpts, fmt.Sprintf("%d", option))
			} else if optionType == WONT || optionType == DONT {
				logStruct.UnsupportedOpts = append(logStruct.UnsupportedOpts, fmt.Sprintf("%d", option))
			}

			// reject all offered options
			if optionType == WILL || optionType == WONT {
				returnOptionType = DONT
			} else if optionType == DO || optionType == DONT {
				returnOptionType = WONT
			} else {
				return errors.New("Unsupported telnet IAC option type" + fmt.Sprintf("%d", optionType))
			}

			retBuffer = append(retBuffer, IAC)
			retBuffer = append(retBuffer, returnOptionType)
			retBuffer = append(retBuffer, option)

			firstUnreadIndex += iacIndex + IAC_CMD_LENGTH
			numDataBytes -= firstUnreadIndex
			readBuffer = readBuffer[firstUnreadIndex:]
		}

		if _, err = conn.Write(retBuffer); err != nil {
			return err
		}

		finishedNegotiating = numBytes != firstUnreadIndex
	}

	// no more IAC commands, just read the resulting data
	if numDataBytes >= 0 {
		logStruct.Banner = string(readBuffer[0:numDataBytes])
	}

	return nil
}
<commit_msg>ignore subsequent IACs after data<commit_after>/*
 * ZGrab Copyright 2015 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package telnet

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"net"
)

// RFC 854 - https://tools.ietf.org/html/rfc854
const (
	IAC                = byte(255) //Interpret as command
	DONT               = byte(254)
	DO                 = byte(253)
	WONT               = byte(252)
	WILL               = byte(251)
	GO_AHEAD           = byte(249) // Special go ahead command
	IAC_CMD_LENGTH     = 3         // IAC commands take 3 bytes (inclusive)
	READ_BUFFER_LENGTH = 8192
)

func GetTelnetBanner(logStruct *TelnetLog, conn net.Conn, maxReadSize int) (err error) {
	if err = NegotiateOptions(logStruct, conn); err != nil {
		return err
	}

	//grab banner
	buffer := make([]byte, READ_BUFFER_LENGTH)

	numBytes := len(buffer)
	rounds := int(math.Ceil(float64(maxReadSize) / READ_BUFFER_LENGTH))
	count := 0
	for numBytes != 0 && count < rounds && numBytes == READ_BUFFER_LENGTH {

		numBytes, err = conn.Read(buffer)
		if err != nil && err != io.EOF {
			return err
		}

		if containsIAC(buffer) {
			continue
		}

		if count == rounds-1 {
			logStruct.Banner = logStruct.Banner + string(buffer[0:maxReadSize%READ_BUFFER_LENGTH])
		} else {
			logStruct.Banner = logStruct.Banner + string(buffer[0:numBytes])
		}
		count += 1
	}

	return nil
}

func NegotiateOptions(logStruct *TelnetLog, conn net.Conn) error {
	var readBuffer, retBuffer []byte
	var option, optionType, returnOptionType byte
	var iacIndex, firstUnreadIndex, numBytes, numDataBytes int
	var err error

	for finishedNegotiating := false; finishedNegotiating == false; {
		readBuffer = make([]byte, READ_BUFFER_LENGTH)
		retBuffer = nil
		numBytes, err = conn.Read(readBuffer)
		numDataBytes = numBytes

		if err != nil {
			return err
		}

		if numBytes == len(readBuffer) {
			return errors.New("Not enough buffer space for telnet options")
		}

		// Negotiate options

		for iacIndex = bytes.IndexByte(readBuffer, IAC); iacIndex != -1; iacIndex = bytes.IndexByte(readBuffer, IAC) {
			firstUnreadIndex = 0
			optionType = readBuffer[iacIndex+1]
			option = readBuffer[iacIndex+2]

			// ignore go ahead
			if optionType == GO_AHEAD {
				readBuffer = readBuffer[0:iacIndex]
				numBytes = iacIndex
				firstUnreadIndex = 0
				break
			}

			// record all offered options
			if optionType == WILL || optionType == DO {
				logStruct.SupportedOpts = append(logStruct.SupportedOpts, fmt.Sprintf("%d", option))
			} else if optionType == WONT || optionType == DONT {
				logStruct.UnsupportedOpts = append(logStruct.UnsupportedOpts, fmt.Sprintf("%d", option))
			}

			// reject all offered options
			if optionType == WILL || optionType == WONT {
				returnOptionType = DONT
			} else if optionType == DO || optionType == DONT {
				returnOptionType = WONT
			} else {
				return errors.New("Unsupported telnet IAC option type" + fmt.Sprintf("%d", optionType))
			}

			retBuffer = append(retBuffer, IAC)
			retBuffer = append(retBuffer, returnOptionType)
			retBuffer = append(retBuffer, option)

			firstUnreadIndex = iacIndex + IAC_CMD_LENGTH
			numDataBytes -= firstUnreadIndex
			readBuffer = readBuffer[firstUnreadIndex:]
		}

		if _, err = conn.Write(retBuffer); err != nil {
			return err
		}

		numIACBytes := numBytes - numDataBytes
		finishedNegotiating = numBytes != numIACBytes
	}

	// no more IAC commands, just read the resulting data
	if numDataBytes >= 0 {
		logStruct.Banner = string(readBuffer[0:numBytes])
	}

	return nil
}

func containsIAC(buffer []byte) bool {
	return bytes.IndexByte(buffer, IAC) != -1
}
<|endoftext|>