<commit_before>package logging

import (
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/rs/xlog"
	"gopkg.in/errgo.v1"
)

var (
	conf *xlog.Config

	closeChan chan<- os.Signal
)

func SetCloseChan(c chan<- os.Signal) {
	closeChan = c
}

// CheckFatal exits the process if err != nil
func CheckFatal(err error) {
	if err != nil {
		pc, file, line, ok := runtime.Caller(1)
		if !ok {
			file = "?"
			line = 0
		}
		fn := runtime.FuncForPC(pc)
		var fnName string
		if fn == nil {
			fnName = "?()"
		} else {
			dotName := filepath.Ext(fn.Name())
			fnName = strings.TrimLeft(dotName, ".") + "()"
		}
		xlog.Errorf("%s:%d %s", file, line, fnName)
		xlog.Error("Fatal Error:", errgo.Details(err))
		if closeChan != nil {
			xlog.Warn("Sending close message")
			closeChan <- os.Interrupt
		}
		os.Exit(1)
	}
}

// SetupLogging will initialize the logger backend and set the flags.
func SetupLogging(w io.Writer) {
	if conf != nil {
		xlog.Error("logging: initializing twice! skipping...")
		return
	}
	conf = new(xlog.Config)
	var outs xlog.MultiOutput
	outs = append(outs, xlog.NewConsoleOutput())
	if w != nil {
		outs = append(outs, xlog.NewLogfmtOutput(w))
	}
	// if runtime.GOOS == "windows" { // colored ttys are rare on windows...
	// 	Underlying.Formatter = &logrus.TextFormatter{DisableColors: true}
	// }
	conf.Level = xlog.LevelWarn
	if lvl := os.Getenv("CRYPTIX_LOGLVL"); lvl != "" {
		var ok bool
		conf.Level, ok = map[string]xlog.Level{
			"debug": xlog.LevelDebug,
			"info":  xlog.LevelInfo,
			"error": xlog.LevelError,
		}[strings.ToLower(lvl)]
		if !ok {
			xlog.Warn("logging: could not match loglvl from env, defaulting to debug")
			conf.Level = xlog.LevelDebug
		}
	}
	conf.Output = outs
	xlog.SetLogger(xlog.New(*conf))
}

// Logger returns an Entry where the module field is set to name
func Logger(name string) xlog.Logger {
	if conf == nil {
		xlog.Error("logging: not initialized")
		os.Exit(1)
	}
	if name == "" {
		xlog.Warn("loging: missing name parameter")
		name = "undefined"
	}
	var thisConf = *conf // copy the logger
	// don't trash it's fields
	switch {
	case len(thisConf.Fields) == 0:
		thisConf.Fields = xlog.F{
			"unit": name,
		}
	case len(thisConf.Fields) > 0:
		thisConf.Fields["unit"] = name
	}
	return xlog.New(thisConf)
}
<commit_msg>logging: saner uninitialized handling<commit_after>package logging

import (
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/rs/xlog"
	"gopkg.in/errgo.v1"
)

var (
	conf *xlog.Config

	closeChan chan<- os.Signal
)

func SetCloseChan(c chan<- os.Signal) {
	closeChan = c
}

// CheckFatal exits the process if err != nil
func CheckFatal(err error) {
	if err != nil {
		pc, file, line, ok := runtime.Caller(1)
		if !ok {
			file = "?"
			line = 0
		}
		fn := runtime.FuncForPC(pc)
		var fnName string
		if fn == nil {
			fnName = "?()"
		} else {
			dotName := filepath.Ext(fn.Name())
			fnName = strings.TrimLeft(dotName, ".") + "()"
		}
		xlog.Errorf("%s:%d %s", file, line, fnName)
		xlog.Error("Fatal Error:", errgo.Details(err))
		if closeChan != nil {
			xlog.Warn("Sending close message")
			closeChan <- os.Interrupt
		}
		os.Exit(1)
	}
}

// SetupLogging will initialize the logger backend and set the flags.
func SetupLogging(w io.Writer) {
	if conf != nil {
		xlog.Error("logging: initializing twice! skipping...")
		return
	}
	conf = new(xlog.Config)
	var outs xlog.MultiOutput
	outs = append(outs, xlog.NewConsoleOutput())
	if w != nil {
		outs = append(outs, xlog.NewLogfmtOutput(w))
	}
	// if runtime.GOOS == "windows" { // colored ttys are rare on windows...
	// 	Underlying.Formatter = &logrus.TextFormatter{DisableColors: true}
	// }
	conf.Level = xlog.LevelWarn
	if lvl := os.Getenv("CRYPTIX_LOGLVL"); lvl != "" {
		var ok bool
		conf.Level, ok = map[string]xlog.Level{
			"debug": xlog.LevelDebug,
			"info":  xlog.LevelInfo,
			"error": xlog.LevelError,
		}[strings.ToLower(lvl)]
		if !ok {
			xlog.Warn("logging: could not match loglvl from env, defaulting to debug")
			conf.Level = xlog.LevelDebug
		}
	}
	conf.Output = outs
	xlog.SetLogger(xlog.New(*conf))
}

// Logger returns an Entry where the module field is set to name
func Logger(name string) xlog.Logger {
	if name == "" {
		xlog.Warn("loging: missing name parameter")
		name = "undefined"
	}
	var thisConf = conf
	if thisConf == nil {
		xlog.Warn("logging: not initialized yet.", xlog.F{"name": name})
		thisConf = &xlog.Config{
			Output: xlog.NewConsoleOutput(),
		}
	}
	l := xlog.New(*thisConf)
	l.SetField("unit", name)
	return l
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

// CompileProtos regenerates all of the generated source code for the Showcase
// API including the generated messages, gRPC services, go gapic clients,
// and the generated CLI. This must be run from the root directory
// of the gapic-showcase repository.
func CompileProtos(version string) {
	// Check if protoc is installed.
	if err := exec.Command("protoc", "--version").Run(); err != nil {
		log.Fatal("Error: 'protoc' is expected to be installed on the path.")
	}

	// Setup paths
	pwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error: unable to get working dir: %+v", err)
	}

	outDir, err := ioutil.TempDir(os.TempDir(), "gapic-showcase")
	if err != nil {
		log.Fatalf("Error: unable to create a temporary dir: %+v\n", err)
	}
	defer os.RemoveAll(outDir)

	protos := filepath.Join("schema", "google", "showcase", version, "*.proto")
	files, err := filepath.Glob(protos)
	if err != nil {
		log.Fatal("Error: failed to find protos in " + protos)
	}

	// Run protoc
	command := []string{
		"protoc",
		"--experimental_allow_proto3_optional",
		"--proto_path=schema/api-common-protos",
		"--proto_path=schema",
		"--go_cli_out=" + filepath.Join("cmd", "gapic-showcase"),
		"--go_cli_opt=root=gapic-showcase",
		"--go_cli_opt=gapic=github.com/googleapis/gapic-showcase/client",
		"--go_cli_opt=fmt=false",
		"--go_gapic_out=" + outDir,
		"--go_gapic_opt=go-gapic-package=github.com/googleapis/gapic-showcase/client;client",
		"--go_gapic_opt=grpc-service-config=schema/google/showcase/v1beta1/showcase_grpc_service_config.json",
		"--go_gapic_opt=api-service-config=schema/google/showcase/v1beta1/showcase_v1beta1.yaml",
		"--go_gapic_opt=metadata",
		"--go_rest_server_out=" + filepath.Join("server", "genrest"),
		"--go_out=plugins=grpc:" + outDir,
	}
	Execute(append(command, files...)...)

	// Copy generated code back into repo.
	tempClient := filepath.Join(outDir, "github.com", "googleapis", "gapic-showcase", "client")
	tempServer := filepath.Join(outDir, "github.com", "googleapis", "gapic-showcase", "server")
	command = []string{
		"cp",
		"-r",
		tempClient,
		tempServer,
		pwd,
	}
	Execute(command...)

	// Fix some generated errors.
	fixes := []struct {
		file string
		fix  string
	}{
		{
			"cmd/gapic-showcase/verify-test.go",
			"/ByteSliceVar/d",
		},
		{
			"cmd/gapic-showcase/wait.go",
			"s/EndEnd_time/EndEndTime/g",
		},
		// Fixes a name collision with the operation helper WaitOperation by renaming the mixin method.
		{
			"client/echo_client.go",
			"1,/WaitOperation(ctx/{s/WaitOperation(ctx/WaitOperationMixin(ctx/;}",
		},
		// Fixes a name collision with the operation helper WaitOperation by renaming the mixin method.
		{
			"client/echo_client_example_test.go",
			"1,/WaitOperation(ctx/{s/WaitOperation(ctx/WaitOperationMixin(ctx/;}",
		},
	}
	command = []string{
		"sed",
		"-i.bak",
	}
	for _, f := range fixes {
		Execute(append(command, f.fix, f.file)...)

		// Remove the backup file.
		Execute("rm", fmt.Sprintf("%s.bak", f.file))
	}

	// Format generated output
	Execute("go", "fmt", "./...")
}
<commit_msg>chore: remove WaitOperation workaround (#719)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

// CompileProtos regenerates all of the generated source code for the Showcase
// API including the generated messages, gRPC services, go gapic clients,
// and the generated CLI. This must be run from the root directory
// of the gapic-showcase repository.
func CompileProtos(version string) {
	// Check if protoc is installed.
	if err := exec.Command("protoc", "--version").Run(); err != nil {
		log.Fatal("Error: 'protoc' is expected to be installed on the path.")
	}

	// Setup paths
	pwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error: unable to get working dir: %+v", err)
	}

	outDir, err := ioutil.TempDir(os.TempDir(), "gapic-showcase")
	if err != nil {
		log.Fatalf("Error: unable to create a temporary dir: %+v\n", err)
	}
	defer os.RemoveAll(outDir)

	protos := filepath.Join("schema", "google", "showcase", version, "*.proto")
	files, err := filepath.Glob(protos)
	if err != nil {
		log.Fatal("Error: failed to find protos in " + protos)
	}

	// Run protoc
	command := []string{
		"protoc",
		"--experimental_allow_proto3_optional",
		"--proto_path=schema/api-common-protos",
		"--proto_path=schema",
		"--go_cli_out=" + filepath.Join("cmd", "gapic-showcase"),
		"--go_cli_opt=root=gapic-showcase",
		"--go_cli_opt=gapic=github.com/googleapis/gapic-showcase/client",
		"--go_cli_opt=fmt=false",
		"--go_gapic_out=" + outDir,
		"--go_gapic_opt=go-gapic-package=github.com/googleapis/gapic-showcase/client;client",
		"--go_gapic_opt=grpc-service-config=schema/google/showcase/v1beta1/showcase_grpc_service_config.json",
		"--go_gapic_opt=api-service-config=schema/google/showcase/v1beta1/showcase_v1beta1.yaml",
		"--go_gapic_opt=metadata",
		"--go_rest_server_out=" + filepath.Join("server", "genrest"),
		"--go_out=plugins=grpc:" + outDir,
	}
	Execute(append(command, files...)...)

	// Copy generated code back into repo.
	tempClient := filepath.Join(outDir, "github.com", "googleapis", "gapic-showcase", "client")
	tempServer := filepath.Join(outDir, "github.com", "googleapis", "gapic-showcase", "server")
	command = []string{
		"cp",
		"-r",
		tempClient,
		tempServer,
		pwd,
	}
	Execute(command...)

	// Fix some generated errors.
	fixes := []struct {
		file string
		fix  string
	}{
		{
			"cmd/gapic-showcase/verify-test.go",
			"/ByteSliceVar/d",
		},
		{
			"cmd/gapic-showcase/wait.go",
			"s/EndEnd_time/EndEndTime/g",
		},
	}
	command = []string{
		"sed",
		"-i.bak",
	}
	for _, f := range fixes {
		Execute(append(command, f.fix, f.file)...)

		// Remove the backup file.
		Execute("rm", fmt.Sprintf("%s.bak", f.file))
	}

	// Format generated output
	Execute("go", "fmt", "./...")
}
<|endoftext|>


<commit_before>package longpolling

import (
    "container/list"
    "encoding/json"
    "fmt"
    "io"
    "log"
    "net/http"
    "strconv"
    "time"

    "github.com/nu7hatch/gouuid"

    "github.com/jcuga/proxyblock/utils"
)

func StartLongpollManager() (chan Event, func(w http.ResponseWriter, r *http.Request)) {
    // TODO: make channel sizes a const or config
    clientRequestChan := make(chan ClientSubscription, 100)
    clientTimeoutChan := make(chan ClientSubPair, 100)
    events := make(chan Event, 100)
    quit := make(chan bool, 1)

    subManager := SubscriptionManager{
        ClientSubscriptions: clientRequestChan,
        ClientTimeouts:      clientTimeoutChan,
        Events:              events,
        ClientSubChannels:   make(map[string]map[uuid.UUID]chan<- []Event),
        SubEventBuffer:      make(map[string]EventBuffer),
        Quit:                quit,
        MaxEventBufferSize:  1000,
    }

    // Start subscription manager
    go subManager.Run()
    return events, getLongPollSubscriptionHandler(clientRequestChan, clientTimeoutChan)
}

// TODO: put sub manager and related event code in its own file leaving only
// lightweight example client

type ClientSubscription struct {
    ClientSubPair
    // used to ensure no events skipped between long polls
    LastEventTime time.Time
    // we channel arrays of events since we need to send everything a client
    // cares about in a single channel send.  This makes channel receives a
    // one shot deal.
    Events chan []Event
}

func NewClientSubscription(subscriptionCategory string, lastEventTime time.Time) (*ClientSubscription, error) {
    u, err := uuid.NewV4()
    if err != nil {
        return nil, err
    }
    subscription := ClientSubscription{
        ClientSubPair{*u, subscriptionCategory},
        lastEventTime,
        make(chan []Event, 1),
    }
    return &subscription, nil
}

// get web handler that has closure around sub chanel and clientTimeout channnel
func getLongPollSubscriptionHandler(subscriptionRequests chan ClientSubscription,
    clientTimeouts chan<- ClientSubPair) func(w http.ResponseWriter, r *http.Request) {
    return func(w http.ResponseWriter, r *http.Request) {
        timeout, err := strconv.Atoi(r.URL.Query().Get("timeout"))
        log.Println("Handling HTTP request at ", r.URL)
        // We are going to return json no matter what:
        w.Header().Set("Content-Type", "application/json")
        // Don't cache response:
        w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") // HTTP 1.1.
        w.Header().Set("Pragma", "no-cache")                                   // HTTP 1.0.
        w.Header().Set("Expires", "0")                                         // Proxies.
        if err != nil || timeout > 180 || timeout < 1 {
            log.Printf("Error: Invalid timeout param.  Must be 1-180. Got: %q.\n",
                r.URL.Query().Get("timeout"))
            io.WriteString(w, "{\"error\": \"Invalid timeout arg.  Must be 1-180.\"}")
            return
        }
        category := r.URL.Query().Get("category")
        if len(category) == 0 || len(category) > 255 {
            // TODO: add any extra validation on category?
            log.Printf("Error: Invalid subscription category.\n")
            io.WriteString(w, "{\"error\": \"Invalid subscription category.\"}")
            return
        }
        if err != nil {
            log.Printf("Error creating new Subscription: %s.\n", err)
            io.WriteString(w, "{\"error\": \"Error creating new Subscription.\"}")
            return
        }
        // Default to only looking for current events
        lastEventTime := time.Now()
        // since_time is string of milliseconds since epoch
        lastEventTimeParam := r.URL.Query().Get("since_time")
        if len(lastEventTimeParam) > 0 {
            // Client is requesting any event from given timestamp
            // parse time
            var parseError error
            lastEventTime, parseError = utils.MillisecondStringToTime(lastEventTimeParam)
            if parseError != nil {
                log.Printf(fmt.Sprintf(
                    "Error parsing last_event_time arg. Parm Value: %s, Error: %s.\n", lastEventTimeParam, err))
                io.WriteString(w, "{\"error\": \"Invalid last_event_time arg.\"}")
                return
            }
        }
        subscription, err := NewClientSubscription(category, lastEventTime)
        subscriptionRequests <- *subscription
        select {
        case <-time.After(time.Duration(timeout) * time.Second):
            // Lets the subscription manager know it can discard this request's
            // channel.
            clientTimeouts <- subscription.ClientSubPair
            io.WriteString(w, "{\"timeout\": \"no events before timeout\"}")
        case event := <-subscription.Events:
            // Consume event.  Subscription manager will automatically discard
            // this client's channel upon sending event
            // NOTE: event is actually []Event
            if jsonData, err := json.Marshal(EventResponse{&event}); err == nil {
                io.WriteString(w, string(jsonData))
            } else {
                io.WriteString(w, "{\"error\": \"json marshaller failed\"}")
            }
        }
    }
}

type EventResponse struct {
    Events *[]Event `json:"events"`
}

type Event struct {
    Timestamp int64  `json:"timestamp"` // milliseconds since epoch to match javascrits Date.getTime()
    Category  string `json:"category"`
    Data      string `json:"data"` // TODO: eventually make byte[] instead?
}

type ClientSubPair struct {
    ClientUUID           uuid.UUID
    SubscriptionCategory string
}

// TODO: make types/members private where ever it makes sense
type SubscriptionManager struct {
    ClientSubscriptions chan ClientSubscription
    ClientTimeouts      <-chan ClientSubPair
    Events              <-chan Event
    // Contains all client sub channels grouped first by sub id then by
    // client uuid
    ClientSubChannels map[string]map[uuid.UUID]chan<- []Event
    SubEventBuffer    map[string]EventBuffer // TODO: ptr to EventBuffer instead of actual value?
    // channel to inform manager to stop running
    Quit <-chan bool
    // How big the buffers are (1-n) before events are discareded FIFO
    // TODO: enforce sane range 1-n where n isn't batshit crazy
    MaxEventBufferSize int
}

// TODO: add func to create sub manager that adds vars for chan and buf sizes
// with validation

func (sm *SubscriptionManager) Run() error {
    log.Printf("SubscriptionManager: Starting run.")
    for {
        select {
        case newClient := <-sm.ClientSubscriptions:
            // before storing client sub request, see if we already have data in
            // the corresponding event buffer that we can use to fufil request
            // without storing it
            doQueueRequest := true
            if buf, found := sm.SubEventBuffer[newClient.SubscriptionCategory]; found {
                // We have a buffer for this sub category, check for buffered events
                if events, err := buf.GetEventsSince(newClient.LastEventTime); err == nil && len(events) > 0 {
                    doQueueRequest = false
                    log.Printf("SubscriptionManager: Skip adding client, sending %d events. (Category: %q Client: %s)",
                        len(events), newClient.SubscriptionCategory, newClient.ClientUUID.String())
                    fmt.Printf("EVENTS: %v\n", events)
                    // Send client buffered events.  Client will immediately consume
                    // and end long poll request, so no need to have manager store
                    newClient.Events <- events
                } else if err != nil {
                    log.Printf("Error getting events from event buffer: %s.", err)
                }
            }
            if doQueueRequest {
                // Couldn't find any immediate events, store for future:
                categoryClients, found := sm.ClientSubChannels[newClient.SubscriptionCategory]
                if !found {
                    // first request for this sub category, add client chan map entry
                    categoryClients = make(map[uuid.UUID]chan<- []Event)
                    sm.ClientSubChannels[newClient.SubscriptionCategory] = categoryClients
                }
                log.Printf("SubscriptionManager: Adding Client (Category: %q Client: %s)",
                    newClient.SubscriptionCategory, newClient.ClientUUID.String())
                // TODO: unit tests to ensure clients add/skip behavior correct 'n tight
                categoryClients[newClient.ClientUUID] = newClient.Events
            }
        case disconnect := <-sm.ClientTimeouts:
            if subCategoryClients, found := sm.ClientSubChannels[disconnect.SubscriptionCategory]; found {
                // NOTE:  The delete function doesn't return anything, and will do nothing if the
                // specified key doesn't exist.
                delete(subCategoryClients, disconnect.ClientUUID)
                log.Printf("SubscriptionManager: Removing Client (Category: %q Client: %s)",
                    disconnect.SubscriptionCategory, disconnect.ClientUUID.String())
            } else {
                // Sub category entry not found.  Weird.  Log this!
                log.Printf("Warning: cleint disconnect for non-existing subscription category: %q",
                    disconnect.SubscriptionCategory)
            }
        case event := <-sm.Events:
            // Send event to any listening client's channels
            if clients, found := sm.ClientSubChannels[event.Category]; found && len(clients) > 0 {
                log.Printf("SubscriptionManager: forwarding event to %d clients. (event: %v)", len(clients), event)
                for clientUUID, clientChan := range clients {
                    log.Printf("SubscriptionManager: sending event to client: %s", clientUUID.String())
                    clientChan <- []Event{event}
                    // boot this client subscription since we found events
                    // In longpolling, subscriptions only last until there is
                    // data (happening here) or a timeout (handled by the
                    //disconnect case above)
                    // NOTE: it IS safe to delete map entries as you iterate
                    // SEE: http://stackoverflow.com/questions/23229975/is-it-safe-to-remove-selected-keys-from-golang-map-within-a-range-loop
                    log.Printf("SubscriptionManager: Removing client after event send: %s", clientUUID.String())
                    delete(clients, clientUUID)
                }
            }
            // Add event buffer for this event's subscription category if doesn't exit
            buf, bufFound := sm.SubEventBuffer[event.Category]
            if !bufFound {
                buf = EventBuffer{
                    list.New(),
                    sm.MaxEventBufferSize,
                }
                sm.SubEventBuffer[event.Category] = buf
            }
            log.Printf("SubscriptionManager: queue event: %v.", event)
            // queue event in event buffer
            buf.QueueEvent(&event)
        case _ = <-sm.Quit:
            log.Printf("SubscriptionManager: received quit signal, stopping.")
            return nil
        }
    }
}

type EventBuffer struct {
    // Doubly linked list of events where new events are added to the back/tail
    // and old events are removed from the front/root
    // NOTE: this is efficient for front/back operations since it is
    // implemented as a ring with root.prev being the tail
    // SEE: https://golang.org/src/container/list/list.go
    *list.List
    MaxBufferSize int
}

func (eb *EventBuffer) QueueEvent(event *Event) error {
    // Cull our buffer if we're at max capacity
    if eb.List.Len() > eb.MaxBufferSize {
        oldestEvent := eb.List.Back()
        if oldestEvent != nil {
            eb.List.Remove(oldestEvent)
        }
    }
    // Add event to front of our list
    eb.List.PushFront(event)
    return nil
}

func (eb *EventBuffer) GetEventsSince(since time.Time) ([]Event, error) {
    events := make([]Event, 0) // NOTE: having init cap > 0 has zero value Event
    // structs which we don't want!
    // events are bufferd FIFO with the most recent event in the front of the
    // buffer (list)
    for element := eb.List.Front(); element != nil; element = element.Next() {
        event, ok := element.Value.(*Event)
        if !ok {
            return nil, fmt.Errorf("Found non-event type in event buffer.")
        }
        // event time is after since time arg? convert since to epoch ms
        if event.Timestamp > utils.TimeToEpochMilliseconds(since) {
            events = append(events, Event{event.Timestamp, event.Category, event.Data})
        } else {
            // we've made it to events we've seen before, stop searching
            break
        }
    }
    // NOTE: events has the most recent event first followed by any older events
    // that occurred since client's last seen event
    // TODO: consider reversing order?  or is it an advantage to have
    // newest first so handled with more priority?  TODO: make this an option?
    return events, nil
}

// adapted from:
// http://stackoverflow.com/questions/13294649/how-to-parse-a-milliseconds-since-epoch-timestamp-string-in-go
func MillisecondStringToTime(ms string) (time.Time, error) {
    msInt, err := strconv.ParseInt(ms, 10, 64)
    if err != nil {
        return time.Time{}, err
    }
    return time.Unix(0, msInt*int64(time.Millisecond)), nil
}

// adapted from:
// http://stackoverflow.com/questions/24122821/go-golang-time-now-unixnano-convert-to-milliseconds
func TimeToEpochMilliseconds(t time.Time) int64 {
    return t.UnixNano() / int64(time.Millisecond)
}
<commit_msg>fixing package name<commit_after>package golongpoll

import (
    "container/list"
    "encoding/json"
    "fmt"
    "io"
    "log"
    "net/http"
    "strconv"
    "time"

    "github.com/nu7hatch/gouuid"

    "github.com/jcuga/proxyblock/utils"
)

func StartLongpollManager() (chan Event, func(w http.ResponseWriter, r *http.Request)) {
    // TODO: make channel sizes a const or config
    clientRequestChan := make(chan ClientSubscription, 100)
    clientTimeoutChan := make(chan ClientSubPair, 100)
    events := make(chan Event, 100)
    quit := make(chan bool, 1)

    subManager := SubscriptionManager{
        ClientSubscriptions: clientRequestChan,
        ClientTimeouts:      clientTimeoutChan,
        Events:              events,
        ClientSubChannels:   make(map[string]map[uuid.UUID]chan<- []Event),
        SubEventBuffer:      make(map[string]EventBuffer),
        Quit:                quit,
        MaxEventBufferSize:  1000,
    }

    // Start subscription manager
    go subManager.Run()
    return events, getLongPollSubscriptionHandler(clientRequestChan, clientTimeoutChan)
}

// TODO: put sub manager and related event code in its own file leaving only
// lightweight example client

type ClientSubscription struct {
    ClientSubPair
    // used to ensure no events skipped between long polls
    LastEventTime time.Time
    // we channel arrays of events since we need to send everything a client
    // cares about in a single channel send.  This makes channel receives a
    // one shot deal.
    Events chan []Event
}

func NewClientSubscription(subscriptionCategory string, lastEventTime time.Time) (*ClientSubscription, error) {
    u, err := uuid.NewV4()
    if err != nil {
        return nil, err
    }
    subscription := ClientSubscription{
        ClientSubPair{*u, subscriptionCategory},
        lastEventTime,
        make(chan []Event, 1),
    }
    return &subscription, nil
}

// get web handler that has closure around sub chanel and clientTimeout channnel
func getLongPollSubscriptionHandler(subscriptionRequests chan ClientSubscription,
    clientTimeouts chan<- ClientSubPair) func(w http.ResponseWriter, r *http.Request) {
    return func(w http.ResponseWriter, r *http.Request) {
        timeout, err := strconv.Atoi(r.URL.Query().Get("timeout"))
        log.Println("Handling HTTP request at ", r.URL)
        // We are going to return json no matter what:
        w.Header().Set("Content-Type", "application/json")
        // Don't cache response:
        w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") // HTTP 1.1.
        w.Header().Set("Pragma", "no-cache")                                   // HTTP 1.0.
        w.Header().Set("Expires", "0")                                         // Proxies.
        if err != nil || timeout > 180 || timeout < 1 {
            log.Printf("Error: Invalid timeout param.  Must be 1-180. Got: %q.\n",
                r.URL.Query().Get("timeout"))
            io.WriteString(w, "{\"error\": \"Invalid timeout arg.  Must be 1-180.\"}")
            return
        }
        category := r.URL.Query().Get("category")
        if len(category) == 0 || len(category) > 255 {
            // TODO: add any extra validation on category?
            log.Printf("Error: Invalid subscription category.\n")
            io.WriteString(w, "{\"error\": \"Invalid subscription category.\"}")
            return
        }
        if err != nil {
            log.Printf("Error creating new Subscription: %s.\n", err)
            io.WriteString(w, "{\"error\": \"Error creating new Subscription.\"}")
            return
        }
        // Default to only looking for current events
        lastEventTime := time.Now()
        // since_time is string of milliseconds since epoch
        lastEventTimeParam := r.URL.Query().Get("since_time")
        if len(lastEventTimeParam) > 0 {
            // Client is requesting any event from given timestamp
            // parse time
            var parseError error
            lastEventTime, parseError = utils.MillisecondStringToTime(lastEventTimeParam)
            if parseError != nil {
                log.Printf(fmt.Sprintf(
                    "Error parsing last_event_time arg. Parm Value: %s, Error: %s.\n", lastEventTimeParam, err))
                io.WriteString(w, "{\"error\": \"Invalid last_event_time arg.\"}")
                return
            }
        }
        subscription, err := NewClientSubscription(category, lastEventTime)
        subscriptionRequests <- *subscription
        select {
        case <-time.After(time.Duration(timeout) * time.Second):
            // Lets the subscription manager know it can discard this request's
            // channel.
            clientTimeouts <- subscription.ClientSubPair
            io.WriteString(w, "{\"timeout\": \"no events before timeout\"}")
        case event := <-subscription.Events:
            // Consume event.  Subscription manager will automatically discard
            // this client's channel upon sending event
            // NOTE: event is actually []Event
            if jsonData, err := json.Marshal(EventResponse{&event}); err == nil {
                io.WriteString(w, string(jsonData))
            } else {
                io.WriteString(w, "{\"error\": \"json marshaller failed\"}")
            }
        }
    }
}

type EventResponse struct {
    Events *[]Event `json:"events"`
}

type Event struct {
    Timestamp int64  `json:"timestamp"` // milliseconds since epoch to match javascrits Date.getTime()
    Category  string `json:"category"`
    Data      string `json:"data"` // TODO: eventually make byte[] instead?
}

type ClientSubPair struct {
    ClientUUID           uuid.UUID
    SubscriptionCategory string
}

// TODO: make types/members private where ever it makes sense
type SubscriptionManager struct {
    ClientSubscriptions chan ClientSubscription
    ClientTimeouts      <-chan ClientSubPair
    Events              <-chan Event
    // Contains all client sub channels grouped first by sub id then by
    // client uuid
    ClientSubChannels map[string]map[uuid.UUID]chan<- []Event
    SubEventBuffer    map[string]EventBuffer // TODO: ptr to EventBuffer instead of actual value?
    // channel to inform manager to stop running
    Quit <-chan bool
    // How big the buffers are (1-n) before events are discareded FIFO
    // TODO: enforce sane range 1-n where n isn't batshit crazy
    MaxEventBufferSize int
}

// TODO: add func to create sub manager that adds vars for chan and buf sizes
// with validation

func (sm *SubscriptionManager) Run() error {
    log.Printf("SubscriptionManager: Starting run.")
    for {
        select {
        case newClient := <-sm.ClientSubscriptions:
            // before storing client sub request, see if we already have data in
            // the corresponding event buffer that we can use to fufil request
            // without storing it
            doQueueRequest := true
            if buf, found := sm.SubEventBuffer[newClient.SubscriptionCategory]; found {
                // We have a buffer for this sub category, check for buffered events
                if events, err := buf.GetEventsSince(newClient.LastEventTime); err == nil && len(events) > 0 {
                    doQueueRequest = false
                    log.Printf("SubscriptionManager: Skip adding client, sending %d events. (Category: %q Client: %s)",
                        len(events), newClient.SubscriptionCategory, newClient.ClientUUID.String())
                    fmt.Printf("EVENTS: %v\n", events)
                    // Send client buffered events.  Client will immediately consume
                    // and end long poll request, so no need to have manager store
                    newClient.Events <- events
                } else if err != nil {
                    log.Printf("Error getting events from event buffer: %s.", err)
                }
            }
            if doQueueRequest {
                // Couldn't find any immediate events, store for future:
                categoryClients, found := sm.ClientSubChannels[newClient.SubscriptionCategory]
                if !found {
                    // first request for this sub category, add client chan map entry
                    categoryClients = make(map[uuid.UUID]chan<- []Event)
                    sm.ClientSubChannels[newClient.SubscriptionCategory] = categoryClients
                }
                log.Printf("SubscriptionManager: Adding Client (Category: %q Client: %s)",
                    newClient.SubscriptionCategory, newClient.ClientUUID.String())
                // TODO: unit tests to ensure clients add/skip behavior correct 'n tight
                categoryClients[newClient.ClientUUID] = newClient.Events
            }
        case disconnect := <-sm.ClientTimeouts:
            if subCategoryClients, found := sm.ClientSubChannels[disconnect.SubscriptionCategory]; found {
                // NOTE:  The delete function doesn't return anything, and will do nothing if the
                // specified key doesn't exist.
                delete(subCategoryClients, disconnect.ClientUUID)
                log.Printf("SubscriptionManager: Removing Client (Category: %q Client: %s)",
                    disconnect.SubscriptionCategory, disconnect.ClientUUID.String())
            } else {
                // Sub category entry not found.  Weird.  Log this!
                log.Printf("Warning: cleint disconnect for non-existing subscription category: %q",
                    disconnect.SubscriptionCategory)
            }
        case event := <-sm.Events:
            // Send event to any listening client's channels
            if clients, found := sm.ClientSubChannels[event.Category]; found && len(clients) > 0 {
                log.Printf("SubscriptionManager: forwarding event to %d clients. (event: %v)", len(clients), event)
                for clientUUID, clientChan := range clients {
                    log.Printf("SubscriptionManager: sending event to client: %s", clientUUID.String())
                    clientChan <- []Event{event}
                    // boot this client subscription since we found events
                    // In longpolling, subscriptions only last until there is
                    // data (happening here) or a timeout (handled by the
                    //disconnect case above)
                    // NOTE: it IS safe to delete map entries as you iterate
                    // SEE: http://stackoverflow.com/questions/23229975/is-it-safe-to-remove-selected-keys-from-golang-map-within-a-range-loop
                    log.Printf("SubscriptionManager: Removing client after event send: %s", clientUUID.String())
                    delete(clients, clientUUID)
                }
            }
            // Add event buffer for this event's subscription category if doesn't exit
            buf, bufFound := sm.SubEventBuffer[event.Category]
            if !bufFound {
                buf = EventBuffer{
                    list.New(),
                    sm.MaxEventBufferSize,
                }
                sm.SubEventBuffer[event.Category] = buf
            }
            log.Printf("SubscriptionManager: queue event: %v.", event)
            // queue event in event buffer
            buf.QueueEvent(&event)
        case _ = <-sm.Quit:
            log.Printf("SubscriptionManager: received quit signal, stopping.")
            return nil
        }
    }
}

type EventBuffer struct {
    // Doubly linked list of events where new events are added to the back/tail
    // and old events are removed from the front/root
    // NOTE: this is efficient for front/back operations since it is
    // implemented as a ring with root.prev being the tail
    // SEE: https://golang.org/src/container/list/list.go
    *list.List
    MaxBufferSize int
}

func (eb *EventBuffer) QueueEvent(event *Event) error {
    // Cull our buffer if we're at max capacity
    if eb.List.Len() > eb.MaxBufferSize {
        oldestEvent := eb.List.Back()
        if oldestEvent != nil {
            eb.List.Remove(oldestEvent)
        }
    }
    // Add event to front of our list
    eb.List.PushFront(event)
    return nil
}

func (eb *EventBuffer) GetEventsSince(since time.Time) ([]Event, error) {
    events := make([]Event, 0) // NOTE: having init cap > 0 has zero value Event
    // structs which we don't want!
    // events are bufferd FIFO with the most recent event in the front of the
    // buffer (list)
    for element := eb.List.Front(); element != nil; element = element.Next() {
        event, ok := element.Value.(*Event)
        if !ok {
            return nil, fmt.Errorf("Found non-event type in event buffer.")
        }
        // event time is after since time arg? convert since to epoch ms
        if event.Timestamp > utils.TimeToEpochMilliseconds(since) {
            events = append(events, Event{event.Timestamp, event.Category, event.Data})
        } else {
            // we've made it to events we've seen before, stop searching
            break
        }
    }
    // NOTE: events has the most recent event first followed by any older events
    // that occurred since client's last seen event
    // TODO: consider reversing order?  or is it an advantage to have
    // newest first so handled with more priority?  TODO: make this an option?
    return events, nil
}

// adapted from:
// http://stackoverflow.com/questions/13294649/how-to-parse-a-milliseconds-since-epoch-timestamp-string-in-go
func MillisecondStringToTime(ms string) (time.Time, error) {
    msInt, err := strconv.ParseInt(ms, 10, 64)
    if err != nil {
        return time.Time{}, err
    }
    return time.Unix(0, msInt*int64(time.Millisecond)), nil
}

// adapted from:
// http://stackoverflow.com/questions/24122821/go-golang-time-now-unixnano-convert-to-milliseconds
func TimeToEpochMilliseconds(t time.Time) int64 {
    return t.UnixNano() / int64(time.Millisecond)
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configgrpc

import (
	"crypto/tls"
	"fmt"
	"net"
	"strings"
	"time"

	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
	"go.opentelemetry.io/otel"
	"google.golang.org/grpc"
	"google.golang.org/grpc/balancer/roundrobin"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/encoding/gzip"
	"google.golang.org/grpc/keepalive"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/configauth"
	"go.opentelemetry.io/collector/config/confignet"
	"go.opentelemetry.io/collector/config/configtls"
)

// Compression gRPC keys for supported compression types within collector.
const (
	CompressionUnsupported = ""
	CompressionGzip        = "gzip"

	PerRPCAuthTypeBearer = "bearer"
)

var (
	// Map of opentelemetry compression types to grpc registered compression types.
	gRPCCompressionKeyMap = map[string]string{
		CompressionGzip: gzip.Name,
	}
)

// Allowed balancer names to be set in grpclb_policy to discover the servers.
var allowedBalancerNames = []string{roundrobin.Name, grpc.PickFirstBalancerName}

// KeepaliveClientConfig exposes the keepalive.ClientParameters to be used by the exporter.
// Refer to the original data-structure for the meaning of each parameter:
// https://godoc.org/google.golang.org/grpc/keepalive#ClientParameters
type KeepaliveClientConfig struct {
	Time                time.Duration `mapstructure:"time,omitempty"`
	Timeout             time.Duration `mapstructure:"timeout,omitempty"`
	PermitWithoutStream bool          `mapstructure:"permit_without_stream,omitempty"`
}

// GRPCClientSettings defines common settings for a gRPC client configuration.
type GRPCClientSettings struct {
	// The target to which the exporter is going to send traces or metrics,
	// using the gRPC protocol. The valid syntax is described at
	// https://github.com/grpc/grpc/blob/master/doc/naming.md.
	Endpoint string `mapstructure:"endpoint"`

	// The compression key for supported compression types within
	// collector. Currently the only supported mode is `gzip`.
	Compression string `mapstructure:"compression"`

	// TLSSetting struct exposes TLS client configuration.
	TLSSetting configtls.TLSClientSetting `mapstructure:",squash"`

	// The keepalive parameters for gRPC client. See grpc.WithKeepaliveParams.
	// (https://godoc.org/google.golang.org/grpc#WithKeepaliveParams).
	Keepalive *KeepaliveClientConfig `mapstructure:"keepalive"`

	// ReadBufferSize for gRPC client. See grpc.WithReadBufferSize.
	// (https://godoc.org/google.golang.org/grpc#WithReadBufferSize).
	ReadBufferSize int `mapstructure:"read_buffer_size"`

	// WriteBufferSize for gRPC gRPC. See grpc.WithWriteBufferSize.
	// (https://godoc.org/google.golang.org/grpc#WithWriteBufferSize).
	WriteBufferSize int `mapstructure:"write_buffer_size"`

	// WaitForReady parameter configures client to wait for ready state before sending data.
	// (https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md)
	WaitForReady bool `mapstructure:"wait_for_ready"`

	// The headers associated with gRPC requests.
	Headers map[string]string `mapstructure:"headers"`

	// Sets the balancer in grpclb_policy to discover the servers. Default is pick_first.
	// https://github.com/grpc/grpc-go/blob/master/examples/features/load_balancing/README.md
	BalancerName string `mapstructure:"balancer_name"`

	// Auth configuration for outgoing RPCs.
	Auth *configauth.Authentication `mapstructure:"auth,omitempty"`
}

// KeepaliveServerConfig is the configuration for keepalive.
type KeepaliveServerConfig struct {
	ServerParameters  *KeepaliveServerParameters  `mapstructure:"server_parameters,omitempty"`
	EnforcementPolicy *KeepaliveEnforcementPolicy `mapstructure:"enforcement_policy,omitempty"`
}

// KeepaliveServerParameters allow configuration of the keepalive.ServerParameters.
// The same default values as keepalive.ServerParameters are applicable and get applied by the server.
// See https://godoc.org/google.golang.org/grpc/keepalive#ServerParameters for details.
type KeepaliveServerParameters struct {
	MaxConnectionIdle     time.Duration `mapstructure:"max_connection_idle,omitempty"`
	MaxConnectionAge      time.Duration `mapstructure:"max_connection_age,omitempty"`
	MaxConnectionAgeGrace time.Duration `mapstructure:"max_connection_age_grace,omitempty"`
	Time                  time.Duration `mapstructure:"time,omitempty"`
	Timeout               time.Duration `mapstructure:"timeout,omitempty"`
}

// KeepaliveEnforcementPolicy allow configuration of the keepalive.EnforcementPolicy.
// The same default values as keepalive.EnforcementPolicy are applicable and get applied by the server.
// See https://godoc.org/google.golang.org/grpc/keepalive#EnforcementPolicy for details.
type KeepaliveEnforcementPolicy struct {
	MinTime             time.Duration `mapstructure:"min_time,omitempty"`
	PermitWithoutStream bool          `mapstructure:"permit_without_stream,omitempty"`
}

// GRPCServerSettings defines common settings for a gRPC server configuration.
type GRPCServerSettings struct {
	// Server net.Addr config. For transport only "tcp" and "unix" are valid options.
	NetAddr confignet.NetAddr `mapstructure:",squash"`

	// Configures the protocol to use TLS.
	// The default value is nil, which will cause the protocol to not use TLS.
	TLSSetting *configtls.TLSServerSetting `mapstructure:"tls_settings,omitempty"`

	// MaxRecvMsgSizeMiB sets the maximum size (in MiB) of messages accepted by the server.
	MaxRecvMsgSizeMiB uint64 `mapstructure:"max_recv_msg_size_mib"`

	// MaxConcurrentStreams sets the limit on the number of concurrent streams to each ServerTransport.
	// It has effect only for streaming RPCs.
	MaxConcurrentStreams uint32 `mapstructure:"max_concurrent_streams"`

	// ReadBufferSize for gRPC server. See grpc.ReadBufferSize.
	// (https://godoc.org/google.golang.org/grpc#ReadBufferSize).
	ReadBufferSize int `mapstructure:"read_buffer_size"`

	// WriteBufferSize for gRPC server. See grpc.WriteBufferSize.
	// (https://godoc.org/google.golang.org/grpc#WriteBufferSize).
	WriteBufferSize int `mapstructure:"write_buffer_size"`

	// Keepalive anchor for all the settings related to keepalive.
	Keepalive *KeepaliveServerConfig `mapstructure:"keepalive,omitempty"`

	// Auth for this receiver
	Auth *configauth.Authentication `mapstructure:"auth,omitempty"`
}

// SanitizedEndpoint strips the prefix of either http:// or https:// from configgrpc.GRPCClientSettings.Endpoint.
func (gcs *GRPCClientSettings) SanitizedEndpoint() string {
	switch {
	case gcs.isSchemeHTTP():
		return strings.TrimPrefix(gcs.Endpoint, "http://")
	case gcs.isSchemeHTTPS():
		return strings.TrimPrefix(gcs.Endpoint, "https://")
	default:
		return gcs.Endpoint
	}
}

func (gcs *GRPCClientSettings) isSchemeHTTP() bool {
	return strings.HasPrefix(gcs.Endpoint, "http://")
}

func (gcs *GRPCClientSettings) isSchemeHTTPS() bool {
	return strings.HasPrefix(gcs.Endpoint, "https://")
}

// ToDialOptions maps configgrpc.GRPCClientSettings to a slice of dial options for gRPC.
func (gcs *GRPCClientSettings) ToDialOptions(ext map[config.ComponentID]component.Extension) ([]grpc.DialOption, error) {
	var opts []grpc.DialOption
	if gcs.Compression != "" {
		if compressionKey := GetGRPCCompressionKey(gcs.Compression); compressionKey != CompressionUnsupported {
			opts = append(opts, grpc.WithDefaultCallOptions(grpc.UseCompressor(compressionKey)))
		} else {
			return nil, fmt.Errorf("unsupported compression type %q", gcs.Compression)
		}
	}

	tlsCfg, err := gcs.TLSSetting.LoadTLSConfig()
	if err != nil {
		return nil, err
	}
	tlsDialOption := grpc.WithInsecure()
	if tlsCfg != nil {
		tlsDialOption = grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))
	} else if gcs.isSchemeHTTPS() {
		tlsDialOption = grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))
	}
	opts = append(opts, tlsDialOption)

	if gcs.ReadBufferSize > 0 {
		opts = append(opts, grpc.WithReadBufferSize(gcs.ReadBufferSize))
	}

	if gcs.WriteBufferSize > 0 {
		opts = append(opts, grpc.WithWriteBufferSize(gcs.WriteBufferSize))
	}

	if gcs.Keepalive != nil {
		keepAliveOption := grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time:                gcs.Keepalive.Time,
			Timeout:             gcs.Keepalive.Timeout,
			PermitWithoutStream: gcs.Keepalive.PermitWithoutStream,
		})
		opts = append(opts, keepAliveOption)
	}

	if gcs.Auth != nil {
		if ext == nil {
			return nil, fmt.Errorf("no extensions configuration available")
		}

		componentID, cperr := config.NewIDFromString(gcs.Auth.AuthenticatorName)
		if cperr != nil {
			return nil, cperr
		}

		grpcAuthenticator, cerr := configauth.GetGRPCClientAuthenticator(ext, componentID)
		if cerr != nil {
			return nil, cerr
		}

		perRPCCredentials, perr := grpcAuthenticator.PerRPCCredentials()
		if perr != nil {
			return nil, err
		}
		opts = append(opts, grpc.WithPerRPCCredentials(perRPCCredentials))
	}

	if gcs.BalancerName != "" {
		valid := validateBalancerName(gcs.BalancerName)
		if !valid {
			return nil, fmt.Errorf("invalid balancer_name: %s", gcs.BalancerName)
		}
		opts = append(opts, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingPolicy":"%s"}`, gcs.BalancerName)))
	}

	// Enable OpenTelemetry observability plugin.
	opts = append(opts, grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()))
	opts = append(opts, grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()))

	return opts, nil
}

func validateBalancerName(balancerName string) bool {
	for _, item := range allowedBalancerNames {
		if item == balancerName {
			return true
		}
	}
	return false
}

// ToListener returns the net.Listener constructed from the settings.
func (gss *GRPCServerSettings) ToListener() (net.Listener, error) {
	return gss.NetAddr.Listen()
}

// ToServerOption maps configgrpc.GRPCServerSettings to a slice of server options for gRPC.
func (gss *GRPCServerSettings) ToServerOption(ext map[config.ComponentID]component.Extension) ([]grpc.ServerOption, error) {
	var opts []grpc.ServerOption

	if gss.TLSSetting != nil {
		tlsCfg, err := gss.TLSSetting.LoadTLSConfig()
		if err != nil {
			return nil, err
		}
		opts = append(opts, grpc.Creds(credentials.NewTLS(tlsCfg)))
	}

	if gss.MaxRecvMsgSizeMiB > 0 {
		opts = append(opts, grpc.MaxRecvMsgSize(int(gss.MaxRecvMsgSizeMiB*1024*1024)))
	}

	if gss.MaxConcurrentStreams > 0 {
		opts = append(opts, grpc.MaxConcurrentStreams(gss.MaxConcurrentStreams))
	}

	if gss.ReadBufferSize > 0 {
		opts = append(opts, grpc.ReadBufferSize(gss.ReadBufferSize))
	}

	if gss.WriteBufferSize > 0 {
		opts = append(opts, grpc.WriteBufferSize(gss.WriteBufferSize))
	}

	// The default values referenced in the GRPC docs are set within the server, so this code doesn't need
	// to apply them over zero/nil values before passing these as grpc.ServerOptions.
	// The following shows the server code for applying default grpc.ServerOptions.
	// https://github.com/grpc/grpc-go/blob/120728e1f775e40a2a764341939b78d666b08260/internal/transport/http2_server.go#L184-L200
	if gss.Keepalive != nil {
		if gss.Keepalive.ServerParameters != nil {
			svrParams := gss.Keepalive.ServerParameters
			opts = append(opts, grpc.KeepaliveParams(keepalive.ServerParameters{
				MaxConnectionIdle:     svrParams.MaxConnectionIdle,
				MaxConnectionAge:      svrParams.MaxConnectionAge,
				MaxConnectionAgeGrace: svrParams.MaxConnectionAgeGrace,
				Time:                  svrParams.Time,
				Timeout:               svrParams.Timeout,
			}))
		}
		// The default values referenced in the GRPC are set within the server, so this code doesn't need
		// to apply them over zero/nil values before passing these as grpc.ServerOptions.
		// The following shows the server code for applying default grpc.ServerOptions.
		// https://github.com/grpc/grpc-go/blob/120728e1f775e40a2a764341939b78d666b08260/internal/transport/http2_server.go#L202-L205
		if gss.Keepalive.EnforcementPolicy != nil {
			enfPol := gss.Keepalive.EnforcementPolicy
			opts = append(opts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
				MinTime:             enfPol.MinTime,
				PermitWithoutStream: enfPol.PermitWithoutStream,
			}))
		}
	}

	if gss.Auth != nil {
		componentID, cperr := config.NewIDFromString(gss.Auth.AuthenticatorName)
		if cperr != nil {
			return nil, cperr
		}

		authenticator, err := configauth.GetServerAuthenticator(ext, componentID)
		if err != nil {
			return nil, err
		}

		opts = append(opts,
			grpc.UnaryInterceptor(authenticator.GRPCUnaryServerInterceptor),
			grpc.StreamInterceptor(authenticator.GRPCStreamServerInterceptor),
		)
	}

	// Enable OpenTelemetry observability plugin.
	// TODO: Pass construct settings to have access to Tracer.
	opts = append(opts, grpc.UnaryInterceptor(
		otelgrpc.UnaryServerInterceptor(
			otelgrpc.WithTracerProvider(otel.GetTracerProvider()),
			otelgrpc.WithPropagators(otel.GetTextMapPropagator()),
		)))
	opts = append(opts, grpc.StreamInterceptor(
		otelgrpc.StreamServerInterceptor(
			otelgrpc.WithTracerProvider(otel.GetTracerProvider()),
			otelgrpc.WithPropagators(otel.GetTextMapPropagator()),
		)))

	return opts, nil
}

// GetGRPCCompressionKey returns the grpc registered compression key if the
// passed in compression key is supported, and CompressionUnsupported otherwise.
func GetGRPCCompressionKey(compressionType string) string {
	compressionKey := strings.ToLower(compressionType)
	if encodingKey, ok := gRPCCompressionKeyMap[compressionKey]; ok {
		return encodingKey
	}
	return CompressionUnsupported
}
<commit_msg>Remove unused public constant from configgrpc (#3702)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configgrpc

import (
	"crypto/tls"
	"fmt"
	"net"
	"strings"
	"time"

	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
	"go.opentelemetry.io/otel"
	"google.golang.org/grpc"
	"google.golang.org/grpc/balancer/roundrobin"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/encoding/gzip"
	"google.golang.org/grpc/keepalive"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/configauth"
	"go.opentelemetry.io/collector/config/confignet"
	"go.opentelemetry.io/collector/config/configtls"
)

// Compression gRPC keys for supported compression types within collector.
const (
	CompressionUnsupported = ""
	CompressionGzip        = "gzip"
)

var (
	// Map of opentelemetry compression types to grpc registered compression types.
	gRPCCompressionKeyMap = map[string]string{
		CompressionGzip: gzip.Name,
	}
)

// Allowed balancer names to be set in grpclb_policy to discover the servers.
var allowedBalancerNames = []string{roundrobin.Name, grpc.PickFirstBalancerName}

// KeepaliveClientConfig exposes the keepalive.ClientParameters to be used by the exporter.
// Refer to the original data-structure for the meaning of each parameter:
// https://godoc.org/google.golang.org/grpc/keepalive#ClientParameters
type KeepaliveClientConfig struct {
	Time                time.Duration `mapstructure:"time,omitempty"`
	Timeout             time.Duration `mapstructure:"timeout,omitempty"`
	PermitWithoutStream bool          `mapstructure:"permit_without_stream,omitempty"`
}

// GRPCClientSettings defines common settings for a gRPC client configuration.
type GRPCClientSettings struct {
	// The target to which the exporter is going to send traces or metrics,
	// using the gRPC protocol. The valid syntax is described at
	// https://github.com/grpc/grpc/blob/master/doc/naming.md.
	Endpoint string `mapstructure:"endpoint"`

	// The compression key for supported compression types within
	// collector. Currently the only supported mode is `gzip`.
	Compression string `mapstructure:"compression"`

	// TLSSetting struct exposes TLS client configuration.
	TLSSetting configtls.TLSClientSetting `mapstructure:",squash"`

	// The keepalive parameters for gRPC client. See grpc.WithKeepaliveParams.
	// (https://godoc.org/google.golang.org/grpc#WithKeepaliveParams).
	Keepalive *KeepaliveClientConfig `mapstructure:"keepalive"`

	// ReadBufferSize for gRPC client. See grpc.WithReadBufferSize.
	// (https://godoc.org/google.golang.org/grpc#WithReadBufferSize).
	ReadBufferSize int `mapstructure:"read_buffer_size"`

	// WriteBufferSize for gRPC gRPC. See grpc.WithWriteBufferSize.
	// (https://godoc.org/google.golang.org/grpc#WithWriteBufferSize).
	WriteBufferSize int `mapstructure:"write_buffer_size"`

	// WaitForReady parameter configures client to wait for ready state before sending data.
	// (https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md)
	WaitForReady bool `mapstructure:"wait_for_ready"`

	// The headers associated with gRPC requests.
	Headers map[string]string `mapstructure:"headers"`

	// Sets the balancer in grpclb_policy to discover the servers. Default is pick_first.
	// https://github.com/grpc/grpc-go/blob/master/examples/features/load_balancing/README.md
	BalancerName string `mapstructure:"balancer_name"`

	// Auth configuration for outgoing RPCs.
	Auth *configauth.Authentication `mapstructure:"auth,omitempty"`
}

// KeepaliveServerConfig is the configuration for keepalive.
type KeepaliveServerConfig struct {
	ServerParameters  *KeepaliveServerParameters  `mapstructure:"server_parameters,omitempty"`
	EnforcementPolicy *KeepaliveEnforcementPolicy `mapstructure:"enforcement_policy,omitempty"`
}

// KeepaliveServerParameters allow configuration of the keepalive.ServerParameters.
// The same default values as keepalive.ServerParameters are applicable and get applied by the server.
// See https://godoc.org/google.golang.org/grpc/keepalive#ServerParameters for details.
type KeepaliveServerParameters struct {
	MaxConnectionIdle     time.Duration `mapstructure:"max_connection_idle,omitempty"`
	MaxConnectionAge      time.Duration `mapstructure:"max_connection_age,omitempty"`
	MaxConnectionAgeGrace time.Duration `mapstructure:"max_connection_age_grace,omitempty"`
	Time                  time.Duration `mapstructure:"time,omitempty"`
	Timeout               time.Duration `mapstructure:"timeout,omitempty"`
}

// KeepaliveEnforcementPolicy allow configuration of the keepalive.EnforcementPolicy.
// The same default values as keepalive.EnforcementPolicy are applicable and get applied by the server.
// See https://godoc.org/google.golang.org/grpc/keepalive#EnforcementPolicy for details.
type KeepaliveEnforcementPolicy struct {
	MinTime             time.Duration `mapstructure:"min_time,omitempty"`
	PermitWithoutStream bool          `mapstructure:"permit_without_stream,omitempty"`
}

// GRPCServerSettings defines common settings for a gRPC server configuration.
type GRPCServerSettings struct {
	// Server net.Addr config. For transport only "tcp" and "unix" are valid options.
	NetAddr confignet.NetAddr `mapstructure:",squash"`

	// Configures the protocol to use TLS.
	// The default value is nil, which will cause the protocol to not use TLS.
	TLSSetting *configtls.TLSServerSetting `mapstructure:"tls_settings,omitempty"`

	// MaxRecvMsgSizeMiB sets the maximum size (in MiB) of messages accepted by the server.
	MaxRecvMsgSizeMiB uint64 `mapstructure:"max_recv_msg_size_mib"`

	// MaxConcurrentStreams sets the limit on the number of concurrent streams to each ServerTransport.
	// It has effect only for streaming RPCs.
	MaxConcurrentStreams uint32 `mapstructure:"max_concurrent_streams"`

	// ReadBufferSize for gRPC server. See grpc.ReadBufferSize.
	// (https://godoc.org/google.golang.org/grpc#ReadBufferSize).
	ReadBufferSize int `mapstructure:"read_buffer_size"`

	// WriteBufferSize for gRPC server. See grpc.WriteBufferSize.
	// (https://godoc.org/google.golang.org/grpc#WriteBufferSize).
	WriteBufferSize int `mapstructure:"write_buffer_size"`

	// Keepalive anchor for all the settings related to keepalive.
	Keepalive *KeepaliveServerConfig `mapstructure:"keepalive,omitempty"`

	// Auth for this receiver
	Auth *configauth.Authentication `mapstructure:"auth,omitempty"`
}

// SanitizedEndpoint strips the prefix of either http:// or https:// from configgrpc.GRPCClientSettings.Endpoint.
func (gcs *GRPCClientSettings) SanitizedEndpoint() string {
	switch {
	case gcs.isSchemeHTTP():
		return strings.TrimPrefix(gcs.Endpoint, "http://")
	case gcs.isSchemeHTTPS():
		return strings.TrimPrefix(gcs.Endpoint, "https://")
	default:
		return gcs.Endpoint
	}
}

func (gcs *GRPCClientSettings) isSchemeHTTP() bool {
	return strings.HasPrefix(gcs.Endpoint, "http://")
}

func (gcs *GRPCClientSettings) isSchemeHTTPS() bool {
	return strings.HasPrefix(gcs.Endpoint, "https://")
}

// ToDialOptions maps configgrpc.GRPCClientSettings to a slice of dial options for gRPC.
func (gcs *GRPCClientSettings) ToDialOptions(ext map[config.ComponentID]component.Extension) ([]grpc.DialOption, error) {
	var opts []grpc.DialOption
	if gcs.Compression != "" {
		if compressionKey := GetGRPCCompressionKey(gcs.Compression); compressionKey != CompressionUnsupported {
			opts = append(opts, grpc.WithDefaultCallOptions(grpc.UseCompressor(compressionKey)))
		} else {
			return nil, fmt.Errorf("unsupported compression type %q", gcs.Compression)
		}
	}

	tlsCfg, err := gcs.TLSSetting.LoadTLSConfig()
	if err != nil {
		return nil, err
	}
	tlsDialOption := grpc.WithInsecure()
	if tlsCfg != nil {
		tlsDialOption = grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))
	} else if gcs.isSchemeHTTPS() {
		tlsDialOption = grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))
	}
	opts = append(opts, tlsDialOption)

	if gcs.ReadBufferSize > 0 {
		opts = append(opts, grpc.WithReadBufferSize(gcs.ReadBufferSize))
	}

	if gcs.WriteBufferSize > 0 {
		opts = append(opts, grpc.WithWriteBufferSize(gcs.WriteBufferSize))
	}

	if gcs.Keepalive != nil {
		keepAliveOption := grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time:                gcs.Keepalive.Time,
			Timeout:             gcs.Keepalive.Timeout,
			PermitWithoutStream: gcs.Keepalive.PermitWithoutStream,
		})
		opts = append(opts, keepAliveOption)
	}

	if gcs.Auth != nil {
		if ext == nil {
			return nil, fmt.Errorf("no extensions configuration available")
		}

		componentID, cperr := config.NewIDFromString(gcs.Auth.AuthenticatorName)
		if cperr != nil {
			return nil, cperr
		}

		grpcAuthenticator, cerr := configauth.GetGRPCClientAuthenticator(ext, componentID)
		if cerr != nil {
			return nil, cerr
		}

		perRPCCredentials, perr := grpcAuthenticator.PerRPCCredentials()
		if perr != nil {
			return nil, err
		}
		opts = append(opts, grpc.WithPerRPCCredentials(perRPCCredentials))
	}

	if gcs.BalancerName != "" {
		valid := validateBalancerName(gcs.BalancerName)
		if !valid {
			return nil, fmt.Errorf("invalid balancer_name: %s", gcs.BalancerName)
		}
		opts = append(opts, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingPolicy":"%s"}`, gcs.BalancerName)))
	}

	// Enable OpenTelemetry observability plugin.
	opts = append(opts, grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()))
	opts = append(opts, grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()))

	return opts, nil
}

func validateBalancerName(balancerName string) bool {
	for _, item := range allowedBalancerNames {
		if item == balancerName {
			return true
		}
	}
	return false
}

// ToListener returns the net.Listener constructed from the settings.
func (gss *GRPCServerSettings) ToListener() (net.Listener, error) {
	return gss.NetAddr.Listen()
}

// ToServerOption maps configgrpc.GRPCServerSettings to a slice of server options for gRPC.
func (gss *GRPCServerSettings) ToServerOption(ext map[config.ComponentID]component.Extension) ([]grpc.ServerOption, error) {
	var opts []grpc.ServerOption

	if gss.TLSSetting != nil {
		tlsCfg, err := gss.TLSSetting.LoadTLSConfig()
		if err != nil {
			return nil, err
		}
		opts = append(opts, grpc.Creds(credentials.NewTLS(tlsCfg)))
	}

	if gss.MaxRecvMsgSizeMiB > 0 {
		opts = append(opts, grpc.MaxRecvMsgSize(int(gss.MaxRecvMsgSizeMiB*1024*1024)))
	}

	if gss.MaxConcurrentStreams > 0 {
		opts = append(opts, grpc.MaxConcurrentStreams(gss.MaxConcurrentStreams))
	}

	if gss.ReadBufferSize > 0 {
		opts = append(opts, grpc.ReadBufferSize(gss.ReadBufferSize))
	}

	if gss.WriteBufferSize > 0 {
		opts = append(opts, grpc.WriteBufferSize(gss.WriteBufferSize))
	}

	// The default values referenced in the GRPC docs are set within the server, so this code doesn't need
	// to apply them over zero/nil values before passing these as grpc.ServerOptions.
	// The following shows the server code for applying default grpc.ServerOptions.
	// https://github.com/grpc/grpc-go/blob/120728e1f775e40a2a764341939b78d666b08260/internal/transport/http2_server.go#L184-L200
	if gss.Keepalive != nil {
		if gss.Keepalive.ServerParameters != nil {
			svrParams := gss.Keepalive.ServerParameters
			opts = append(opts, grpc.KeepaliveParams(keepalive.ServerParameters{
				MaxConnectionIdle:     svrParams.MaxConnectionIdle,
				MaxConnectionAge:      svrParams.MaxConnectionAge,
				MaxConnectionAgeGrace: svrParams.MaxConnectionAgeGrace,
				Time:                  svrParams.Time,
				Timeout:               svrParams.Timeout,
			}))
		}
		// The default values referenced in the GRPC are set within the server, so this code doesn't need
		// to apply them over zero/nil values before passing these as grpc.ServerOptions.
		// The following shows the server code for applying default grpc.ServerOptions.
		// https://github.com/grpc/grpc-go/blob/120728e1f775e40a2a764341939b78d666b08260/internal/transport/http2_server.go#L202-L205
		if gss.Keepalive.EnforcementPolicy != nil {
			enfPol := gss.Keepalive.EnforcementPolicy
			opts = append(opts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
				MinTime:             enfPol.MinTime,
				PermitWithoutStream: enfPol.PermitWithoutStream,
			}))
		}
	}

	if gss.Auth != nil {
		componentID, cperr := config.NewIDFromString(gss.Auth.AuthenticatorName)
		if cperr != nil {
			return nil, cperr
		}

		authenticator, err := configauth.GetServerAuthenticator(ext, componentID)
		if err != nil {
			return nil, err
		}

		opts = append(opts,
			grpc.UnaryInterceptor(authenticator.GRPCUnaryServerInterceptor),
			grpc.StreamInterceptor(authenticator.GRPCStreamServerInterceptor),
		)
	}

	// Enable OpenTelemetry observability plugin.
	// TODO: Pass construct settings to have access to Tracer.
	opts = append(opts, grpc.UnaryInterceptor(
		otelgrpc.UnaryServerInterceptor(
			otelgrpc.WithTracerProvider(otel.GetTracerProvider()),
			otelgrpc.WithPropagators(otel.GetTextMapPropagator()),
		)))
	opts = append(opts, grpc.StreamInterceptor(
		otelgrpc.StreamServerInterceptor(
			otelgrpc.WithTracerProvider(otel.GetTracerProvider()),
			otelgrpc.WithPropagators(otel.GetTextMapPropagator()),
		)))

	return opts, nil
}

// GetGRPCCompressionKey returns the grpc registered compression key if the
// passed in compression key is supported, and CompressionUnsupported otherwise.
func GetGRPCCompressionKey(compressionType string) string {
	compressionKey := strings.ToLower(compressionType)
	if encodingKey, ok := gRPCCompressionKeyMap[compressionKey]; ok {
		return encodingKey
	}
	return CompressionUnsupported
}
<|endoftext|>


<commit_before>// package fsrepo
//
// TODO explain the package roadmap...
//
//   .go-ipfs/
//   ├── client/
//   |   ├── client.lock          <------ protects client/ + signals its own pid
//   │   ├── ipfs-client.cpuprof
//   │   ├── ipfs-client.memprof
//   │   └── logs/
//   ├── config
//   ├── daemon/
//   │   ├── daemon.lock          <------ protects daemon/ + signals its own address
//   │   ├── ipfs-daemon.cpuprof
//   │   ├── ipfs-daemon.memprof
//   │   └── logs/
//   ├── datastore/
//   ├── repo.lock                <------ protects datastore/ and config
//   └── version
package fsrepo
<commit_msg>doc todo<commit_after>// package fsrepo
//
// TODO explain the package roadmap...
//
//   .go-ipfs/
//   ├── client/
//   |   ├── client.lock          <------ protects client/ + signals its own pid
//   │   ├── ipfs-client.cpuprof
//   │   ├── ipfs-client.memprof
//   │   └── logs/
//   ├── config
//   ├── daemon/
//   │   ├── daemon.lock          <------ protects daemon/ + signals its own address
//   │   ├── ipfs-daemon.cpuprof
//   │   ├── ipfs-daemon.memprof
//   │   └── logs/
//   ├── datastore/
//   ├── repo.lock                <------ protects datastore/ and config
//   └── version
package fsrepo

// TODO prevent multiple daemons from running
<|endoftext|>


<commit_before>package repository

import (
	"encoding/json"
	"path"
	"path/filepath"
	"strings"

	"citadelapp.io/citadel"
	"github.com/coreos/go-etcd/etcd"
)

type etcdRepository struct {
	client *etcd.Client
}

func NewEtcdRepository(machines []string, sync bool) Repository {
	r := &etcdRepository{
		client: etcd.NewClient(machines),
	}
	if sync {
		r.client.SyncCluster()
	}
	return r
}

// name == /local
// translatted to /citadel/services/local/services
func (e *etcdRepository) FetchServices(name string) ([]*citadel.ServiceData, error) {
	out := []*citadel.ServiceData{}

	resp, err := e.client.Get(path.Join("/citadel/services", buildServiceName(name, "services")), true, true)
	if err != nil {
		if isNotFoundErr(err) {
			return out, nil
		}
		return nil, err
	}

	for _, n := range resp.Node.Nodes {
		for _, sdir := range n.Nodes {
			if sdir.Key == path.Join(n.Key, "config") {
				var s *citadel.ServiceData
				if err := e.unmarshal(sdir.Value, &s); err != nil {
					return nil, err
				}
				out = append(out, s)
			}
		}
	}

	return out, nil
}

func (e *etcdRepository) FetchService(name string) (*citadel.ServiceData, error) {
	resp, err := e.client.Get(path.Join("/citadel/services", buildServiceName(name, "config")), true, true)
	if err != nil {
		return nil, err
	}

	var s *citadel.ServiceData
	if err := e.unmarshal(resp.Node.Value, &s); err != nil {
		return nil, err
	}
	return s, nil
}

// name == local/redis
// translatted to /citadel/services/local/services/redis/config
func (e *etcdRepository) SaveService(name string, s *citadel.ServiceData) error {
	data, err := e.marshal(s)
	if err != nil {
		return err
	}

	_, err = e.client.Set(path.Join("/citadel/services", buildServiceName(name, "config")), data, 0)
	return err
}

func (e *etcdRepository) FetchConfig() (*citadel.Config, error) {
	resp, err := e.client.Get("/citadel/config", false, false)
	if err != nil {
		return nil, err
	}
	var c *citadel.Config
	if err := e.unmarshal(resp.Node.Value, &c); err != nil {
		return nil, err
	}
	return c, nil
}

// marshal encodes the value into a string via the json encoder
func (e *etcdRepository) marshal(v interface{}) (string, error) {
	data, err := json.Marshal(v)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

// unmarshal decodes the data using the json decoder into the value v
func (e *etcdRepository) unmarshal(data string, v interface{}) error {
	return json.Unmarshal([]byte(data), v)
}

// isNotFoundErr returns true if the error is of type Key Not Found
func isNotFoundErr(err error) bool {
	return strings.Contains(err.Error(), "Key not found")
}

func buildServiceName(fullPath, prefix string) string {
	if fullPath == "" || fullPath == "/" {
		return "/"
	}

	dir, name := filepath.Split(fullPath)

	var (
		parts = strings.Split(dir, "/")
		full  = []string{}
	)

	switch len(parts) {
	case 0:
		return path.Join(name, prefix)
	}

	for _, p := range parts {
		if p != "" {
			full = append(full, p, "services")
		}
	}

	return path.Join(append(full, name, prefix)...)
}
<commit_msg>Update servcies repo<commit_after>package repository

import (
	"encoding/json"
	"path"
	"path/filepath"
	"strings"

	"citadelapp.io/citadel"
	"github.com/coreos/go-etcd/etcd"
)

type etcdRepository struct {
	client *etcd.Client
}

func NewEtcdRepository(machines []string, sync bool) Repository {
	r := &etcdRepository{
		client: etcd.NewClient(machines),
	}
	if sync {
		r.client.SyncCluster()
	}
	return r
}

// name == /local
// translatted to /citadel/services/local/services
func (e *etcdRepository) FetchServices(name string) ([]*citadel.ServiceData, error) {
	out := []*citadel.ServiceData{}

	resp, err := e.client.Get(path.Join("/citadel/services", buildServiceName(name, "services")), true, true)
	if err != nil {
		if isNotFoundErr(err) {
			return out, nil
		}
		return nil, err
	}

	if err := e.createServiceData(resp.Node.Nodes, &out); err != nil {
		return nil, err
	}

	return out, nil
}

func (e *etcdRepository) createServiceData(nodes etcd.Nodes, out *[]*citadel.ServiceData) error {
	for _, n := range nodes {
		for _, sdir := range n.Nodes {
			_, name := filepath.Split(sdir.Key)

			switch name {
			case "config":
				var s *citadel.ServiceData
				if err := e.unmarshal(sdir.Value, &s); err != nil {
					return err
				}
				s.Name = strings.TrimSuffix(strings.TrimPrefix(sdir.Key, "/citadel/services"), "/config")
				*out = append(*out, s)
			case "services":
				if err := e.createServiceData(sdir.Nodes, out); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (e *etcdRepository) FetchService(name string) (*citadel.ServiceData, error) {
	resp, err := e.client.Get(path.Join("/citadel/services", buildServiceName(name, "config")), true, true)
	if err != nil {
		return nil, err
	}

	var s *citadel.ServiceData
	if err := e.unmarshal(resp.Node.Value, &s); err != nil {
		return nil, err
	}
	return s, nil
}

// name == local/redis
// translatted to /citadel/services/local/services/redis/config
func (e *etcdRepository) SaveService(name string, s *citadel.ServiceData) error {
	data, err := e.marshal(s)
	if err != nil {
		return err
	}

	_, err = e.client.Set(path.Join("/citadel/services", buildServiceName(name, "config")), data, 0)
	return err
}

func (e *etcdRepository) FetchConfig() (*citadel.Config, error) {
	resp, err := e.client.Get("/citadel/config", false, false)
	if err != nil {
		return nil, err
	}
	var c *citadel.Config
	if err := e.unmarshal(resp.Node.Value, &c); err != nil {
		return nil, err
	}
	return c, nil
}

// marshal encodes the value into a string via the json encoder
func (e *etcdRepository) marshal(v interface{}) (string, error) {
	data, err := json.Marshal(v)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

// unmarshal decodes the data using the json decoder into the value v
func (e *etcdRepository) unmarshal(data string, v interface{}) error {
	return json.Unmarshal([]byte(data), v)
}

// isNotFoundErr returns true if the error is of type Key Not Found
func isNotFoundErr(err error) bool {
	return strings.Contains(err.Error(), "Key not found")
}

func buildServiceName(fullPath, prefix string) string {
	if fullPath == "" || fullPath == "/" {
		return "/"
	}

	dir, name := filepath.Split(fullPath)

	var (
		parts = strings.Split(dir, "/")
		full  = []string{}
	)

	switch len(parts) {
	case 0:
		return path.Join(name, prefix)
	}

	for _, p := range parts {
		if p != "" {
			full = append(full, p, "services")
		}
	}

	return path.Join(append(full, name, prefix)...)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/fatih/goset"
	"sync"
)

type KiteDependency struct {
	r map[string]*goset.Set
	sync.RWMutex
}

func NewDependency() *KiteDependency {
	return &KiteDependency{
		r: make(map[string]*goset.Set),
	}
}

// Add relationsips to kite
func (k *KiteDependency) Add(source, target string) {
	if target == "" || source == "" {
		return
	}

	k.RLock()
	s := k.r[source]
	k.RUnlock()

	if s == nil {
		s = goset.New()
	}

	k.Lock()
	s.Add(target)
	k.r[source] = s
	k.Unlock()
}

func (k *KiteDependency) Remove(source string) {
	if source == "" {
		return
	}

	k.RLock()
	s := k.r[source]
	k.RUnlock()

	if s == nil {
		s = goset.New()
	}

	k.Lock()
	s.Clear()
	k.r[source] = s
	k.Unlock()
}

func (k *KiteDependency) Has(source string) bool {
	if source == "" {
		return false
	}

	k.RLock()
	s, ok := k.r[source]
	k.RUnlock()
	if !ok {
		return false
	}

	if s == nil {
		return false
	}

	return true
}

// ListRelationship returns a slice of kite names that depends on "source"
// It returns an empty slice if the kite doesn't have any relationships.
func (k *KiteDependency) List(source string) []string {
	k.RLock()
	defer k.RUnlock()
	s, ok := k.r[source]
	if !ok {
		return make([]string, 0)
	}

	return s.StringSlice()
}
<commit_msg>kites/dependencies: simplify our dependency package<commit_after>package main

import (
	"github.com/fatih/goset"
	"sync"
)

type KiteDependency struct {
	r map[string]*goset.Set
	sync.Mutex
}

func NewDependency() *KiteDependency {
	return &KiteDependency{
		r: make(map[string]*goset.Set),
	}
}

// Add relationsips to kite
func (k *KiteDependency) Add(source, target string) {
	if target == "" || source == "" {
		return
	}

	k.Lock()
	defer k.Unlock()

	if k.r[source] == nil {
		k.r[source] = goset.New()
	}

	k.r[source].Add(target)
}

func (k *KiteDependency) Remove(source string) {
	if source == "" {
		return
	}

	k.Lock()
	defer k.Unlock()

	if k.r[source] == nil {
		return
	}

	k.r[source].Clear()
}

func (k *KiteDependency) Has(source string) bool {
	if source == "" {
		return false
	}
	k.Lock()
	defer k.Unlock()

	s, ok := k.r[source]
	if !ok {
		return false
	}

	if s == nil {
		return false
	}

	return true
}

// ListRelationship returns a slice of kite names that depends on "source"
// It returns an empty slice if the kite doesn't have any relationships.
func (k *KiteDependency) List(source string) []string {
	k.Lock()
	defer k.Unlock()

	s, ok := k.r[source]
	if !ok {
		return make([]string, 0)
	}

	return s.StringSlice()
}
<|endoftext|>


<commit_before>package stripe

import "encoding/json"

// TaxIDType is the list of allowed values for the tax id's type..
type TaxIDType string

// List of values that TaxIDType can take.
const (
	TaxIDTypeAUABN   TaxIDType = "au_abn"
	TaxIDTypeCABN    TaxIDType = "ca_bn"
	TaxIDTypeCHVAT   TaxIDType = "ch_vat"
	TaxIDTypeESCIF   TaxIDType = "es_cif"
	TaxIDTypeEUVAT   TaxIDType = "eu_vat"
	TaxIDTypeHKBR    TaxIDType = "hk_br"
	TaxIDTypeINGST   TaxIDType = "in_gst"
	TaxIDTypeMXRFC   TaxIDType = "mx_rfc"
	TaxIDTypeNOVAT   TaxIDType = "no_vat"
	TaxIDTypeNZGST   TaxIDType = "nz_gst"
	TaxIDTypeRUINN   TaxIDType = "ru_inn"
	TaxIDTypeSGUEN   TaxIDType = "sg_uen"
	TaxIDTypeZAVAT   TaxIDType = "za_vat"
	TaxIDTypeUnknown TaxIDType = "unknown"
)

// TaxIDVerificationStatus is the list of allowed values for the tax id's verification status..
type TaxIDVerificationStatus string

// List of values that TaxIDDuration can take.
const (
	TaxIDVerificationStatusPending     TaxIDVerificationStatus = "pending"
	TaxIDVerificationStatusUnavailable TaxIDVerificationStatus = "unavailable"
	TaxIDVerificationStatusUnverified  TaxIDVerificationStatus = "unverified"
	TaxIDVerificationStatusVerified    TaxIDVerificationStatus = "verified"
)

// TaxIDParams is the set of parameters that can be used when creating a tax id.
// For more details see https://stripe.com/docs/api/customers/create_tax_id
type TaxIDParams struct {
	Params   `form:"*"`
	Customer *string `form:"-"`
	Type     *string `form:"type"`
	Value    *string `form:"value"`
}

// TaxIDListParams is the set of parameters that can be used when listing tax ids.
// For more detail see https://stripe.com/docs/api/customers/tax_ids
type TaxIDListParams struct {
	ListParams `form:"*"`
	Customer   *string `form:"-"`
}

// TaxIDVerification represents the verification details of a customer's tax id.
type TaxIDVerification struct {
	Status          TaxIDVerificationStatus `json:"status"`
	VerifiedAddress string                  `json:"verified_address"`
	VerifiedName    string                  `json:"verified_name"`
}

// TaxID is the resource representing a customer's tax id.
// For more details see https://stripe.com/docs/api/customers/tax_id_object
type TaxID struct {
	Country      string             `json:"country"`
	Created      int64              `json:"created"`
	Customer     *Customer          `json:"customer"`
	Deleted      bool               `json:"deleted"`
	ID           string             `json:"id"`
	Livemode     bool               `json:"livemode"`
	Object       string             `json:"object"`
	Type         TaxIDType          `json:"type"`
	Value        string             `json:"value"`
	Verification *TaxIDVerification `json:"verification"`
}

// TaxIDList is a list of tax ids as retrieved from a list endpoint.
type TaxIDList struct {
	ListMeta
	Data []*TaxID `json:"data"`
}

// UnmarshalJSON handles deserialization of a TaxID.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (c *TaxID) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		c.ID = id
		return nil
	}

	type taxid TaxID
	var v taxid
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*c = TaxID(v)
	return nil
}
<commit_msg>Add support for TaxIDTypeTHVAT and TaxIDTypeTWVAT on TaxId<commit_after>package stripe

import "encoding/json"

// TaxIDType is the list of allowed values for the tax id's type..
type TaxIDType string

// List of values that TaxIDType can take.
const (
	TaxIDTypeAUABN   TaxIDType = "au_abn"
	TaxIDTypeCABN    TaxIDType = "ca_bn"
	TaxIDTypeCHVAT   TaxIDType = "ch_vat"
	TaxIDTypeESCIF   TaxIDType = "es_cif"
	TaxIDTypeEUVAT   TaxIDType = "eu_vat"
	TaxIDTypeHKBR    TaxIDType = "hk_br"
	TaxIDTypeINGST   TaxIDType = "in_gst"
	TaxIDTypeMXRFC   TaxIDType = "mx_rfc"
	TaxIDTypeNOVAT   TaxIDType = "no_vat"
	TaxIDTypeNZGST   TaxIDType = "nz_gst"
	TaxIDTypeRUINN   TaxIDType = "ru_inn"
	TaxIDTypeSGUEN   TaxIDType = "sg_uen"
	TaxIDTypeTHVAT   TaxIDType = "th_vat"
	TaxIDTypeTWVAT   TaxIDType = "tw_vat"
	TaxIDTypeZAVAT   TaxIDType = "za_vat"
	TaxIDTypeUnknown TaxIDType = "unknown"
)

// TaxIDVerificationStatus is the list of allowed values for the tax id's verification status..
type TaxIDVerificationStatus string

// List of values that TaxIDDuration can take.
const (
	TaxIDVerificationStatusPending     TaxIDVerificationStatus = "pending"
	TaxIDVerificationStatusUnavailable TaxIDVerificationStatus = "unavailable"
	TaxIDVerificationStatusUnverified  TaxIDVerificationStatus = "unverified"
	TaxIDVerificationStatusVerified    TaxIDVerificationStatus = "verified"
)

// TaxIDParams is the set of parameters that can be used when creating a tax id.
// For more details see https://stripe.com/docs/api/customers/create_tax_id
type TaxIDParams struct {
	Params   `form:"*"`
	Customer *string `form:"-"`
	Type     *string `form:"type"`
	Value    *string `form:"value"`
}

// TaxIDListParams is the set of parameters that can be used when listing tax ids.
// For more detail see https://stripe.com/docs/api/customers/tax_ids
type TaxIDListParams struct {
	ListParams `form:"*"`
	Customer   *string `form:"-"`
}

// TaxIDVerification represents the verification details of a customer's tax id.
type TaxIDVerification struct {
	Status          TaxIDVerificationStatus `json:"status"`
	VerifiedAddress string                  `json:"verified_address"`
	VerifiedName    string                  `json:"verified_name"`
}

// TaxID is the resource representing a customer's tax id.
// For more details see https://stripe.com/docs/api/customers/tax_id_object
type TaxID struct {
	Country      string             `json:"country"`
	Created      int64              `json:"created"`
	Customer     *Customer          `json:"customer"`
	Deleted      bool               `json:"deleted"`
	ID           string             `json:"id"`
	Livemode     bool               `json:"livemode"`
	Object       string             `json:"object"`
	Type         TaxIDType          `json:"type"`
	Value        string             `json:"value"`
	Verification *TaxIDVerification `json:"verification"`
}

// TaxIDList is a list of tax ids as retrieved from a list endpoint.
type TaxIDList struct {
	ListMeta
	Data []*TaxID `json:"data"`
}

// UnmarshalJSON handles deserialization of a TaxID.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (c *TaxID) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		c.ID = id
		return nil
	}

	type taxid TaxID
	var v taxid
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*c = TaxID(v)
	return nil
}
<|endoftext|>