<commit_before>/* syncwatcher

   This package is a recursive wrapper for fsnotify.
   The interface is intended to be compatible with fsnotify.

   When a directory is "Watch"ed, so are all its subdirectories

   When a watched directory is moved, within, into, or out of, another watched
   directory, it is unwatched and (re)watched, as appropriate. As a special
   case, each root directory (as passed to "Watch"), is never unwatched, even
   if deleted or moved.

   WARNING: when a directory is moved there is a brief period in which other
   events inside that directory may be missed. You should assume that anything
   may have happened in that time.
*/

package main

import (
	"errors"
	"github.com/go-fsnotify/fsnotify"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

type SyncWatcher struct {
	Error chan error
	Event chan fsnotify.Event

	watcher        *fsnotify.Watcher
	paths          map[string]string
	mainPath       string
	ignorePaths    []string
	ignorePatterns []Pattern
	roots          map[string]int
	pathMutex      *sync.Mutex
}

func NewSyncWatcher(mainPath string, ignorePaths []string, ignorePatterns []Pattern) (*SyncWatcher, error) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	sw := &SyncWatcher{
		make(chan error),
		make(chan fsnotify.Event),
		watcher,
		make(map[string]string),
		mainPath,
		ignorePaths,
		ignorePatterns,
		make(map[string]int),
		&sync.Mutex{},
	}

	// Handle events from fsnotify, deal with them
	// and forward the interesting ones to the caller
	go func() {
		// Loop until both incoming channels are closed
		for openEvent, openErr := true, true; openEvent || openErr; {
			select {
			case ev, openEvent := <-watcher.Events:
				if openEvent {
					// Add or remove watches as appropriate
					sw.pathMutex.Lock()
					_, present := sw.paths[ev.Name]
					sw.pathMutex.Unlock()
					if present {
						// If we recognise the path then it must be a directory
						// that means its changed, and the old watches must be
						// removed.  New watches will be added when the corresponding
						// "create" event arrives.
						// This uses "removeWatch" not "RemoveWatch" on purpose
						sw.removeWatch(ev.Name)
					} else if info, err := os.Lstat(ev.Name); err == nil && info.IsDir() {
						// A new, unrecognised directory was created.
						sw.watch(ev.Name)
					}

					// Forward the event to our client.
					sw.Event <- ev
				}
			case err, openErr = <-watcher.Errors:
				if openErr {
					// Forward error events to our client
					sw.Error <- err
				}
			}
		}
		// If we get here then the incoming channels are closed,
		// so close the outgoing channels.
		close(sw.Event)
		close(sw.Error)
	}()

	return sw, nil
}

func (w *SyncWatcher) Close() error {
	// We close the fsnotify watcher.
	// That will close our incoming channels, and so close the SyncWatcher
	// indirectly.
	err := w.watcher.Close()
	if err != nil {
		return err
	}
	return nil
}

// This is like RemoveWatch except that it does not unwatch the root directory.
func (w *SyncWatcher) removeWatch(path string) error {
	w.pathMutex.Lock()
	defer w.pathMutex.Unlock()

	// Recursively remove all the watches from the given directory, and its
	// subdirectories. The root directory will not be unwatched (RemoveWatch
	// takes care of that).
	var recursive_remove func(dir string) error
	recursive_remove = func(dir string) error {
		children, ok := w.paths[dir]
		if ok {
			for _, child := range strings.Split(children, "\000") {
				if len(child) > 0 {
					// deliberately ignore errors from child watches
					recursive_remove(filepath.Join(dir, child))
				}
			}
			if _, isroot := w.roots[dir]; !isroot {
				delete(w.paths, dir)
				return w.watcher.Remove(dir)
			}
		}
		return errors.New("cannot remove uknown watch: " + dir)
	}

	return recursive_remove(path)
}

func (w *SyncWatcher) RemoveWatch(path string) error {
	// We want to unwatch the whole tree, including to root.
	// If we unregister the root then removeWatch will take care of the rest.
	w.pathMutex.Lock()
	if _, isroot := w.roots[path]; isroot {
		delete(w.roots, path)
	}
	w.pathMutex.Unlock()
	return w.removeWatch(path)
}

func (w *SyncWatcher) watch(path string) error {
	w.pathMutex.Lock()
	defer w.pathMutex.Unlock()

	return filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
		if err == nil && info.IsDir() {
			if shouldIgnore(w.ignorePaths, w.ignorePatterns, relativePath(p, w.mainPath)) {
				Debug.Println("Ignoring: " + p)
				return filepath.SkipDir
			}
			err = w.watcher.Add(p)
			if err == nil {
				w.paths[p] = ""
				parent := filepath.Dir(p)
				if _, ok := w.paths[parent]; ok {
					// Record the directory structure so that it can be
					// walked again when we need to remove the watches.
					w.paths[parent] += filepath.Base(p) + "\000"
				}
			}
		}
		return err
	})
}

func (w *SyncWatcher) Watch(path string) error {
	w.pathMutex.Lock()
	_, present := w.paths[path]

	if present {
		w.pathMutex.Unlock()
		return errors.New("cannot watch path twice: " + path)
	}
	w.roots[path] = 1
	w.pathMutex.Unlock()

	return w.watch(path)
}

func (w *SyncWatcher) String() string {
	w.pathMutex.Lock()
	defer w.pathMutex.Unlock()

	str := "SyncWatch:"
	for path := range w.paths {
		str += " " + path + " \"" + w.paths[path] + "\""
	}
	return str
}
<commit_msg>Do not follow looping symlinks (fixes #19)<commit_after>/* syncwatcher

   This package is a recursive wrapper for fsnotify.
   The interface is intended to be compatible with fsnotify.

   When a directory is "Watch"ed, so are all its subdirectories

   When a watched directory is moved, within, into, or out of, another watched
   directory, it is unwatched and (re)watched, as appropriate. As a special
   case, each root directory (as passed to "Watch"), is never unwatched, even
   if deleted or moved.

   WARNING: when a directory is moved there is a brief period in which other
   events inside that directory may be missed. You should assume that anything
   may have happened in that time.
*/

package main

import (
	"errors"
	"github.com/go-fsnotify/fsnotify"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

type SyncWatcher struct {
	Error chan error
	Event chan fsnotify.Event

	watcher        *fsnotify.Watcher
	paths          map[string]string
	mainPath       string
	ignorePaths    []string
	ignorePatterns []Pattern
	roots          map[string]int
	pathMutex      *sync.Mutex
}

func NewSyncWatcher(mainPath string, ignorePaths []string, ignorePatterns []Pattern) (*SyncWatcher, error) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	sw := &SyncWatcher{
		make(chan error),
		make(chan fsnotify.Event),
		watcher,
		make(map[string]string),
		mainPath,
		ignorePaths,
		ignorePatterns,
		make(map[string]int),
		&sync.Mutex{},
	}

	// Handle events from fsnotify, deal with them
	// and forward the interesting ones to the caller
	go func() {
		// Loop until both incoming channels are closed
		for openEvent, openErr := true, true; openEvent || openErr; {
			select {
			case ev, openEvent := <-watcher.Events:
				if openEvent {
					// Add or remove watches as appropriate
					sw.pathMutex.Lock()
					_, present := sw.paths[ev.Name]
					sw.pathMutex.Unlock()
					if present {
						// If we recognise the path then it must be a directory
						// that means its changed, and the old watches must be
						// removed.  New watches will be added when the corresponding
						// "create" event arrives.
						// This uses "removeWatch" not "RemoveWatch" on purpose
						sw.removeWatch(ev.Name)
					} else if info, err := os.Lstat(ev.Name); err == nil && info.IsDir() {
						// A new, unrecognised directory was created.
						sw.watch(ev.Name)
					}

					// Forward the event to our client.
					sw.Event <- ev
				}
			case err, openErr = <-watcher.Errors:
				if openErr {
					// Forward error events to our client
					sw.Error <- err
				}
			}
		}
		// If we get here then the incoming channels are closed,
		// so close the outgoing channels.
		close(sw.Event)
		close(sw.Error)
	}()

	return sw, nil
}

func (w *SyncWatcher) Close() error {
	// We close the fsnotify watcher.
	// That will close our incoming channels, and so close the SyncWatcher
	// indirectly.
	err := w.watcher.Close()
	if err != nil {
		return err
	}
	return nil
}

// This is like RemoveWatch except that it does not unwatch the root directory.
func (w *SyncWatcher) removeWatch(path string) error {
	w.pathMutex.Lock()
	defer w.pathMutex.Unlock()

	// Recursively remove all the watches from the given directory, and its
	// subdirectories. The root directory will not be unwatched (RemoveWatch
	// takes care of that).
	var recursive_remove func(dir string) error
	recursive_remove = func(dir string) error {
		children, ok := w.paths[dir]
		if ok {
			for _, child := range strings.Split(children, "\000") {
				if len(child) > 0 {
					// deliberately ignore errors from child watches
					recursive_remove(filepath.Join(dir, child))
				}
			}
			if _, isroot := w.roots[dir]; !isroot {
				delete(w.paths, dir)
				return w.watcher.Remove(dir)
			}
		}
		return errors.New("cannot remove uknown watch: " + dir)
	}

	return recursive_remove(path)
}

func (w *SyncWatcher) RemoveWatch(path string) error {
	// We want to unwatch the whole tree, including to root.
	// If we unregister the root then removeWatch will take care of the rest.
	w.pathMutex.Lock()
	if _, isroot := w.roots[path]; isroot {
		delete(w.roots, path)
	}
	w.pathMutex.Unlock()
	return w.removeWatch(path)
}

func (w *SyncWatcher) watch(path string) error {
	w.pathMutex.Lock()
	defer w.pathMutex.Unlock()

	seen := make(map[string]bool) // Used to block recursive symlinks
	return filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
		if err == nil && info.IsDir() {
			if shouldIgnore(w.ignorePaths, w.ignorePatterns, relativePath(p, w.mainPath)) {
				Debug.Println("Ignoring: " + p)
				return filepath.SkipDir
			}
			sp, _ := filepath.EvalSymlinks(p)
			if _, ok := seen[sp]; ok {
				Debug.Println("Ignoring recursive loop: " + p)
				return filepath.SkipDir
			}
			seen[sp] = true
			err = w.watcher.Add(p)
			if err == nil {
				w.paths[p] = ""
				parent := filepath.Dir(p)
				if _, ok := w.paths[parent]; ok {
					// Record the directory structure so that it can be
					// walked again when we need to remove the watches.
					w.paths[parent] += filepath.Base(p) + "\000"
				}
			}
		}
		return err
	})
}

func (w *SyncWatcher) Watch(path string) error {
	w.pathMutex.Lock()
	_, present := w.paths[path]

	if present {
		w.pathMutex.Unlock()
		return errors.New("cannot watch path twice: " + path)
	}
	w.roots[path] = 1
	w.pathMutex.Unlock()

	return w.watch(path)
}

func (w *SyncWatcher) String() string {
	w.pathMutex.Lock()
	defer w.pathMutex.Unlock()

	str := "SyncWatch:"
	for path := range w.paths {
		str += " " + path + " \"" + w.paths[path] + "\""
	}
	return str
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"time"

	"github.com/admpub/log"
	"github.com/admpub/sockjs-go/sockjs"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/engine/fasthttp"
	"github.com/webx-top/echo/engine/standard"
	ws "github.com/webx-top/echo/handler/sockjs"
	mw "github.com/webx-top/echo/middleware"
)

var homeTemplate = template.Must(template.New("").Parse(`
<!DOCTYPE html>

Chat : <input id='chat' type='text' value='' size='20'/>
<button onclick="sent()">Sent</button>

`))

func main() {
	e := echo.New()
	e.Use(mw.Log())

	e.Get("/", func(c echo.Context) error {
		homeTemplate.Execute(c.Response(), map[string]string{
			"echo":   "http://" + c.Request().Host() + "/websocket",
			"sockjs": sockjs.DefaultOptions.SockJSURL,
			"notice": "http://" + c.Request().Host() + "/notice",
		})
		return nil
	})
	//e.Get("/websocket", ws.Websocket(nil))

	e.HandlerWrapper = ws.HanderWrapper

	options := ws.Options{
		Handle: func(c sockjs.Session) error {
			//push(writer)
			go func() {
				var counter int
				for {
					if counter >= 10 { //测试只推10条
						return
					}
					time.Sleep(5 * time.Second)
					message := time.Now().String()
					log.Info(`Push message: `, message)
					if err := c.Send(message); err != nil {
						log.Error(`Push error: `, err.Error())
						return
					}
					counter++
				}
			}()
			// */

			//echo
			ws.DefaultExecuter(c)
			return nil
		},
		Options: sockjs.DefaultOptions,
		Prefix:  "/websocket",
	}
	options.Wrapper(e)

	options.Handle = func(c sockjs.Session) error {
		for {
			time.Sleep(5 * time.Second)
			message := time.Now().String()
			log.Info(`Push message: `, message)
			if err := c.Send(message); err != nil {
				return err
			}
		}
		return nil
	}
	options.Prefix = "/notice"

	options.Wrapper(e)

	switch `` {
	case `fast`:

		// FastHTTP
		e.Run(fasthttp.New(":4444"))

	default:
		// Standard
		e.Run(standard.New(":4444"))
	}
}
<commit_msg>update<commit_after>package main

import (
	"html/template"
	"time"

	"github.com/admpub/log"
	"github.com/admpub/sockjs-go/sockjs"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/engine/fasthttp"
	"github.com/webx-top/echo/engine/standard"
	ws "github.com/webx-top/echo/handler/sockjs"
	mw "github.com/webx-top/echo/middleware"
)

var homeTemplate = template.Must(template.New("").Parse(`
<!DOCTYPE html>

Chat : <input id='chat' type='text' value='' size='20'/>
<button onclick="sent()">Sent</button>

`))

func main() {
	e := echo.New()
	e.Use(mw.Log())

	e.Get("/", func(c echo.Context) error {
		homeTemplate.Execute(c.Response(), map[string]string{
			"echo":   "http://" + c.Request().Host() + "/websocket",
			"sockjs": sockjs.DefaultOptions.SockJSURL,
			"notice": "http://" + c.Request().Host() + "/notice",
		})
		return nil
	})
	//e.Get("/websocket", ws.Websocket(nil))

	e.HandlerWrapper = ws.HanderWrapper

	options := ws.Options{
		Handle: func(c sockjs.Session) error {
			//push(writer)
			go func() {
				var counter int
				for {
					if counter >= 10 { //测试只推10条
						return
					}
					time.Sleep(5 * time.Second)
					message := time.Now().String()
					log.Info(`Push message: `, message)
					if err := c.Send(message); err != nil {
						log.Error(`Push error: `, err.Error())
						return
					}
					counter++
				}
			}()
			// */

			//echo
			ws.DefaultExecuter(c)
			return nil
		},
		Options: sockjs.DefaultOptions,
		Prefix:  "/websocket",
	}
	options.Wrapper(e)

	options.Handle = func(c sockjs.Session) error {
		for {
			time.Sleep(5 * time.Second)
			message := time.Now().String()
			log.Info(`Push message: `, message)
			if err := c.Send(message); err != nil {
				return err
			}
		}
		return nil
	}
	options.Prefix = "/notice"

	options.Wrapper(e)

	switch `` {
	case `fast`:
		panic(`Unimplemented`)
		// FastHTTP
		e.Run(fasthttp.New(":4444"))

	default:
		// Standard
		e.Run(standard.New(":4444"))
	}
}
<|endoftext|>


<commit_before>package cluster

import (
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/testutil"
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
)

func TestCluster_StartStop(t *testing.T) {
	a := assert.New(t)

	conf := Config{ID: 1, Host: "localhost", Port: 10000, Remotes: []string{"127.0.0.1:10000"}}
	cl := New(&conf)

	//start the cluster
	err := cl.Start()
	a.Nil(err, "No error should be raised when Starting the Cluster")

	//stop the cluster
	err = cl.Stop()
	a.Nil(err, "No error should be raised when Stopping the Cluster")
}

func TestCluster_BroadcastMessage(t *testing.T) {
	defer testutil.EnableDebugForMethod()
	a := assert.New(t)

	confNode1 := Config{ID: 1, Host: "localhost", Port: 10000, Remotes: []string{"127.0.0.1:10000"}}
	node1 := New(&confNode1)

	//start the cluster
	err := node1.Start()
	a.Nil(err, "No error should be raised when Starting the Cluster")

	confNode2 := Config{ID: 1, Host: "localhost", Port: 10001, Remotes: []string{"127.0.0.1:10000"}}
	node2 := New(&confNode2)
	//start the cluster
	err = node2.Start()
	a.Nil(err, "No error should be raised when Starting the Cluster")

	// Send a String Message
	str := "TEST"
	err = node1.BroadcastString(&str)
	a.Nil(err, "No error should be raised when sending to Cluster")

	// end a protocol message
	pmsg := protocol.Message{
		ID:            1,
		Path:          "/stuff",
		UserID:        "id",
		ApplicationID: "appId",
		Time:          time.Now().Unix(),
		HeaderJSON:    "{}",
		Body:          []byte("test"),
		NodeID:        1}
	err = node1.BroadcastMessage(&pmsg)
	a.Nil(err, "No error should be raised when sending procotocol message to Cluster")
}
<commit_msg>fixes in test<commit_after>package cluster

import (
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/testutil"
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
)

func TestCluster_StartStop(t *testing.T) {
	a := assert.New(t)

	conf := Config{ID: 1, Host: "localhost", Port: 10000, Remotes: []string{"127.0.0.1:10000"}}
	node := New(&conf)
	node.MessageHandler = DummyMessageHandler{}

	//start the cluster
	err := node.Start()
	a.Nil(err, "No error should be raised when Starting the Cluster")

	//stop the cluster
	err = node.Stop()
	a.Nil(err, "No error should be raised when Stopping the Cluster")
}

func TestCluster_BroadcastMessage(t *testing.T) {
	defer testutil.EnableDebugForMethod()
	a := assert.New(t)

	confNode1 := Config{ID: 1, Host: "localhost", Port: 10001, Remotes: []string{"127.0.0.1:10001"}}
	node1 := New(&confNode1)
	node1.MessageHandler = DummyMessageHandler{}

	//start the cluster node 1
	err := node1.Start()
	defer node1.Stop()
	a.Nil(err, "No error should be raised when Starting the Cluster (node 1)")

	confNode2 := Config{ID: 2, Host: "localhost", Port: 10002, Remotes: []string{"127.0.0.1:10002"}}
	node2 := New(&confNode2)
	node2.MessageHandler = DummyMessageHandler{}

	//start the cluster node 2
	err = node2.Start()
	defer node2.Stop()
	a.Nil(err, "No error should be raised when Starting the Cluster (node 2)")

	// Send a String Message
	str := "TEST"
	err = node1.BroadcastString(&str)
	a.Nil(err, "No error should be raised when sending a string to Cluster")

	// and a protocol message
	pmsg := protocol.Message{
		ID:            1,
		Path:          "/stuff",
		UserID:        "id",
		ApplicationID: "appId",
		Time:          time.Now().Unix(),
		HeaderJSON:    "{}",
		Body:          []byte("test"),
		NodeID:        1}
	err = node1.BroadcastMessage(&pmsg)
	a.Nil(err, "No error should be raised when sending a procotocol message to Cluster")
}

type DummyMessageHandler struct {
}

func (dmh DummyMessageHandler) HandleMessage(pmsg *protocol.Message) error {
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"encoding/json"

	"github.com/cosminrentea/gobbler/client"
	"github.com/cosminrentea/gobbler/server/configstring"
	"github.com/cosminrentea/gobbler/server/connector"
	"github.com/cosminrentea/gobbler/server/fcm"
	"github.com/cosminrentea/gobbler/server/service"
	"github.com/cosminrentea/gobbler/testutil"
	"github.com/stretchr/testify/assert"
)

var (
	testHttpPort         = 11000
	timeoutForOneMessage = 150 * time.Millisecond
)

type fcmMetricsMap struct {
	CurrentErrorsCount            int `json:"current_errors_count"`
	CurrentMessagesCount          int `json:"current_messages_count"`
	CurrentMessagesTotalLatencies int `json:"current_messages_total_latencies_nanos"`
	CurrentErrorsTotalLatencies   int `json:"current_errors_total_latencies_nanos"`
}

type fcmMetrics struct {
	TotalSentMessages      int           `json:"fcm.total_sent_messages"`
	TotalSentMessageErrors int           `json:"fcm.total_sent_message_errors"`
	Minute                 fcmMetricsMap `json:"fcm.minute"`
	Hour                   fcmMetricsMap `json:"fcm.hour"`
	Day                    fcmMetricsMap `json:"fcm.day"`
}

type routerMetrics struct {
	CurrentRoutes        int `json:"router.current_routes"`
	CurrentSubscriptions int `json:"router.current_subscriptions"`
}

type expectedValues struct {
	ZeroLatencies        bool
	MessageCount         int
	CurrentRoutes        int
	CurrentSubscriptions int
}

// Test that restarting the service continues to fetch messages from store for a subscription from lastID
func TestFCMRestart(t *testing.T) {

	defer testutil.SkipIfDisabled(t)
	//defer testutil.SkipIfShort(t)

	//defer testutil.EnableDebugForMethod()()
	defer testutil.ResetDefaultRegistryHealthCheck()
	a := assert.New(t)

	receiveC := make(chan bool)
	s, cleanup := serviceSetUp(t)
	defer cleanup()

	assertMetrics(a, s, expectedValues{true, 0, 0, 0})

	var fcmConn connector.ResponsiveConnector
	var ok bool
	for _, iface := range s.ModulesSortedByStartOrder() {
		fcmConn, ok = iface.(connector.ResponsiveConnector)
		if ok {
			break
		}
	}
	a.True(ok, "There should be a module of type FCMConnector")

	// add a high timeout so the messages are processed slow
	sender, err := fcm.CreateFcmSender(fcm.SuccessFCMResponse, receiveC, 10*time.Millisecond)
	a.NoError(err)
	fcmConn.SetSender(sender)

	// create subscription on topic
	subscriptionSetUp(t, s)
	time.Sleep(time.Second)
	assertMetrics(a, s, expectedValues{true, 0, 1, 1})

	c := clientSetUp(t, s)
	defer c.Close()

	// send 3 messages in the router but read only one and close the service
	for i := 0; i < 3; i++ {
		c.Send(testTopic, "dummy body", `{"Correlation-Id": "id"}`)
	}

	// receive one message only from FCM
	select {
	case <-receiveC:
	case <-time.After(timeoutForOneMessage):
		a.Fail("Initial FCM message not received")
	}

	time.Sleep(time.Second)

	assertMetrics(a, s, expectedValues{false, 1, 1, 1})
	close(receiveC)
	// restart the service
	a.NoError(s.Stop())

	// remake the sender
	receiveC = make(chan bool)
	sender, err = fcm.CreateFcmSender(fcm.SuccessFCMResponse, receiveC, 10*time.Millisecond)
	a.NoError(err)
	fcmConn.SetSender(sender)

	time.Sleep(50 * time.Millisecond)
	testutil.ResetDefaultRegistryHealthCheck()
	a.NoError(s.Start())

	//TODO Cosmin Bogdan add 2 calls to assertMetrics before and after the next block

	// read the other 2 messages
	for i := 0; i < 1; i++ {
		select {
		case <-receiveC:
		case <-time.After(2 * timeoutForOneMessage):
			a.Fail("FCM message not received")
		}
	}
}

func serviceSetUp(t *testing.T) (*service.Service, func()) {
	dir, errTempDir := ioutil.TempDir("", "guble_fcm_test")
	assert.NoError(t, errTempDir)

	*Config.KVS = "memory"
	*Config.MS = "file"
	*Config.Cluster.NodeID = 0
	*Config.StoragePath = dir
	*Config.MetricsEndpoint = "/admin/metrics-old"
	*Config.WS.Enabled = true
	*Config.WS.Prefix = "/stream/"
	*Config.FCM.Enabled = true
	*Config.FCM.APIKey = "WILL BE OVERWRITTEN"
	*Config.FCM.Prefix = "/fcm/"
	*Config.FCM.Workers = 1 // use only one worker so we can control the number of messages that go to FCM
	*Config.APNS.Enabled = false
	*Config.KafkaProducer.Brokers = configstring.List{}

	var s *service.Service
	for s == nil {
		testHttpPort++
		logger.WithField("port", testHttpPort).Debug("trying to use HTTP Port")
		*Config.HttpListen = fmt.Sprintf("127.0.0.1:%d", testHttpPort)
		s = StartService()
	}
	return s, func() {
		errRemove := os.RemoveAll(dir)
		if errRemove != nil {
			logger.WithError(errRemove).WithField("module", "testing").Error("Could not remove directory")
		}
	}
}

func clientSetUp(t *testing.T, service *service.Service) client.Client {
	wsURL := "ws://" + service.WebServer().GetAddr() + "/stream/user/user01"
	c, err := client.Open(wsURL, "http://localhost/", 1000, false)
	assert.NoError(t, err)
	return c
}

func subscriptionSetUp(t *testing.T, service *service.Service) {
	a := assert.New(t)

	urlFormat := fmt.Sprintf("http://%s/fcm/%%d/gcmId%%d/%%s", service.WebServer().GetAddr())
	// create GCM subscription
	response, errPost := http.Post(
		fmt.Sprintf(urlFormat, 1, 1, strings.TrimPrefix(testTopic, "/")),
		"text/plain",
		bytes.NewBufferString(""),
	)
	a.NoError(errPost)
	a.Equal(response.StatusCode, 200)

	body, errReadAll := ioutil.ReadAll(response.Body)
	a.NoError(errReadAll)
	a.Equal(fmt.Sprintf(`{"subscribed":"%s"}`, testTopic), string(body))
}

func assertMetrics(a *assert.Assertions, s *service.Service, expected expectedValues) {
	time.Sleep(50 * time.Millisecond)

	httpClient := &http.Client{}
	u := fmt.Sprintf("http://%s%s", s.WebServer().GetAddr(), defaultMetricsEndpoint)
	request, err := http.NewRequest(http.MethodGet, u, nil)
	a.NoError(err)

	response, err := httpClient.Do(request)
	a.NoError(err)
	defer response.Body.Close()

	a.Equal(http.StatusOK, response.StatusCode)
	bodyBytes, err := ioutil.ReadAll(response.Body)
	a.NoError(err)
	logger.WithField("body", string(bodyBytes)).Debug("metrics response")

	mFCM := &fcmMetrics{}
	err = json.Unmarshal(bodyBytes, mFCM)
	a.NoError(err)

	a.Equal(0, mFCM.TotalSentMessageErrors)
	a.Equal(expected.MessageCount, mFCM.TotalSentMessages)

	a.Equal(0, mFCM.Minute.CurrentErrorsCount)
	a.Equal(expected.MessageCount, mFCM.Minute.CurrentMessagesCount)
	a.Equal(0, mFCM.Minute.CurrentErrorsTotalLatencies)
	a.Equal(expected.ZeroLatencies, mFCM.Minute.CurrentMessagesTotalLatencies == 0)

	a.Equal(0, mFCM.Hour.CurrentErrorsCount)
	a.Equal(expected.MessageCount, mFCM.Hour.CurrentMessagesCount)
	a.Equal(0, mFCM.Hour.CurrentErrorsTotalLatencies)
	a.Equal(expected.ZeroLatencies, mFCM.Hour.CurrentMessagesTotalLatencies == 0)

	a.Equal(0, mFCM.Day.CurrentErrorsCount)
	a.Equal(expected.MessageCount, mFCM.Day.CurrentMessagesCount)
	a.Equal(0, mFCM.Day.CurrentErrorsTotalLatencies)
	a.Equal(expected.ZeroLatencies, mFCM.Day.CurrentMessagesTotalLatencies == 0)

	mRouter := &routerMetrics{}
	err = json.Unmarshal(bodyBytes, mRouter)
	a.NoError(err)

	a.Equal(expected.CurrentRoutes, mRouter.CurrentRoutes)
	a.Equal(expected.CurrentSubscriptions, mRouter.CurrentSubscriptions)
}
<commit_msg>short+disabled<commit_after>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"encoding/json"

	"github.com/cosminrentea/gobbler/client"
	"github.com/cosminrentea/gobbler/server/configstring"
	"github.com/cosminrentea/gobbler/server/connector"
	"github.com/cosminrentea/gobbler/server/fcm"
	"github.com/cosminrentea/gobbler/server/service"
	"github.com/cosminrentea/gobbler/testutil"
	"github.com/stretchr/testify/assert"
)

var (
	testHttpPort         = 11000
	timeoutForOneMessage = 150 * time.Millisecond
)

type fcmMetricsMap struct {
	CurrentErrorsCount            int `json:"current_errors_count"`
	CurrentMessagesCount          int `json:"current_messages_count"`
	CurrentMessagesTotalLatencies int `json:"current_messages_total_latencies_nanos"`
	CurrentErrorsTotalLatencies   int `json:"current_errors_total_latencies_nanos"`
}

type fcmMetrics struct {
	TotalSentMessages      int           `json:"fcm.total_sent_messages"`
	TotalSentMessageErrors int           `json:"fcm.total_sent_message_errors"`
	Minute                 fcmMetricsMap `json:"fcm.minute"`
	Hour                   fcmMetricsMap `json:"fcm.hour"`
	Day                    fcmMetricsMap `json:"fcm.day"`
}

type routerMetrics struct {
	CurrentRoutes        int `json:"router.current_routes"`
	CurrentSubscriptions int `json:"router.current_subscriptions"`
}

type expectedValues struct {
	ZeroLatencies        bool
	MessageCount         int
	CurrentRoutes        int
	CurrentSubscriptions int
}

// Test that restarting the service continues to fetch messages from store for a subscription from lastID
func TestFCMRestart(t *testing.T) {

	defer testutil.SkipIfDisabled(t)
	defer testutil.SkipIfShort(t)

	//defer testutil.EnableDebugForMethod()()
	defer testutil.ResetDefaultRegistryHealthCheck()
	a := assert.New(t)

	receiveC := make(chan bool)
	s, cleanup := serviceSetUp(t)
	defer cleanup()

	assertMetrics(a, s, expectedValues{true, 0, 0, 0})

	var fcmConn connector.ResponsiveConnector
	var ok bool
	for _, iface := range s.ModulesSortedByStartOrder() {
		fcmConn, ok = iface.(connector.ResponsiveConnector)
		if ok {
			break
		}
	}
	a.True(ok, "There should be a module of type FCMConnector")

	// add a high timeout so the messages are processed slow
	sender, err := fcm.CreateFcmSender(fcm.SuccessFCMResponse, receiveC, 10*time.Millisecond)
	a.NoError(err)
	fcmConn.SetSender(sender)

	// create subscription on topic
	subscriptionSetUp(t, s)
	time.Sleep(time.Second)
	assertMetrics(a, s, expectedValues{true, 0, 1, 1})

	c := clientSetUp(t, s)
	defer c.Close()

	// send 3 messages in the router but read only one and close the service
	for i := 0; i < 3; i++ {
		c.Send(testTopic, "dummy body", `{"Correlation-Id": "id"}`)
	}

	// receive one message only from FCM
	select {
	case <-receiveC:
	case <-time.After(timeoutForOneMessage):
		a.Fail("Initial FCM message not received")
	}

	time.Sleep(time.Second)

	assertMetrics(a, s, expectedValues{false, 1, 1, 1})
	close(receiveC)
	// restart the service
	a.NoError(s.Stop())

	// remake the sender
	receiveC = make(chan bool)
	sender, err = fcm.CreateFcmSender(fcm.SuccessFCMResponse, receiveC, 10*time.Millisecond)
	a.NoError(err)
	fcmConn.SetSender(sender)

	time.Sleep(50 * time.Millisecond)
	testutil.ResetDefaultRegistryHealthCheck()
	a.NoError(s.Start())

	//TODO Cosmin Bogdan add 2 calls to assertMetrics before and after the next block

	// read the other 2 messages
	for i := 0; i < 1; i++ {
		select {
		case <-receiveC:
		case <-time.After(2 * timeoutForOneMessage):
			a.Fail("FCM message not received")
		}
	}
}

func serviceSetUp(t *testing.T) (*service.Service, func()) {
	dir, errTempDir := ioutil.TempDir("", "guble_fcm_test")
	assert.NoError(t, errTempDir)

	*Config.KVS = "memory"
	*Config.MS = "file"
	*Config.Cluster.NodeID = 0
	*Config.StoragePath = dir
	*Config.MetricsEndpoint = "/admin/metrics-old"
	*Config.WS.Enabled = true
	*Config.WS.Prefix = "/stream/"
	*Config.FCM.Enabled = true
	*Config.FCM.APIKey = "WILL BE OVERWRITTEN"
	*Config.FCM.Prefix = "/fcm/"
	*Config.FCM.Workers = 1 // use only one worker so we can control the number of messages that go to FCM
	*Config.APNS.Enabled = false
	*Config.KafkaProducer.Brokers = configstring.List{}

	var s *service.Service
	for s == nil {
		testHttpPort++
		logger.WithField("port", testHttpPort).Debug("trying to use HTTP Port")
		*Config.HttpListen = fmt.Sprintf("127.0.0.1:%d", testHttpPort)
		s = StartService()
	}
	return s, func() {
		errRemove := os.RemoveAll(dir)
		if errRemove != nil {
			logger.WithError(errRemove).WithField("module", "testing").Error("Could not remove directory")
		}
	}
}

func clientSetUp(t *testing.T, service *service.Service) client.Client {
	wsURL := "ws://" + service.WebServer().GetAddr() + "/stream/user/user01"
	c, err := client.Open(wsURL, "http://localhost/", 1000, false)
	assert.NoError(t, err)
	return c
}

func subscriptionSetUp(t *testing.T, service *service.Service) {
	a := assert.New(t)

	urlFormat := fmt.Sprintf("http://%s/fcm/%%d/gcmId%%d/%%s", service.WebServer().GetAddr())
	// create GCM subscription
	response, errPost := http.Post(
		fmt.Sprintf(urlFormat, 1, 1, strings.TrimPrefix(testTopic, "/")),
		"text/plain",
		bytes.NewBufferString(""),
	)
	a.NoError(errPost)
	a.Equal(response.StatusCode, 200)

	body, errReadAll := ioutil.ReadAll(response.Body)
	a.NoError(errReadAll)
	a.Equal(fmt.Sprintf(`{"subscribed":"%s"}`, testTopic), string(body))
}

func assertMetrics(a *assert.Assertions, s *service.Service, expected expectedValues) {
	time.Sleep(50 * time.Millisecond)

	httpClient := &http.Client{}
	u := fmt.Sprintf("http://%s%s", s.WebServer().GetAddr(), defaultMetricsEndpoint)
	request, err := http.NewRequest(http.MethodGet, u, nil)
	a.NoError(err)

	response, err := httpClient.Do(request)
	a.NoError(err)
	defer response.Body.Close()

	a.Equal(http.StatusOK, response.StatusCode)
	bodyBytes, err := ioutil.ReadAll(response.Body)
	a.NoError(err)
	logger.WithField("body", string(bodyBytes)).Debug("metrics response")

	mFCM := &fcmMetrics{}
	err = json.Unmarshal(bodyBytes, mFCM)
	a.NoError(err)

	a.Equal(0, mFCM.TotalSentMessageErrors)
	a.Equal(expected.MessageCount, mFCM.TotalSentMessages)

	a.Equal(0, mFCM.Minute.CurrentErrorsCount)
	a.Equal(expected.MessageCount, mFCM.Minute.CurrentMessagesCount)
	a.Equal(0, mFCM.Minute.CurrentErrorsTotalLatencies)
	a.Equal(expected.ZeroLatencies, mFCM.Minute.CurrentMessagesTotalLatencies == 0)

	a.Equal(0, mFCM.Hour.CurrentErrorsCount)
	a.Equal(expected.MessageCount, mFCM.Hour.CurrentMessagesCount)
	a.Equal(0, mFCM.Hour.CurrentErrorsTotalLatencies)
	a.Equal(expected.ZeroLatencies, mFCM.Hour.CurrentMessagesTotalLatencies == 0)

	a.Equal(0, mFCM.Day.CurrentErrorsCount)
	a.Equal(expected.MessageCount, mFCM.Day.CurrentMessagesCount)
	a.Equal(0, mFCM.Day.CurrentErrorsTotalLatencies)
	a.Equal(expected.ZeroLatencies, mFCM.Day.CurrentMessagesTotalLatencies == 0)

	mRouter := &routerMetrics{}
	err = json.Unmarshal(bodyBytes, mRouter)
	a.NoError(err)

	a.Equal(expected.CurrentRoutes, mRouter.CurrentRoutes)
	a.Equal(expected.CurrentSubscriptions, mRouter.CurrentSubscriptions)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package convert

import (
	"io"
	"math"
	"sync"
)

var (
	// cosTable contains values of cosine applied to the range [0, π/2).
	// It must be initialised the first time it is referenced
	// in a function via its lazy load wrapper getCosTable().
	cosTable     []float64
	cosTableOnce sync.Once
)

func getCosTable() []float64 {
	cosTableOnce.Do(func() {
		cosTable = make([]float64, 65536)
		for i := range cosTable {
			cosTable[i] = math.Cos(float64(i) * math.Pi / 2 / float64(len(cosTable)))
		}
	})
	return cosTable
}

func fastCos01(x float64) float64 {
	if x < 0 {
		x = -x
	}
	i := int(4 * float64(len(getCosTable())) * x)
	if 4*len(cosTable) < i {
		i %= 4 * len(cosTable)
	}
	sign := 1
	switch {
	case i < len(cosTable):
	case i < len(cosTable)*2:
		i = len(cosTable)*2 - i
		sign = -1
	case i < len(cosTable)*3:
		i -= len(cosTable) * 2
		sign = -1
	default:
		i = len(cosTable)*4 - i
	}
	if i == len(cosTable) {
		return 0
	}
	return float64(sign) * cosTable[i]
}

func fastSin01(x float64) float64 {
	return fastCos01(x - 0.25)
}

func sinc01(x float64) float64 {
	if math.Abs(x) < 1e-8 {
		return 1
	}
	return fastSin01(x) / (x * 2 * math.Pi)
}

type Resampling struct {
	source       io.ReadSeeker
	size         int64
	from         int
	to           int
	pos          int64
	srcBlock     int64
	srcBufL      map[int64][]float64
	srcBufR      map[int64][]float64
	lruSrcBlocks []int64
}

func NewResampling(source io.ReadSeeker, size int64, from, to int) *Resampling {
	r := &Resampling{
		source:   source,
		size:     size,
		from:     from,
		to:       to,
		srcBlock: -1,
		srcBufL:  map[int64][]float64{},
		srcBufR:  map[int64][]float64{},
	}
	return r
}

func (r *Resampling) Length() int64 {
	s := int64(float64(r.size) * float64(r.to) / float64(r.from))
	return s / 4 * 4
}

func (r *Resampling) src(i int64) (float64, float64, error) {
	const resamplingBufferSize = 4096

	if i < 0 {
		return 0, 0, nil
	}
	if r.size/4 <= int64(i) {
		return 0, 0, nil
	}
	nextPos := int64(i) / resamplingBufferSize
	if _, ok := r.srcBufL[nextPos]; !ok {
		if r.srcBlock+1 != nextPos {
			if _, err := r.source.Seek(nextPos*resamplingBufferSize*4, io.SeekStart); err != nil {
				return 0, 0, err
			}
		}
		buf := make([]byte, resamplingBufferSize*4)
		c := 0
		for c < len(buf) {
			n, err := r.source.Read(buf[c:])
			c += n
			if err != nil {
				if err == io.EOF {
					break
				}
				return 0, 0, err
			}
		}
		buf = buf[:c]
		sl := make([]float64, resamplingBufferSize)
		sr := make([]float64, resamplingBufferSize)
		for i := 0; i < len(buf)/4; i++ {
			sl[i] = float64(int16(buf[4*i])|(int16(buf[4*i+1])<<8)) / (1<<15 - 1)
			sr[i] = float64(int16(buf[4*i+2])|(int16(buf[4*i+3])<<8)) / (1<<15 - 1)
		}
		r.srcBlock = nextPos
		r.srcBufL[r.srcBlock] = sl
		r.srcBufR[r.srcBlock] = sr
		// To keep srcBufL/R not too big, let's remove the least used buffers.
		if len(r.lruSrcBlocks) >= 4 {
			p := r.lruSrcBlocks[0]
			delete(r.srcBufL, p)
			delete(r.srcBufR, p)
			copy(r.lruSrcBlocks, r.lruSrcBlocks[1:])
			r.lruSrcBlocks = r.lruSrcBlocks[:len(r.lruSrcBlocks)-1]
		}
		r.lruSrcBlocks = append(r.lruSrcBlocks, r.srcBlock)
	} else {
		r.srcBlock = nextPos
		idx := -1
		for i, p := range r.lruSrcBlocks {
			if p == r.srcBlock {
				idx = i
				break
			}
		}
		if idx == -1 {
			panic("not reach")
		}
		r.lruSrcBlocks = append(r.lruSrcBlocks[:idx], r.lruSrcBlocks[idx+1:]...)
		r.lruSrcBlocks = append(r.lruSrcBlocks, r.srcBlock)
	}
	ii := i % resamplingBufferSize
	return r.srcBufL[r.srcBlock][ii], r.srcBufR[r.srcBlock][ii], nil
}

func (r *Resampling) at(t int64) (float64, float64, error) {
	windowSize := 8.0
	tInSrc := float64(t) * float64(r.from) / float64(r.to)
	startN := int64(tInSrc - windowSize)
	if startN < 0 {
		startN = 0
	}
	if r.size/4 <= startN {
		startN = r.size/4 - 1
	}
	endN := int64(tInSrc + windowSize)
	if r.size/4 <= endN {
		endN = r.size/4 - 1
	}
	lv := 0.0
	rv := 0.0
	for n := startN; n <= endN; n++ {
		srcL, srcR, err := r.src(n)
		if err != nil {
			return 0, 0, err
		}
		d := tInSrc - float64(n)
		w := 0.5 + 0.5*fastCos01(d/(windowSize*2+1))
		s := sinc01(d/2) * w
		lv += srcL * s
		rv += srcR * s
	}
	if lv < -1 {
		lv = -1
	}
	if lv > 1 {
		lv = 1
	}
	if rv < -1 {
		rv = -1
	}
	if rv > 1 {
		rv = 1
	}
	return lv, rv, nil
}

func (r *Resampling) Read(b []byte) (int, error) {
	if r.pos == r.Length() {
		return 0, io.EOF
	}
	n := len(b) / 4 * 4
	if r.Length()-r.pos <= int64(n) {
		n = int(r.Length() - r.pos)
	}
	for i := 0; i < n/4; i++ {
		l, r, err := r.at(r.pos/4 + int64(i))
		if err != nil {
			return 0, err
		}
		l16 := int16(l * (1<<15 - 1))
		r16 := int16(r * (1<<15 - 1))
		b[4*i] = byte(l16)
		b[4*i+1] = byte(l16 >> 8)
		b[4*i+2] = byte(r16)
		b[4*i+3] = byte(r16 >> 8)
	}
	r.pos += int64(n)
	return n, nil
}

func (r *Resampling) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case io.SeekStart:
		r.pos = offset
	case io.SeekCurrent:
		r.pos += offset
	case io.SeekEnd:
		r.pos += r.Length() + offset
	}
	if r.pos < 0 {
		r.pos = 0
	}
	if r.Length() <= r.pos {
		r.pos = r.Length()
	}
	return r.pos, nil
}
<commit_msg>audio/internal/convert: refactoring<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package convert

import (
	"io"
	"math"
	"sync"
)

var (
	// cosTable contains values of cosine applied to the range [0, π/2).
	// It must be initialised the first time it is referenced
	// in a function via its lazy load wrapper getCosTable().
	cosTable     []float64
	cosTableOnce sync.Once
)

func ensureCosTable() []float64 {
	cosTableOnce.Do(func() {
		cosTable = make([]float64, 65536)
		for i := range cosTable {
			cosTable[i] = math.Cos(float64(i) * math.Pi / 2 / float64(len(cosTable)))
		}
	})
	return cosTable
}

func fastCos01(x float64) float64 {
	if x < 0 {
		x = -x
	}

	cosTable := ensureCosTable()
	i := int(4 * float64(len(cosTable)) * x)
	if 4*len(cosTable) < i {
		i %= 4 * len(cosTable)
	}
	sign := 1
	switch {
	case i < len(cosTable):
	case i < len(cosTable)*2:
		i = len(cosTable)*2 - i
		sign = -1
	case i < len(cosTable)*3:
		i -= len(cosTable) * 2
		sign = -1
	default:
		i = len(cosTable)*4 - i
	}
	if i == len(cosTable) {
		return 0
	}
	return float64(sign) * cosTable[i]
}

func fastSin01(x float64) float64 {
	return fastCos01(x - 0.25)
}

func sinc01(x float64) float64 {
	if math.Abs(x) < 1e-8 {
		return 1
	}
	return fastSin01(x) / (x * 2 * math.Pi)
}

type Resampling struct {
	source       io.ReadSeeker
	size         int64
	from         int
	to           int
	pos          int64
	srcBlock     int64
	srcBufL      map[int64][]float64
	srcBufR      map[int64][]float64
	lruSrcBlocks []int64
}

func NewResampling(source io.ReadSeeker, size int64, from, to int) *Resampling {
	r := &Resampling{
		source:   source,
		size:     size,
		from:     from,
		to:       to,
		srcBlock: -1,
		srcBufL:  map[int64][]float64{},
		srcBufR:  map[int64][]float64{},
	}
	return r
}

func (r *Resampling) Length() int64 {
	s := int64(float64(r.size) * float64(r.to) / float64(r.from))
	return s / 4 * 4
}

func (r *Resampling) src(i int64) (float64, float64, error) {
	const resamplingBufferSize = 4096

	if i < 0 {
		return 0, 0, nil
	}
	if r.size/4 <= int64(i) {
		return 0, 0, nil
	}
	nextPos := int64(i) / resamplingBufferSize
	if _, ok := r.srcBufL[nextPos]; !ok {
		if r.srcBlock+1 != nextPos {
			if _, err := r.source.Seek(nextPos*resamplingBufferSize*4, io.SeekStart); err != nil {
				return 0, 0, err
			}
		}
		buf := make([]byte, resamplingBufferSize*4)
		c := 0
		for c < len(buf) {
			n, err := r.source.Read(buf[c:])
			c += n
			if err != nil {
				if err == io.EOF {
					break
				}
				return 0, 0, err
			}
		}
		buf = buf[:c]
		sl := make([]float64, resamplingBufferSize)
		sr := make([]float64, resamplingBufferSize)
		for i := 0; i < len(buf)/4; i++ {
			sl[i] = float64(int16(buf[4*i])|(int16(buf[4*i+1])<<8)) / (1<<15 - 1)
			sr[i] = float64(int16(buf[4*i+2])|(int16(buf[4*i+3])<<8)) / (1<<15 - 1)
		}
		r.srcBlock = nextPos
		r.srcBufL[r.srcBlock] = sl
		r.srcBufR[r.srcBlock] = sr
		// To keep srcBufL/R not too big, let's remove the least used buffers.
		if len(r.lruSrcBlocks) >= 4 {
			p := r.lruSrcBlocks[0]
			delete(r.srcBufL, p)
			delete(r.srcBufR, p)
			copy(r.lruSrcBlocks, r.lruSrcBlocks[1:])
			r.lruSrcBlocks = r.lruSrcBlocks[:len(r.lruSrcBlocks)-1]
		}
		r.lruSrcBlocks = append(r.lruSrcBlocks, r.srcBlock)
	} else {
		r.srcBlock = nextPos
		idx := -1
		for i, p := range r.lruSrcBlocks {
			if p == r.srcBlock {
				idx = i
				break
			}
		}
		if idx == -1 {
			panic("not reach")
		}
		r.lruSrcBlocks = append(r.lruSrcBlocks[:idx], r.lruSrcBlocks[idx+1:]...)
		r.lruSrcBlocks = append(r.lruSrcBlocks, r.srcBlock)
	}
	ii := i % resamplingBufferSize
	return r.srcBufL[r.srcBlock][ii], r.srcBufR[r.srcBlock][ii], nil
}

func (r *Resampling) at(t int64) (float64, float64, error) {
	windowSize := 8.0
	tInSrc := float64(t) * float64(r.from) / float64(r.to)
	startN := int64(tInSrc - windowSize)
	if startN < 0 {
		startN = 0
	}
	if r.size/4 <= startN {
		startN = r.size/4 - 1
	}
	endN := int64(tInSrc + windowSize)
	if r.size/4 <= endN {
		endN = r.size/4 - 1
	}
	lv := 0.0
	rv := 0.0
	for n := startN; n <= endN; n++ {
		srcL, srcR, err := r.src(n)
		if err != nil {
			return 0, 0, err
		}
		d := tInSrc - float64(n)
		w := 0.5 + 0.5*fastCos01(d/(windowSize*2+1))
		s := sinc01(d/2) * w
		lv += srcL * s
		rv += srcR * s
	}
	if lv < -1 {
		lv = -1
	}
	if lv > 1 {
		lv = 1
	}
	if rv < -1 {
		rv = -1
	}
	if rv > 1 {
		rv = 1
	}
	return lv, rv, nil
}

func (r *Resampling) Read(b []byte) (int, error) {
	if r.pos == r.Length() {
		return 0, io.EOF
	}
	n := len(b) / 4 * 4
	if r.Length()-r.pos <= int64(n) {
		n = int(r.Length() - r.pos)
	}
	for i := 0; i < n/4; i++ {
		l, r, err := r.at(r.pos/4 + int64(i))
		if err != nil {
			return 0, err
		}
		l16 := int16(l * (1<<15 - 1))
		r16 := int16(r * (1<<15 - 1))
		b[4*i] = byte(l16)
		b[4*i+1] = byte(l16 >> 8)
		b[4*i+2] = byte(r16)
		b[4*i+3] = byte(r16 >> 8)
	}
	r.pos += int64(n)
	return n, nil
}

func (r *Resampling) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case io.SeekStart:
		r.pos = offset
	case io.SeekCurrent:
		r.pos += offset
	case io.SeekEnd:
		r.pos += r.Length() + offset
	}
	if r.pos < 0 {
		r.pos = 0
	}
	if r.Length() <= r.pos {
		r.pos = r.Length()
	}
	return r.pos, nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, rt)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	var errStr string
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil {
			errStr += fmt.Sprintf("%s => %s: %v\n", r.url, r.file, r.err)
		}
	}
	if len(errStr) > 0 {
		return nil, errors.New(errStr)
	}

	return imap, nil
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = f.slurpRemoteBytes(u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func (f *Fetcher) slurpFragment(url string) ([]types.Node, error) {
	res, err := f.fetchRemote(url, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func (f *Fetcher) fetch(name string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return f.fetchRemote(name, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func (f *Fetcher) fetchRemote(urlStr string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return f.fetchDriveFile(urlStr, nometa)
	}
	return f.fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func (f *Fetcher) fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func (f *Fetcher) fetchDriveFile(id string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)

	if nometa {
		res, err := retryGet(f.authHelper.DriveClient(), exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(f.authHelper.DriveClient(), u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(f.authHelper.DriveClient(), exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func (f *Fetcher) slurpRemoteBytes(url string, n int) ([]byte, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Attempts are spaced out with exponential backoff.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<commit_msg>don't bother user with oauth if exporting from a local file (#347)<commit_after>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
	roundTripper http.RoundTripper
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	return &Fetcher{
		authHelper:   nil,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
		roundTripper: rt,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	_, err := os.Stat(src)
	// Only setup oauth if this source is not a local file.
	if os.IsNotExist(err) {
		if f.authHelper == nil {
			f.authHelper, err = auth.NewHelper(f.authToken, auth.ProviderGoogle, f.roundTripper)
			if err != nil {
				return nil, err
			}
		}
	}
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	var errStr string
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil {
			errStr += fmt.Sprintf("%s => %s: %v\n", r.url, r.file, r.err)
		}
	}
	if len(errStr) > 0 {
		return nil, errors.New(errStr)
	}

	return imap, nil
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = f.slurpRemoteBytes(u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func (f *Fetcher) slurpFragment(url string) ([]types.Node, error) {
	res, err := f.fetchRemote(url, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func (f *Fetcher) fetch(name string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return f.fetchRemote(name, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func (f *Fetcher) fetchRemote(urlStr string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return f.fetchDriveFile(urlStr, nometa)
	}
	return f.fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func (f *Fetcher) fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func (f *Fetcher) fetchDriveFile(id string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)

	if nometa {
		res, err := retryGet(f.authHelper.DriveClient(), exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(f.authHelper.DriveClient(), u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(f.authHelper.DriveClient(), exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func (f *Fetcher) slurpRemoteBytes(url string, n int) ([]byte, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Attempts are spaced out with exponential backoff.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<|endoftext|>


<commit_before>package authorisation

import (
	"time"

	"github.com/dgrijalva/jwt-go"
)

// IRequestTokenCreator - Creates a authorisation token that is appended to request as header
type IRequestTokenCreator interface {
	Create(string, string, string, string) (string, error)
}

// JWTReqestTokenCreator - JWT implementation of IRequestTokenCreator
type JWTReqestTokenCreator struct {
	_claimKeyRequestSignature string
	_claimKeyPayloadSignature string
}

// Create - Creates a JWT token. JwtTokenCreator uses HMAC-SHA256 signature, by default.
func (j *JWTReqestTokenCreator) Create(resourcePath string, body string, issuer string, secret string) (string, error) {

	token := jwt.New(jwt.SigningMethodHS256)

	token.Claims["foo"] = "bar"
	token.Claims["exp"] = time.Now().Add(time.Hour * 72).Unix()

	tokenString, err := token.SignedString(j._claimKeyRequestSignature)

	return tokenString, err

}
<commit_msg>Issuer and Issued At<commit_after>package authorisation

import (
	"time"

	"github.com/dgrijalva/jwt-go"
)

// IRequestTokenCreator - Creates a authorisation token that is appended to request as header
type IRequestTokenCreator interface {
	Create(string, string, string, string) (string, error)
}

// JWTReqestTokenCreator - JWT implementation of IRequestTokenCreator
type JWTReqestTokenCreator struct {
	_claimKeyRequestSignature string
	_claimKeyPayloadSignature string
}

// Create - Creates a JWT token. JwtTokenCreator uses HMAC-SHA256 signature, by default.
func (j *JWTReqestTokenCreator) Create(resourcePath string, body string, issuer string, secret string) (string, error) {

	token := jwt.New(jwt.SigningMethodHS256)

	token.Claims["ISSUER"] = issuer
	token.Claims["ISSUED_AT"] = time.Now()

	tokenString, err := token.SignedString(j._claimKeyRequestSignature)

	return tokenString, err

}
<|endoftext|>


<commit_before>package io

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
)

//	写文件
func WriteLines(filePath string, lines []string) error {

	//	打开文件
	file, err := openForWrite(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	for _, line := range lines {

		//	将字符串写入文件
		_, err = file.WriteString(line + "\n")
		if err != nil {
			return err
		}
	}

	return nil
}

//	写入字符串
func WriteString(filePath, content string) error {

	//	打开文件
	file, err := openForWrite(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.WriteString(content)

	return err
}

//	写入缓冲区
func WriteBytes(filePath string, buffer []byte) error {
	//	打开文件
	file, err := openForWrite(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write(buffer)

	return err
}

//	打开文件
func openForWrite(filePath string) (*os.File, error) {
	//	检查文件所处目录是否存在
	fileDir := filepath.Dir(filePath)
	_, err := os.Stat(fileDir)
	if os.IsNotExist(err) {
		//	如果不存在就先创建目录
		err = os.Mkdir(fileDir, 0660)
		if err != nil {
			return nil, err
		}
	}

	//	打开文件
	return os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY, 0660)
}

//	打开文件
func openForRead(filePath string) (*os.File, error) {
	//	检查文件
	_, err := os.Stat(filePath)
	if err != nil {
		return nil, err
	}

	//	打开文件
	return os.OpenFile(filePath, os.O_RDONLY, 0660)
}

//	读取文件
func ReadLines(filePath string) ([]string, error) {
	//	打开文件
	file, err := openForRead(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	//	读取
	scanner := bufio.NewScanner(file)
	lines := make([]string, 0)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	return lines, scanner.Err()
}

//	读取所有
func ReadAllBytes(filePath string) ([]byte, error) {
	//	打开文件
	file, err := openForRead(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ioutil.ReadAll(file)
}

//	读取所有
func ReadAllString(filePath string) (string, error) {
	buffer, err := ReadAllBytes(filePath)
	if err != nil {
		return "", err
	}

	return string(buffer), nil
}
<commit_msg>创建的文件和目录第三方都可以读取<commit_after>package io

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
)

//	写文件
func WriteLines(filePath string, lines []string) error {

	//	打开文件
	file, err := openForWrite(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	for _, line := range lines {

		//	将字符串写入文件
		_, err = file.WriteString(line + "\n")
		if err != nil {
			return err
		}
	}

	return nil
}

//	写入字符串
func WriteString(filePath, content string) error {

	//	打开文件
	file, err := openForWrite(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.WriteString(content)

	return err
}

//	写入缓冲区
func WriteBytes(filePath string, buffer []byte) error {
	//	打开文件
	file, err := openForWrite(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write(buffer)

	return err
}

//	打开文件
func openForWrite(filePath string) (*os.File, error) {
	//	检查文件所处目录是否存在
	fileDir := filepath.Dir(filePath)
	_, err := os.Stat(fileDir)
	if os.IsNotExist(err) {
		//	如果不存在就先创建目录
		err = os.Mkdir(fileDir, 0664)
		if err != nil {
			return nil, err
		}
	}

	//	打开文件
	return os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY, 0664)
}

//	打开文件
func openForRead(filePath string) (*os.File, error) {
	//	检查文件
	_, err := os.Stat(filePath)
	if err != nil {
		return nil, err
	}

	//	打开文件
	return os.OpenFile(filePath, os.O_RDONLY, 0664)
}

//	读取文件
func ReadLines(filePath string) ([]string, error) {
	//	打开文件
	file, err := openForRead(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	//	读取
	scanner := bufio.NewScanner(file)
	lines := make([]string, 0)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	return lines, scanner.Err()
}

//	读取所有
func ReadAllBytes(filePath string) ([]byte, error) {
	//	打开文件
	file, err := openForRead(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ioutil.ReadAll(file)
}

//	读取所有
func ReadAllString(filePath string) (string, error) {
	buffer, err := ReadAllBytes(filePath)
	if err != nil {
		return "", err
	}

	return string(buffer), nil
}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"

	"github.com/remind101/pkg/httpx"
	"golang.org/x/net/context"
)

// HeaderRequestID is the default name of the header to extract request ids
// from.
var HeaderRequestID = []string{"X-Request-Id", "Request-Id"}

// RequestID is middleware that extracts a request id from the headers and
// inserts it into the context.
type RequestID struct {
	// Extractor is a function that can extract a request id from an
	// http.Request. The zero value is a function that will pull a request
	// id from the `X-Request-ID` or `Request-ID` headers.
	Extractor func(*http.Request) string

	// handler is the wrapped httpx.Handler.
	handler httpx.Handler
}

func ExtractRequestID(h httpx.Handler) *RequestID {
	return &RequestID{
		handler: h,
	}
}

// ServeHTTPContext implements the httpx.Handler interface. It extracts a
// request id from the headers and inserts it into the context.
func (h *RequestID) ServeHTTPContext(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
	e := h.Extractor
	if e == nil {
		e = extractRequestID
	}
	requestID := e(r)

	ctx = httpx.WithRequestID(ctx, requestID)
	return h.handler.ServeHTTPContext(ctx, w, r)
}

func extractRequestID(r *http.Request) string {
	for _, h := range HeaderRequestID {
		v := r.Header.Get(h)
		if v != "" {
			return v
		}
	}

	return ""
}
<commit_msg>Cleanup.<commit_after>package middleware

import (
	"net/http"

	"github.com/remind101/pkg/httpx"
	"golang.org/x/net/context"
)

// DefaultRequestIDExtractor is the default function to use to extract a request
// id from an http.Request.
var DefaultRequestIDExtractor = HeaderExtractor([]string{"X-Request-Id", "Request-Id"})

// RequestID is middleware that extracts a request id from the headers and
// inserts it into the context.
type RequestID struct {
	// Extractor is a function that can extract a request id from an
	// http.Request. The zero value is a function that will pull a request
	// id from the `X-Request-ID` or `Request-ID` headers.
	Extractor func(*http.Request) string

	// handler is the wrapped httpx.Handler.
	handler httpx.Handler
}

func ExtractRequestID(h httpx.Handler) *RequestID {
	return &RequestID{
		handler: h,
	}
}

// ServeHTTPContext implements the httpx.Handler interface. It extracts a
// request id from the headers and inserts it into the context.
func (h *RequestID) ServeHTTPContext(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
	e := h.Extractor
	if e == nil {
		e = DefaultRequestIDExtractor
	}
	requestID := e(r)

	ctx = httpx.WithRequestID(ctx, requestID)
	return h.handler.ServeHTTPContext(ctx, w, r)
}

// HeaderExtractor returns a function that can extract a request id from a list
// of headers.
func HeaderExtractor(headers []string) func(*http.Request) string {
	return func(r *http.Request) string {
		for _, h := range headers {
			v := r.Header.Get(h)
			if v != "" {
				return v
			}
		}

		return ""
	}
}
<|endoftext|>


<commit_before>package clevels

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
)

var Stats *statsd.Client

//go:generate stringer -type=AusterityLevel
type AusterityLevel int

// There are four levels of logs. The names and definitions are adapted
// from Google's usage.
//
// For more information, see Chapter 21 of Site Reliability Engineering
// by Beyer, Jones, Petoff, and Murphy
//
// Sheddable is the least-important, and we can expect frequent partial unavailability
// of Sheddable logs as well as occassional full unavailability without
// disruption.
//
// SheddablePlus is the default level, and we can expect occasional partial availability
// and rare full unavailability.
//
// Critical is used for lines for which partial unavailability is rare.
//
// CriticalPlus is used for lines which should only be unavailable
// in the most extreme circumstances.
const (
	Sheddable AusterityLevel = iota
	SheddablePlus
	Critical
	CriticalPlus
)

// Define two helper constants here. Since our default policies may change
// later, this means we only have to change the definitions at one point

// The default criticality line for any log line
// that doesn't have an explicit clevel tag
const DefaultCriticality AusterityLevel = SheddablePlus

// The default system austerity level, in case
// we are unable to determine it from the state file
const DefaultAusterity AusterityLevel = Sheddable

func (a AusterityLevel) AusterityLevel() string {
	return ""
}

// AusterityBuffer ensures that high-volume services will never block
// when trying to read the system austerity level, even if the process
// that updates the cache is slow. The tradeoff is that, when the system
// austerity level is changed, it will read an extra $AusterityBuffer log
// lines before the change takes effect.
const AusterityBuffer = 100

var CacheInterval = 30 * time.Second

// To determine the current austerity level, read from SystemAusterityLevel.
// Austerity is updated according to CacheInterval,
// and this channel is buffered (size AusterityBuffer) to avoid
// blocking a critical code path.
// As a result, services which have a low log volume will take longer
// to have their austerity level changes come into effect.
var SystemAusterityLevel = make(chan AusterityLevel, AusterityBuffer)

// AusterityFile is the full path to a file that contains the current
// system austerity level.
var AusterityFile string

var InvalidAusterityLevel = errors.New("Invalid austerity level")

// LoadLevel loads the AusterityFile and parses it to determine
// the system austerity level. If it encounters an error, it will
// return the DefaultAusterity.
func LoadLevel() (AusterityLevel, error) {
	f, err := os.Open(AusterityFile)
	if err != nil {
		return DefaultAusterity, err
	}
	defer f.Close()
	level, err := ParseLevel(f)

	return level, err
}

var canonicalRegex = regexp.MustCompile(`(?i)CANONICAL-\w+?-LINE`)
var cLevelRegex = regexp.MustCompile(`\[clevel: (\w*?)\]`)
var cLevelChalkRegex = regexp.MustCompile(`\sclevel=(\w+?)\b`)

var clevelRegexes = []*regexp.Regexp{cLevelRegex, cLevelChalkRegex}

// criticality parses the criticality level
// of a log line. Defaults to the value of DefaultCriticality.
func Criticality(line string) AusterityLevel {
	// Things like CANONICAL-API-LINE should never be dropped
	if canonicalRegex.MatchString(line) {
		return CriticalPlus
	}

	for _, r := range clevelRegexes {
		matches := r.FindStringSubmatch(line)
		if len(matches) < 2 {
			continue
		}

		level, err := ParseLevel(strings.NewReader(matches[1]))
		// we don't really care about any error here
		// and want to default to not dropping anything
		if err != nil {
			continue
		}
		// if we've made it this far, we've succesfully parsed
		// the clevel
		return level
	}

	return DefaultCriticality
}

func ParseLevel(r io.Reader) (AusterityLevel, error) {
	bts, err := ioutil.ReadAll(r)
	if err != nil {
		return DefaultAusterity, err
	}

	level := strings.ToLower(strings.TrimSpace(string(bts)))
	switch level {
	case strings.ToLower(Sheddable.String()):
		return Sheddable, nil
	case strings.ToLower(SheddablePlus.String()):
		return SheddablePlus, nil
	case strings.ToLower(Critical.String()):
		return Critical, nil
	case strings.ToLower(CriticalPlus.String()):
		return CriticalPlus, nil
	}
	return DefaultAusterity, InvalidAusterityLevel
}

func SendSystemAusterityLevel() {
	// This is the cached austerity level
	// that will be sent anytime Unilog requests the austerity level,
	// so that there is never any delay.
	// By default, there is no austerity.
	var currentLevel = Sheddable

	// shadow this variable so we can override it in tests, if needed.
	// multiple goroutines write to SystemAusterityLevel in tests, so this shadowing
	// gives us the ability to separate the channels in tests.
	var _SystemAusterityLevel = SystemAusterityLevel

	newLevelCh := make(chan AusterityLevel)
	go func(updatedLevel chan<- AusterityLevel) {
		c := time.Tick(CacheInterval)
		for _ = range c {
			l, err := LoadLevel()
			if err != nil {
				if Stats != nil {
					Stats.Count("unilog.errors.load_level", 1, nil, 1)
				}
				continue
			}
			updatedLevel <- l
		}
	}(newLevelCh)

	// Loop forever on this
	for {
		// This select statement will be on a hotpath
		// so it should never block for long
		select {
		// Continuously send the current austerity level
		// to whoever asks for it
		case _SystemAusterityLevel <- currentLevel:

		case newLevel := <-newLevelCh:
			go ReportAusterity(newLevel)
			currentLevel = newLevel
		}
	}
}

func ReportAusterity(l AusterityLevel) {
	if Stats != nil {
		Stats.Gauge("unilog.austerity.box", float64(l), nil, 1)
	}
}
<commit_msg>Report austerity loading errors and successes<commit_after>package clevels

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
)

var Stats *statsd.Client

//go:generate stringer -type=AusterityLevel
type AusterityLevel int

// There are four levels of logs. The names and definitions are adapted
// from Google's usage.
//
// For more information, see Chapter 21 of Site Reliability Engineering
// by Beyer, Jones, Petoff, and Murphy
//
// Sheddable is the least-important, and we can expect frequent partial unavailability
// of Sheddable logs as well as occassional full unavailability without
// disruption.
//
// SheddablePlus is the default level, and we can expect occasional partial availability
// and rare full unavailability.
//
// Critical is used for lines for which partial unavailability is rare.
//
// CriticalPlus is used for lines which should only be unavailable
// in the most extreme circumstances.
const (
	Sheddable AusterityLevel = iota
	SheddablePlus
	Critical
	CriticalPlus
)

// Define two helper constants here. Since our default policies may change
// later, this means we only have to change the definitions at one point

// The default criticality line for any log line
// that doesn't have an explicit clevel tag
const DefaultCriticality AusterityLevel = SheddablePlus

// The default system austerity level, in case
// we are unable to determine it from the state file
const DefaultAusterity AusterityLevel = Sheddable

func (a AusterityLevel) AusterityLevel() string {
	return ""
}

// AusterityBuffer ensures that high-volume services will never block
// when trying to read the system austerity level, even if the process
// that updates the cache is slow. The tradeoff is that, when the system
// austerity level is changed, it will read an extra $AusterityBuffer log
// lines before the change takes effect.
const AusterityBuffer = 100

var CacheInterval = 30 * time.Second

// To determine the current austerity level, read from SystemAusterityLevel.
// Austerity is updated according to CacheInterval,
// and this channel is buffered (size AusterityBuffer) to avoid
// blocking a critical code path.
// As a result, services which have a low log volume will take longer
// to have their austerity level changes come into effect.
var SystemAusterityLevel = make(chan AusterityLevel, AusterityBuffer)

// AusterityFile is the full path to a file that contains the current
// system austerity level.
var AusterityFile string

var InvalidAusterityLevel = errors.New("Invalid austerity level")

// LoadLevel loads the AusterityFile and parses it to determine
// the system austerity level. If it encounters an error, it will
// return the DefaultAusterity.
func LoadLevel() (AusterityLevel, error) {
	f, err := os.Open(AusterityFile)
	if err != nil {
		return DefaultAusterity, err
	}
	defer f.Close()
	level, err := ParseLevel(f)

	return level, err
}

var canonicalRegex = regexp.MustCompile(`(?i)CANONICAL-\w+?-LINE`)
var cLevelRegex = regexp.MustCompile(`\[clevel: (\w*?)\]`)
var cLevelChalkRegex = regexp.MustCompile(`\sclevel=(\w+?)\b`)

var clevelRegexes = []*regexp.Regexp{cLevelRegex, cLevelChalkRegex}

// criticality parses the criticality level
// of a log line. Defaults to the value of DefaultCriticality.
func Criticality(line string) AusterityLevel {
	// Things like CANONICAL-API-LINE should never be dropped
	if canonicalRegex.MatchString(line) {
		return CriticalPlus
	}

	for _, r := range clevelRegexes {
		matches := r.FindStringSubmatch(line)
		if len(matches) < 2 {
			continue
		}

		level, err := ParseLevel(strings.NewReader(matches[1]))
		// we don't really care about any error here
		// and want to default to not dropping anything
		if err != nil {
			continue
		}
		// if we've made it this far, we've succesfully parsed
		// the clevel
		return level
	}

	return DefaultCriticality
}

func ParseLevel(r io.Reader) (AusterityLevel, error) {
	bts, err := ioutil.ReadAll(r)
	if err != nil {
		return DefaultAusterity, err
	}

	level := strings.ToLower(strings.TrimSpace(string(bts)))
	switch level {
	case strings.ToLower(Sheddable.String()):
		return Sheddable, nil
	case strings.ToLower(SheddablePlus.String()):
		return SheddablePlus, nil
	case strings.ToLower(Critical.String()):
		return Critical, nil
	case strings.ToLower(CriticalPlus.String()):
		return CriticalPlus, nil
	}
	return DefaultAusterity, InvalidAusterityLevel
}

func reportLoadStatus(err error) {
	if Stats != nil {
		metric := 0
		if err != nil {
			metric = 1
		}
		Stats.Count("unilog.errors.load_level", int64(metric), nil, 1)
	}
}

func SendSystemAusterityLevel() {
	// This is the cached austerity level
	// that will be sent anytime Unilog requests the austerity level,
	// so that there is never any delay.
	// By default, there is no austerity.
	var currentLevel = Sheddable

	// shadow this variable so we can override it in tests, if needed.
	// multiple goroutines write to SystemAusterityLevel in tests, so this shadowing
	// gives us the ability to separate the channels in tests.
	var _SystemAusterityLevel = SystemAusterityLevel

	newLevelCh := make(chan AusterityLevel)
	go func(updatedLevel chan<- AusterityLevel) {
		c := time.Tick(CacheInterval)
		for _ = range c {
			l, err := LoadLevel()
			reportLoadStatus(err)
			if err != nil {
				continue
			}
			updatedLevel <- l
		}
	}(newLevelCh)

	// Loop forever on this
	for {
		// This select statement will be on a hotpath
		// so it should never block for long
		select {
		// Continuously send the current austerity level
		// to whoever asks for it
		case _SystemAusterityLevel <- currentLevel:

		case newLevel := <-newLevelCh:
			go ReportAusterity(newLevel)
			currentLevel = newLevel
		}
	}
}

func ReportAusterity(l AusterityLevel) {
	if Stats != nil {
		Stats.Gauge("unilog.austerity.box", float64(l), nil, 1)
	}
}
<|endoftext|>