<commit_before>package router

import (
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/boatilus/peppercorn/middleware"
	"github.com/boatilus/peppercorn/paths"
	"github.com/boatilus/peppercorn/routes"
	"github.com/pressly/chi"
	chiMiddleware "github.com/pressly/chi/middleware"
)

const staticDir = "static"

func Create() (http.Handler, error) {
	middleware.InitCSP()

	r := chi.NewRouter()
	r.Use(chiMiddleware.RealIP)
	r.Use(middleware.VisitorID)
	r.Use(chiMiddleware.Logger)
	r.Use(chiMiddleware.Recoverer)
	r.Use(chiMiddleware.CloseNotify)
	r.Use(chiMiddleware.Timeout(60 * time.Second))
	r.Use(chiMiddleware.DefaultCompress)
	r.Use(middleware.SetSecurity())

	// TODO: Refactor these routes by protected/unprotected
	// GET
	r.With(middleware.Validate).Get("/", routes.IndexGetHandler)
	r.Get(paths.Get.SignIn, routes.SignInGetHandler)
	r.Get(paths.Get.Forgot, routes.ForgotGetHandler)
	r.Get(paths.Get.ResetPassword, routes.ResetPasswordGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.SignOut, routes.SignOutGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.Page, routes.PageGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.Single, routes.SingleGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.SingleRemove, routes.SingleRemoveGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.TotalPostCount, routes.CountGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.Me, routes.MeGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.MeRevoke, routes.MeRevokeGetHandler)
	r.With(middleware.Validate).Get(paths.Get.EnableTwoFactorAuthentication, routes.EnableTwoFactorAuthenticationGetHandler)
	r.With(middleware.Validate).Get(paths.Get.EnterCode, routes.EnterCodeGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.DisableTwoFactorAuthentication, routes.DisableTwoFactorAuthenticationGetHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.RecoveryCodes, routes.RecoveryCodesGetHandler)

	// POST
	r.Post(paths.Post.SignIn, routes.SignInPostHandler)
	r.Post(paths.Post.Forgot, routes.ForgotPostHandler)
	r.Post(paths.Post.ResetPassword, routes.ResetPasswordPostHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Post(paths.Post.Me, routes.MePostHandler)
	r.With(middleware.Validate, middleware.ValidateMFA).Post(paths.Post.SubmitPost, routes.PostsPostHandler)
	r.With(middleware.Validate).Post(paths.Post.EnableTwoFactorAuthentication, routes.EnableTwoFactorAuthenticationPostHandler)
	r.With(middleware.Validate).Post(paths.Post.EnterCode, routes.EnterCodePostHandler)

	// PATCH
	r.With(middleware.Validate).Patch(paths.Patch.Single, routes.SinglePatchHandler)

	workDir, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	filesDir := filepath.Join(workDir, staticDir)

	r.FileServer("/"+staticDir, http.Dir(filesDir))

	return r, nil
}
<commit_msg>router: replace fileserver with custom handler<commit_after>package router

import (
	"io"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/boatilus/ovao/log"
	"github.com/boatilus/peppercorn/middleware"
	"github.com/boatilus/peppercorn/paths"
	"github.com/boatilus/peppercorn/routes"
	"github.com/pressly/chi"
	chiMiddleware "github.com/pressly/chi/middleware"
)

const staticDir = "static"

func Create() (http.Handler, error) {
	middleware.InitCSP()

	r := chi.NewRouter()
	r.Use(chiMiddleware.Recoverer)
	r.Use(chiMiddleware.Timeout(30 * time.Second))
	r.Use(chiMiddleware.CloseNotify) // TODO: investigate whether this is causing issues

	r.Route("/static", func(r chi.Router) {
		r.Get("/*", serveFile)
	})

	// TODO: Refactor these routes by protected/unprotected
	r.Route("/", func(r chi.Router) {
		r.Use(chiMiddleware.RealIP)
		r.Use(middleware.VisitorID)
		r.Use(chiMiddleware.Logger)
		r.Use(chiMiddleware.DefaultCompress)
		r.Use(middleware.SetSecurity())

		// GET
		r.With(middleware.Validate).Get("/", routes.IndexGetHandler)
		r.Get(paths.Get.SignIn, routes.SignInGetHandler)
		r.Get(paths.Get.Forgot, routes.ForgotGetHandler)
		r.Get(paths.Get.ResetPassword, routes.ResetPasswordGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.SignOut, routes.SignOutGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.Page, routes.PageGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.Single, routes.SingleGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.SingleRemove, routes.SingleRemoveGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.TotalPostCount, routes.CountGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.Me, routes.MeGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.MeRevoke, routes.MeRevokeGetHandler)
		r.With(middleware.Validate).Get(paths.Get.EnableTwoFactorAuthentication, routes.EnableTwoFactorAuthenticationGetHandler)
		r.With(middleware.Validate).Get(paths.Get.EnterCode, routes.EnterCodeGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.DisableTwoFactorAuthentication, routes.DisableTwoFactorAuthenticationGetHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Get(paths.Get.RecoveryCodes, routes.RecoveryCodesGetHandler)

		// POST
		r.Post(paths.Post.SignIn, routes.SignInPostHandler)
		r.Post(paths.Post.Forgot, routes.ForgotPostHandler)
		r.Post(paths.Post.ResetPassword, routes.ResetPasswordPostHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Post(paths.Post.Me, routes.MePostHandler)
		r.With(middleware.Validate, middleware.ValidateMFA).Post(paths.Post.SubmitPost, routes.PostsPostHandler)
		r.With(middleware.Validate).Post(paths.Post.EnableTwoFactorAuthentication, routes.EnableTwoFactorAuthenticationPostHandler)
		r.With(middleware.Validate).Post(paths.Post.EnterCode, routes.EnterCodePostHandler)

		// PATCH
		r.With(middleware.Validate).Patch(paths.Patch.Single, routes.SinglePatchHandler)
	})

	return r, nil
}

func serveFile(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path[1:]
	origPath := path

	// If the client can accept compressed responses, we'll send back a pre-compressed file instead.
	// We prefer Brotli to gzip.
	ae := r.Header.Get("Accept-Encoding")
	if strings.Contains(ae, "br") {
		w.Header().Add("Content-Encoding", "br")
		path += ".br"
	} else if strings.Contains(ae, "gzip") {
		w.Header().Add("Content-Encoding", "gzip")
		path += ".gz"
	}

	f, err := os.Open(path)
	if os.IsPermission(err) {
		http.Error(w, "", http.StatusForbidden)
		return
	}
	if os.IsNotExist(err) {
		path = origPath

		var nextErr error

		// Attempt to load the original.
		f, nextErr = os.Open(path)
		if os.IsNotExist(nextErr) {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		if nextErr != nil {
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}
	if err != nil {
		// Our attempt to load a pre-compressed version failed, so fall back to the uncompressed
		// version.
		w.Header().Del("Content-Encoding")
	}

	defer f.Close()

	// We always want to send the content type of the original file rather than that of the gzip'ed
	// file, which is that of application/x-gzip.
	ext := filepath.Ext(origPath)

	// May as well set cache headers on SVGs, since we'll probably never need to modify those.
	if ext == ".svg" {
		w.Header().Add("Cache-Control", "max-age=31536000, immutable")
	}

	contentType := mime.TypeByExtension(ext)
	if contentType != "" {
		w.Header().Add("Content-Type", contentType)
	}

	fileInfo, err := f.Stat()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	fileBytes := fileInfo.Size()
	contentLength := strconv.FormatInt(fileBytes, 10)

	w.Header().Add("Content-Length", contentLength)

	wrote, err := io.CopyN(w, f, fileBytes)
	if err != nil {
		log.ErrorFrom("router", err)
	}
	if wrote != fileBytes {
		log.InfofFrom("router", "wrote %v bytes; file is %v bytes", wrote, fileBytes)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The ContainerOps Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package router

import (
	"gopkg.in/macaron.v1"

	"github.com/containerops/dockyard/handler"
)

/*
  Dockyard Router Definition
*/
func SetRouters(m *macaron.Macaron) {
	//Web API
	m.Get("/", handler.IndexV1Handler)

	//Docker Registry V1
	m.Group("/v1", func() {
		m.Get("/_ping", handler.GetPingV1Handler)

		m.Get("/users", handler.GetUsersV1Handler)
		m.Post("/users", handler.PostUsersV1Handler)

		m.Group("/repositories", func() {
			m.Put("/:namespace/:repository/tags/:tag", handler.PutTagV1Handler)
			m.Put("/:namespace/:repository/images", handler.PutRepositoryImagesV1Handler)
			m.Get("/:namespace/:repository/images", handler.GetRepositoryImagesV1Handler)
			m.Get("/:namespace/:repository/tags", handler.GetTagV1Handler)
			m.Put("/:namespace/:repository", handler.PutRepositoryV1Handler)
		})

		m.Group("/images", func() {
			m.Get("/:image/ancestry", handler.GetImageAncestryV1Handler)
			m.Get("/:image/json", handler.GetImageJSONV1Handler)
			m.Get("/:image/layer", handler.GetImageLayerV1Handler)
			m.Put("/:image/json", handler.PutImageJSONV1Handler)
			m.Put("/:image/layer", handler.PutImageLayerV1Handler)
			m.Put("/:image/checksum", handler.PutImageChecksumV1Handler)
		})
	})

	//Docker Registry V2
	m.Group("/v2", func() {
		m.Get("/", handler.GetPingV2Handler)
		m.Get("/_catalog", handler.GetCatalogV2Handler)

		//user mode: /namespace/repository:tag
		m.Head("/:namespace/:repository/blobs/:digest", handler.HeadBlobsV2Handler)
		m.Post("/:namespace/:repository/blobs/uploads", handler.PostBlobsV2Handler)
		m.Patch("/:namespace/:repository/blobs/uploads/:uuid", handler.PatchBlobsV2Handler)
		m.Put("/:namespace/:repository/blobs/uploads/:uuid", handler.PutBlobsV2Handler)
		m.Get("/:namespace/:repository/blobs/:digest", handler.GetBlobsV2Handler)
		m.Put("/:namespace/:repository/manifests/:tag", handler.PutManifestsV2Handler)
		m.Get("/:namespace/:repository/tags/list", handler.GetTagsListV2Handler)
		m.Get("/:namespace/:repository/manifests/:tag", handler.GetManifestsV2Handler)
		m.Delete("/:namespace/:repository/blobs/:digest", handler.DeleteBlobsV2Handler)
		m.Delete("/:namespace/:repository/manifests/:reference", handler.DeleteManifestsV2Handler)

		//library mode: /repository:tag
		m.Head("/:repository/blobs/:digest", handler.HeadBlobsV2LibraryHandler)
		m.Post("/:repository/blobs/uploads", handler.PostBlobsV2LibraryHandler)
		m.Patch("/:repository/blobs/uploads/:uuid", handler.PatchBlobsV2LibraryHandler)
		m.Put("/:repository/blobs/uploads/:uuid", handler.PutBlobsV2LibraryHandler)
		m.Get("/:repository/blobs/:digest", handler.GetBlobsV2LibraryHandler)
		m.Put("/:repository/manifests/:tag", handler.PutManifestsV2LibraryHandler)
		m.Get("/:repository/tags/list", handler.GetTagsListV2LibraryHandler)
		m.Get("/:repository/manifests/:tag", handler.GetManifestsV2LibraryHandler)
		m.Delete("/:repository/blobs/:digest", handler.DeleteBlobsV2LibraryHandler)
		m.Delete("/:repository/manifests/:reference", handler.DeleteManifestsV2LibraryHandler)
	})

	//App Discovery
	m.Group("/app", func() {
		m.Group("/v1", func() {
			//Global Search
			m.Get("/search", handler.AppGlobalSearchV1Handler)

			m.Group("/:namespace/:repository", func() {
				//Discovery
				m.Get("/?app-discovery=1", handler.AppDiscoveryV1Handler)

				//Scoped Search
				m.Get("/search", handler.AppScopedSearchV1Handlers)

				//Pull
				m.Get("/:os/:arch/:app", handler.AppGetFileV1Handler)

				//Push
				m.Post("/", handler.AppPostV1Handler)
				m.Put("/:os/:arch/:app", handler.AppPutFileV1Handler)
				m.Put("/:os/:arch/:app/manifests", handler.AppPutManifestV1Handler)
				m.Patch("/:os/:arch/:app/:status", handler.AppPatchFileV1Handler)
				m.Delete("/:os/:arch/:app", handler.AppDeleteFileV1Handler)
			})
		})
	})

	//Appc Discovery
	m.Group("/appc", func() {
		m.Group("/v1", func() {
			m.Group("/:namespace/:repository", func() {
				//Discovery
				m.Get("/?ac-discovery=1", handler.AppcDiscoveryV1Handler)

				//Pull
				m.Get("/pubkeys", handler.AppcGetPubkeysV1Handler)
				m.Get("/:os/:arch/:aci", handler.AppcGetACIV1Handler)
			})

		})
	})

	//VM Image Discovery
	m.Group("/image", func() {
		m.Group("/v1", func() {
			//Global Search
			m.Get("/search", handler.ImageGlobalSearchV1Handler)

			m.Group("/:namespace/:repository", func() {
				//Discovery
				m.Get("/?image-discovery=1", handler.ImageDiscoveryV1Handler)

				//Scoped Search
				m.Get("/search", handler.ImageScopedSearchV1Handlers)

				//Pull
				m.Get("/:os/:arch/:image", handler.ImageGetFileV1Handler)

				//Push
				m.Post("/", handler.ImagePostV1Handler)
				m.Put("/:os/:arch/:image", handler.ImagePutFileV1Handler)
				m.Put("/:os/:arch/:app/manifests", handler.ImagePutManifestV1Handler)
				m.Patch("/:os/:arch/:image/:status", handler.ImagePatchFileV1Handler)
				m.Delete("/:os/:arch/:image", handler.ImageDeleteFileV1Handler)
			})
		})
	})

	//Admin APIs
}
<commit_msg>Fix handler function name bug<commit_after>/*
Copyright 2015 The ContainerOps Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package router

import (
	"gopkg.in/macaron.v1"

	"github.com/containerops/dockyard/handler"
)

/*
  Dockyard Router Definition
*/
func SetRouters(m *macaron.Macaron) {
	//Web API
	m.Get("/", handler.IndexV1Handler)

	//Docker Registry V1
	m.Group("/v1", func() {
		m.Get("/_ping", handler.GetPingV1Handler)

		m.Get("/users", handler.GetUsersV1Handler)
		m.Post("/users", handler.PostUsersV1Handler)

		m.Group("/repositories", func() {
			m.Put("/:namespace/:repository/tags/:tag", handler.PutTagV1Handler)
			m.Put("/:namespace/:repository/images", handler.PutRepositoryImagesV1Handler)
			m.Get("/:namespace/:repository/images", handler.GetRepositoryImagesV1Handler)
			m.Get("/:namespace/:repository/tags", handler.GetTagV1Handler)
			m.Put("/:namespace/:repository", handler.PutRepositoryV1Handler)
		})

		m.Group("/images", func() {
			m.Get("/:image/ancestry", handler.GetImageAncestryV1Handler)
			m.Get("/:image/json", handler.GetImageJSONV1Handler)
			m.Get("/:image/layer", handler.GetImageLayerV1Handler)
			m.Put("/:image/json", handler.PutImageJSONV1Handler)
			m.Put("/:image/layer", handler.PutImageLayerV1Handler)
			m.Put("/:image/checksum", handler.PutImageChecksumV1Handler)
		})
	})

	//Docker Registry V2
	m.Group("/v2", func() {
		m.Get("/", handler.GetPingV2Handler)
		m.Get("/_catalog", handler.GetCatalogV2Handler)

		//user mode: /namespace/repository:tag
		m.Head("/:namespace/:repository/blobs/:digest", handler.HeadBlobsV2Handler)
		m.Post("/:namespace/:repository/blobs/uploads", handler.PostBlobsV2Handler)
		m.Patch("/:namespace/:repository/blobs/uploads/:uuid", handler.PatchBlobsV2Handler)
		m.Put("/:namespace/:repository/blobs/uploads/:uuid", handler.PutBlobsV2Handler)
		m.Get("/:namespace/:repository/blobs/:digest", handler.GetBlobsV2Handler)
		m.Put("/:namespace/:repository/manifests/:tag", handler.PutManifestsV2Handler)
		m.Get("/:namespace/:repository/tags/list", handler.GetTagsListV2Handler)
		m.Get("/:namespace/:repository/manifests/:tag", handler.GetManifestsV2Handler)
		m.Delete("/:namespace/:repository/blobs/:digest", handler.DeleteBlobsV2Handler)
		m.Delete("/:namespace/:repository/manifests/:reference", handler.DeleteManifestsV2Handler)

		//library mode: /repository:tag
		m.Head("/:repository/blobs/:digest", handler.HeadBlobsV2LibraryHandler)
		m.Post("/:repository/blobs/uploads", handler.PostBlobsV2LibraryHandler)
		m.Patch("/:repository/blobs/uploads/:uuid", handler.PatchBlobsV2LibraryHandler)
		m.Put("/:repository/blobs/uploads/:uuid", handler.PutBlobsV2LibraryHandler)
		m.Get("/:repository/blobs/:digest", handler.GetBlobsV2LibraryHandler)
		m.Put("/:repository/manifests/:tag", handler.PutManifestsV2LibraryHandler)
		m.Get("/:repository/tags/list", handler.GetTagsListV2LibraryHandler)
		m.Get("/:repository/manifests/:tag", handler.GetManifestsV2LibraryHandler)
		m.Delete("/:repository/blobs/:digest", handler.DeleteBlobsV2LibraryHandler)
		m.Delete("/:repository/manifests/:reference", handler.DeleteManifestsV2LibraryHandler)
	})

	//App Discovery
	m.Group("/app", func() {
		m.Group("/v1", func() {
			//Global Search
			m.Get("/search", handler.AppGlobalSearchV1Handler)

			m.Group("/:namespace/:repository", func() {
				//Discovery
				m.Get("/?app-discovery=1", handler.AppDiscoveryV1Handler)

				//Scoped Search
				m.Get("/search", handler.AppScopedSearchV1Handler)

				//Pull
				m.Get("/:os/:arch/:app", handler.AppGetFileV1Handler)

				//Push
				m.Post("/", handler.AppPostV1Handler)
				m.Put("/:os/:arch/:app", handler.AppPutFileV1Handler)
				m.Put("/:os/:arch/:app/manifests", handler.AppPutManifestV1Handler)
				m.Patch("/:os/:arch/:app/:status", handler.AppPatchFileV1Handler)
				m.Delete("/:os/:arch/:app", handler.AppDeleteFileV1Handler)
			})
		})
	})

	//Appc Discovery
	m.Group("/appc", func() {
		m.Group("/v1", func() {
			m.Group("/:namespace/:repository", func() {
				//Discovery
				m.Get("/?ac-discovery=1", handler.AppcDiscoveryV1Handler)

				//Pull
				m.Get("/pubkeys", handler.AppcGetPubkeysV1Handler)
				m.Get("/:os/:arch/:aci", handler.AppcGetACIV1Handler)
			})

		})
	})

	//VM Image Discovery
	m.Group("/image", func() {
		m.Group("/v1", func() {
			//Global Search
			m.Get("/search", handler.ImageGlobalSearchV1Handler)

			m.Group("/:namespace/:repository", func() {
				//Discovery
				m.Get("/?image-discovery=1", handler.ImageDiscoveryV1Handler)

				//Scoped Search
				m.Get("/search", handler.ImageScopedSearchV1Handler)

				//Pull
				m.Get("/:os/:arch/:image", handler.ImageGetFileV1Handler)

				//Push
				m.Post("/", handler.ImagePostV1Handler)
				m.Put("/:os/:arch/:image", handler.ImagePutFileV1Handler)
				m.Put("/:os/:arch/:app/manifests", handler.ImagePutManifestV1Handler)
				m.Patch("/:os/:arch/:image/:status", handler.ImagePatchFileV1Handler)
				m.Delete("/:os/:arch/:image", handler.ImageDeleteFileV1Handler)
			})
		})
	})

	//Admin APIs
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package leaktest provides tools to detect leaked goroutines in tests.
// To use it, call "defer leaktest.Check(t)()" at the beginning of each
// test that may use goroutines.
// copied out of the cockroachdb source tree with slight modifications to be
// more re-useable
package leaktest

import (
	"context"
	"fmt"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"
)

type goroutine struct {
	id    uint64
	stack string
}

type goroutineByID []*goroutine

func (g goroutineByID) Len() int           { return len(g) }
func (g goroutineByID) Less(i, j int) bool { return g[i].id < g[j].id }
func (g goroutineByID) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }

func interestingGoroutine(g string) (*goroutine, error) {
	sl := strings.SplitN(g, "\n", 2)
	if len(sl) != 2 {
		return nil, fmt.Errorf("error parsing stack: %q", g)
	}
	stack := strings.TrimSpace(sl[1])
	if strings.HasPrefix(stack, "testing.RunTests") {
		return nil, nil
	}

	if stack == "" ||
		// Below are the stacks ignored by the upstream leaktest code.
		strings.Contains(stack, "testing.Main(") ||
		strings.Contains(stack, "testing.(*T).Run(") ||
		strings.Contains(stack, "runtime.goexit") ||
		strings.Contains(stack, "created by runtime.gc") ||
		strings.Contains(stack, "interestingGoroutines") ||
		strings.Contains(stack, "runtime.MHeap_Scavenger") ||
		strings.Contains(stack, "signal.signal_recv") ||
		strings.Contains(stack, "sigterm.handler") ||
		strings.Contains(stack, "runtime_mcall") ||
		strings.Contains(stack, "goroutine in C code") {
		return nil, nil
	}

	// Parse the goroutine's ID from the header line.
	h := strings.SplitN(sl[0], " ", 3)
	if len(h) < 3 {
		return nil, fmt.Errorf("error parsing stack header: %q", sl[0])
	}
	id, err := strconv.ParseUint(h[1], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("error parsing goroutine id: %s", err)
	}

	return &goroutine{id: id, stack: strings.TrimSpace(g)}, nil
}

// interestingGoroutines returns all goroutines we care about for the purpose
// of leak checking. It excludes testing or runtime ones.
func interestingGoroutines(t ErrorReporter) []*goroutine {
	buf := make([]byte, 2<<20)
	buf = buf[:runtime.Stack(buf, true)]
	var gs []*goroutine
	for _, g := range strings.Split(string(buf), "\n\n") {
		gr, err := interestingGoroutine(g)
		if err != nil {
			t.Errorf("leaktest: %s", err)
			continue
		} else if gr == nil {
			continue
		}
		gs = append(gs, gr)
	}
	sort.Sort(goroutineByID(gs))
	return gs
}

// ErrorReporter is a tiny subset of a testing.TB to make testing not such a
// massive pain
type ErrorReporter interface {
	Errorf(format string, args ...interface{})
}

// Check snapshots the currently-running goroutines and returns a
// function to be run at the end of tests to see whether any
// goroutines leaked, waiting up to 5 seconds in error conditions
func Check(t ErrorReporter) func() {
	return CheckTimeout(t, 5*time.Second)
}

// CheckTimeout is the same as Check, but with a configurable timeout
func CheckTimeout(t ErrorReporter, dur time.Duration) func() {
	ctx, cancel := context.WithCancel(context.Background())
	fn := CheckContext(ctx, t)
	return func() {
		timer := time.AfterFunc(dur, cancel)
		fn()
		// Remember to clean up the timer and context
		timer.Stop()
		cancel()
	}
}

// CheckContext is the same as Check, but uses a context.Context for
// cancellation and timeout control
func CheckContext(ctx context.Context, t ErrorReporter) func() {
	orig := map[uint64]bool{}
	for _, g := range interestingGoroutines(t) {
		orig[g.id] = true
	}
	return func() {
		var leaked []string
		for {
			select {
			case <-ctx.Done():
				t.Errorf("leaktest: timed out checking goroutines")
			default:
				leaked = make([]string, 0)
				for _, g := range interestingGoroutines(t) {
					if !orig[g.id] {
						leaked = append(leaked, g.stack)
					}
				}
				if len(leaked) == 0 {
					return
				}
				// don't spin needlessly
				time.Sleep(time.Millisecond * 50)
				continue
			}
			break
		}
		for _, g := range leaked {
			t.Errorf("leaktest: leaked goroutine: %v", g)
		}
	}
}
<commit_msg>Ignore httpkeepalive in leaktest<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package leaktest provides tools to detect leaked goroutines in tests.
// To use it, call "defer leaktest.Check(t)()" at the beginning of each
// test that may use goroutines.
// copied out of the cockroachdb source tree with slight modifications to be
// more re-useable
package leaktest

import (
	"context"
	"fmt"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"
)

type goroutine struct {
	id    uint64
	stack string
}

type goroutineByID []*goroutine

func (g goroutineByID) Len() int           { return len(g) }
func (g goroutineByID) Less(i, j int) bool { return g[i].id < g[j].id }
func (g goroutineByID) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }

func interestingGoroutine(g string) (*goroutine, error) {
	sl := strings.SplitN(g, "\n", 2)
	if len(sl) != 2 {
		return nil, fmt.Errorf("error parsing stack: %q", g)
	}
	stack := strings.TrimSpace(sl[1])
	if strings.HasPrefix(stack, "testing.RunTests") {
		return nil, nil
	}

	if stack == "" ||
		// Ignore HTTP keep alives
		strings.Contains(stack, ").readLoop(") ||
		strings.Contains(stack, ").writeLoop(") ||
		// Below are the stacks ignored by the upstream leaktest code.
		strings.Contains(stack, "testing.Main(") ||
		strings.Contains(stack, "testing.(*T).Run(") ||
		strings.Contains(stack, "runtime.goexit") ||
		strings.Contains(stack, "created by runtime.gc") ||
		strings.Contains(stack, "interestingGoroutines") ||
		strings.Contains(stack, "runtime.MHeap_Scavenger") ||
		strings.Contains(stack, "signal.signal_recv") ||
		strings.Contains(stack, "sigterm.handler") ||
		strings.Contains(stack, "runtime_mcall") ||
		strings.Contains(stack, "goroutine in C code") {
		return nil, nil
	}

	// Parse the goroutine's ID from the header line.
	h := strings.SplitN(sl[0], " ", 3)
	if len(h) < 3 {
		return nil, fmt.Errorf("error parsing stack header: %q", sl[0])
	}
	id, err := strconv.ParseUint(h[1], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("error parsing goroutine id: %s", err)
	}

	return &goroutine{id: id, stack: strings.TrimSpace(g)}, nil
}

// interestingGoroutines returns all goroutines we care about for the purpose
// of leak checking. It excludes testing or runtime ones.
func interestingGoroutines(t ErrorReporter) []*goroutine {
	buf := make([]byte, 2<<20)
	buf = buf[:runtime.Stack(buf, true)]
	var gs []*goroutine
	for _, g := range strings.Split(string(buf), "\n\n") {
		gr, err := interestingGoroutine(g)
		if err != nil {
			t.Errorf("leaktest: %s", err)
			continue
		} else if gr == nil {
			continue
		}
		gs = append(gs, gr)
	}
	sort.Sort(goroutineByID(gs))
	return gs
}

// ErrorReporter is a tiny subset of a testing.TB to make testing not such a
// massive pain
type ErrorReporter interface {
	Errorf(format string, args ...interface{})
}

// Check snapshots the currently-running goroutines and returns a
// function to be run at the end of tests to see whether any
// goroutines leaked, waiting up to 5 seconds in error conditions
func Check(t ErrorReporter) func() {
	return CheckTimeout(t, 5*time.Second)
}

// CheckTimeout is the same as Check, but with a configurable timeout
func CheckTimeout(t ErrorReporter, dur time.Duration) func() {
	ctx, cancel := context.WithCancel(context.Background())
	fn := CheckContext(ctx, t)
	return func() {
		timer := time.AfterFunc(dur, cancel)
		fn()
		// Remember to clean up the timer and context
		timer.Stop()
		cancel()
	}
}

// CheckContext is the same as Check, but uses a context.Context for
// cancellation and timeout control
func CheckContext(ctx context.Context, t ErrorReporter) func() {
	orig := map[uint64]bool{}
	for _, g := range interestingGoroutines(t) {
		orig[g.id] = true
	}
	return func() {
		var leaked []string
		for {
			select {
			case <-ctx.Done():
				t.Errorf("leaktest: timed out checking goroutines")
			default:
				leaked = make([]string, 0)
				for _, g := range interestingGoroutines(t) {
					if !orig[g.id] {
						leaked = append(leaked, g.stack)
					}
				}
				if len(leaked) == 0 {
					return
				}
				// don't spin needlessly
				time.Sleep(time.Millisecond * 50)
				continue
			}
			break
		}
		for _, g := range leaked {
			t.Errorf("leaktest: leaked goroutine: %v", g)
		}
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"bytes"
	"encoding/base64"
	"github.com/ThomsonReutersEikon/go-ntlm/ntlm"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

func (c *Configuration) NTLMSession(creds Creds) ntlm.ClientSession {

	if c.ntlmSession != nil {
		return c.ntlmSession
	}

	splits := strings.Split(creds["username"], "\\")

	var session, _  = ntlm.CreateClientSession(ntlm.Version2, ntlm.ConnectionOrientedMode)
	session.SetUserInfo(splits[0], creds["password"], splits[1])

	c.ntlmSession = session

	return session
}

func DoNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
	handReq := cloneRequest(request)
	res, nil := InitHandShake(handReq)

	//If the status is 401 then we need to re-authenticate, otherwise it was successful
	if res.StatusCode == 401 {

		creds, _ := getCredsForNTLM(request)

		negotiateReq := cloneRequest(request)
		challengeMessage := negotiate(negotiateReq, getNegotiateMessage())

		challengeReq := cloneRequest(request)
		res, _ := challenge(challengeReq, challengeMessage, creds)

		//If the status is 401 then we need to re-authenticate
		if res.StatusCode == 401 && retry == true {
			return DoNTLMRequest(challengeReq, false)
		}

		saveCredentials(creds, res)

		return res, nil
	}
	return res, nil
}

func InitHandShake(request *http.Request) (*http.Response, error){
	var response, err = Config.HttpClient().Do(request)

	if err != nil {
		return nil, Error(err)
	}

	return response, nil
}

func negotiate(request *http.Request, message string) []byte{
	request.Header.Add("Authorization", message)
	var response, err = Config.HttpClient().Do(request)

	if err != nil{
		panic(err.Error())
	}

	ret := parseChallengeResponse(response)

	//Always close negotiate to keep the connection alive
	//We never return the response from negotiate so we
	//can't trust decodeApiResponse to close it
	io.Copy(ioutil.Discard, response.Body)
	response.Body.Close()

	return ret;
}

func challenge(request *http.Request, challengeBytes []byte, creds Creds) (*http.Response, error){
	challenge, err := ntlm.ParseChallengeMessage(challengeBytes)

	if err != nil {
		return nil, Error(err)
	}

	Config.NTLMSession(creds).ProcessChallengeMessage(challenge)
	authenticate, err := Config.NTLMSession(creds).GenerateAuthenticateMessage()

	if err != nil {
		return nil, Error(err)
	}

	authenticateMessage := concatS("NTLM ", base64.StdEncoding.EncodeToString(authenticate.Bytes()))

	request.Header.Add("Authorization", authenticateMessage)
	response, err := Config.HttpClient().Do(request)

	return response, nil
}

func parseChallengeResponse(response *http.Response) []byte{
	if headers, ok := response.Header["Www-Authenticate"]; ok{

		//parse out the "NTLM " at the beginning of the response
		challenge := headers[0][5:]
		val, err := base64.StdEncoding.DecodeString(challenge)

		if err != nil{
			panic(err.Error())
		}
		return []byte(val)
	}

	panic("www-Authenticate header is not present")
}

func cloneRequest(request *http.Request) *http.Request {
	var rdr1, rdr2 myReader
	var clonedReq *http.Request

	if request.Body != nil {
		//If we have a body (POST/PUT etc.)
		//We need to do some magic to copy the request without closing the body stream

		buf, _ := ioutil.ReadAll(request.Body)
		rdr1 = myReader{bytes.NewBuffer(buf)}
		rdr2 = myReader{bytes.NewBuffer(buf)}
		request.Body = rdr2 // OK since rdr2 implements the io.ReadCloser interface
		clonedReq, _ = http.NewRequest(request.Method, request.URL.String(), rdr1)
	}else{
		clonedReq, _ = http.NewRequest(request.Method, request.URL.String(), nil)
	}

	for k, v := range request.Header {
		clonedReq.Header.Add(k,v[0])
	}

	clonedReq.ContentLength = request.ContentLength

	return clonedReq
}

func getNegotiateMessage() string{
	return "NTLM TlRMTVNTUAABAAAAB7IIogwADAAzAAAACwALACgAAAAKAAAoAAAAD1dJTExISS1NQUlOTk9SVEhBTUVSSUNB"
}

func concatS(ar ...string) string {

    var buffer bytes.Buffer

    for _, s := range ar{
        buffer.WriteString(s)
    }

    return buffer.String()
}

func concat(ar ...[]byte) []byte {
	return bytes.Join(ar, nil)
}

type myReader struct {
    *bytes.Buffer
}

// So that myReader implements the io.ReadCloser interface
func (m myReader) Close() error { return nil } <commit_msg>Fix NTLM Domain Handling Logic<commit_after>package lfs

import (
	"bytes"
	"encoding/base64"
	"github.com/ThomsonReutersEikon/go-ntlm/ntlm"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

func (c *Configuration) NTLMSession(creds Creds) ntlm.ClientSession {

	if c.ntlmSession != nil {
		return c.ntlmSession
	}

	splits := strings.Split(creds["username"], "\\")
	var session, _  = ntlm.CreateClientSession(ntlm.Version2, ntlm.ConnectionOrientedMode)
	session.SetUserInfo(splits[1], creds["password"], strings.ToUpper(splits[0]))

	c.ntlmSession = session

	return session
}

func DoNTLMRequest(request *http.Request, retry bool) (*http.Response, error) {
	handReq := cloneRequest(request)
	res, nil := InitHandShake(handReq)

	//If the status is 401 then we need to re-authenticate, otherwise it was successful
	if res.StatusCode == 401 {

		creds, _ := getCredsForNTLM(request)

		negotiateReq := cloneRequest(request)
		challengeMessage := negotiate(negotiateReq, getNegotiateMessage())

		challengeReq := cloneRequest(request)
		res, _ := challenge(challengeReq, challengeMessage, creds)

		//If the status is 401 then we need to re-authenticate
		if res.StatusCode == 401 && retry == true {
			return DoNTLMRequest(challengeReq, false)
		}

		saveCredentials(creds, res)

		return res, nil
	}
	return res, nil
}

func InitHandShake(request *http.Request) (*http.Response, error){
	var response, err = Config.HttpClient().Do(request)

	if err != nil {
		return nil, Error(err)
	}

	return response, nil
}

func negotiate(request *http.Request, message string) []byte{
	request.Header.Add("Authorization", message)
	var response, err = Config.HttpClient().Do(request)

	if err != nil{
		panic(err.Error())
	}

	ret := parseChallengeResponse(response)

	//Always close negotiate to keep the connection alive
	//We never return the response from negotiate so we
	//can't trust decodeApiResponse to close it
	io.Copy(ioutil.Discard, response.Body)
	response.Body.Close()

	return ret;
}

func challenge(request *http.Request, challengeBytes []byte, creds Creds) (*http.Response, error){
	challenge, err := ntlm.ParseChallengeMessage(challengeBytes)

	if err != nil {
		return nil, Error(err)
	}

	Config.NTLMSession(creds).ProcessChallengeMessage(challenge)
	authenticate, err := Config.NTLMSession(creds).GenerateAuthenticateMessage()

	if err != nil {
		return nil, Error(err)
	}

	authenticateMessage := concatS("NTLM ", base64.StdEncoding.EncodeToString(authenticate.Bytes()))

	request.Header.Add("Authorization", authenticateMessage)
	response, err := Config.HttpClient().Do(request)

	return response, nil
}

func parseChallengeResponse(response *http.Response) []byte{
	if headers, ok := response.Header["Www-Authenticate"]; ok{

		//parse out the "NTLM " at the beginning of the response
		challenge := headers[0][5:]
		val, err := base64.StdEncoding.DecodeString(challenge)

		if err != nil{
			panic(err.Error())
		}
		return []byte(val)
	}

	panic("www-Authenticate header is not present")
}

func cloneRequest(request *http.Request) *http.Request {
	var rdr1, rdr2 myReader
	var clonedReq *http.Request

	if request.Body != nil {
		//If we have a body (POST/PUT etc.)
		//We need to do some magic to copy the request without closing the body stream

		buf, _ := ioutil.ReadAll(request.Body)
		rdr1 = myReader{bytes.NewBuffer(buf)}
		rdr2 = myReader{bytes.NewBuffer(buf)}
		request.Body = rdr2 // OK since rdr2 implements the io.ReadCloser interface
		clonedReq, _ = http.NewRequest(request.Method, request.URL.String(), rdr1)
	}else{
		clonedReq, _ = http.NewRequest(request.Method, request.URL.String(), nil)
	}

	for k, v := range request.Header {
		clonedReq.Header.Add(k,v[0])
	}

	clonedReq.ContentLength = request.ContentLength

	return clonedReq
}

func getNegotiateMessage() string{
	return "NTLM TlRMTVNTUAABAAAAB7IIogwADAAzAAAACwALACgAAAAKAAAoAAAAD1dJTExISS1NQUlOTk9SVEhBTUVSSUNB"
}

func concatS(ar ...string) string {

    var buffer bytes.Buffer

    for _, s := range ar{
        buffer.WriteString(s)
    }

    return buffer.String()
}

func concat(ar ...[]byte) []byte {
	return bytes.Join(ar, nil)
}

type myReader struct {
    *bytes.Buffer
}

// So that myReader implements the io.ReadCloser interface
func (m myReader) Close() error { return nil } <|endoftext|>


<commit_before>package citadel

import (
	"errors"
	"log"
	"strconv"
	"strings"
	"sync"

	"github.com/rcrowley/go-metrics"
	"github.com/samalba/dockerclient"
)

var (
	ErrNoSchedulerForType = errors.New("no scheduler installed for container type")
	ErrUnableToSchedule   = errors.New("unable to schedule container")
	ErrSchedulerExists    = errors.New("scheduler exists for type")
)

// ClusterManager manages changes to the state of the cluster
type ClusterManager struct {
	engines         map[string]*Engine
	resourceManager *ResourceManager

	schedulers map[string]Scheduler

	// timer records the time it takes to comple a scheduling operation
	timer metrics.Timer

	logger *log.Logger
	mux    sync.Mutex
}

// NewClusterManager returns a new cluster manager initialized with the registry
// and a logger
func NewClusterManager(engines []*Engine, logger *log.Logger) *ClusterManager {
	m := &ClusterManager{
		engines:         make(map[string]*Engine),
		schedulers:      make(map[string]Scheduler),
		resourceManager: newEngineManger(logger),
		logger:          logger,
		timer:           metrics.NewTimer(),
	}

	for _, e := range engines {
		m.engines[e.ID] = e
	}

	metrics.Register("citadel-timer", m.timer)

	return m
}

// ScheduleContainer uses the schedulers registered with the cluster and finds
// a resource that is able to run the container.
//
// If not scheduling decision can be made an ErrUnableToSchedule error is returned.
func (m *ClusterManager) ScheduleContainer(c *Container) (*Transaction, error) {
	if err := ValidateContainer(c); err != nil {
		return nil, err
	}

	m.mux.Lock()

	t := newTransaction(c)
	defer func() {
		t.Close()
		m.mux.Unlock()
		m.timer.UpdateSince(t.Started)
	}()

	m.logger.Printf("task=%q image=%q cpus=%f memory=%f type=%q\n", "schedule", c.Image, c.Cpus, c.Memory, c.Type)

	// find the correct scheduler for the container's type
	scheduler := m.schedulers[c.Type]

	if scheduler == nil {
		return nil, ErrNoSchedulerForType
	}

	accepted := []*Engine{}

	for _, e := range m.engines {
		// ensure that we preload all the containers for an engine to be used in the scheduling decison
		if err := e.loadContainers(); err != nil {
			return nil, err
		}

		canrun, err := scheduler.Schedule(c, e)
		if err != nil {
			return nil, err
		}

		if canrun {
			accepted = append(accepted, e)
		}
	}

	m.logger.Printf("task=%q image=%q resource.count=%d\n", "schedule", c.Image, len(accepted))

	// check with the resource manager to ensure that the engines that the scheduler is able
	// to run the container and to place the container on the resource with the best utilization
	// score to maximize effenciency
	engine, err := m.resourceManager.PlaceContainer(c, accepted)
	if err != nil {
		return nil, err
	}

	m.logger.Printf("task=%q image=%q placement=%q\n", "schedule", c.Image, engine.Addr)

	t.place(engine)

	if err := m.runContainer(t); err != nil {
		return nil, err
	}

	return t, nil
}

// AddEngine adds a new engine to the cluster for use
func (m *ClusterManager) AddEngine(e *Engine) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	m.engines[e.ID] = e

	return nil

}

// RemoveEngine removes the engine from the cluster
func (m *ClusterManager) RemoveEngine(e *Engine) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	delete(m.engines, e.ID)

	return nil
}

// RegisterScheduler registers the scheduler for a specific container type within the
// cluster.  An ErrSchedulerExists error is returned if the cluster already has a
// scheduler registered for that specific type.
func (m *ClusterManager) RegisterScheduler(tpe string, s Scheduler) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	m.logger.Printf("task=%q type=%q", "register\n", tpe)

	if _, exists := m.schedulers[tpe]; exists {
		return ErrSchedulerExists
	}

	m.schedulers[tpe] = s

	return nil
}

func (m *ClusterManager) runContainer(t *Transaction) error {
	if err := t.Container.Run(t.Placement.Engine); err != nil {
		return err
	}

	info, err := t.Placement.Engine.client.InspectContainer(t.Container.Name)
	if err != nil {
		return err
	}

	t.Placement.InternalIP = info.NetworkSettings.IpAddress

	for pp, b := range info.NetworkSettings.Ports {
		proto := strings.Split(pp, "/")[1]
		port, err := strconv.Atoi(b[0].HostPort)
		if err != nil {
			return err
		}

		t.Placement.Ports = append(t.Placement.Ports, &Port{
			Proto: proto,
			Port:  port,
		})
	}

	return nil
}

// ListContainers returns all the running containers in the cluster
func (m *ClusterManager) ListContainers() ([]*Container, error) {
	m.mux.Lock()
	defer m.mux.Unlock()

	containers := []*Container{}

	for _, engine := range m.engines {
		if err := engine.loadContainers(); err != nil {
			return nil, err
		}

		containers = append(containers, engine.containers...)
	}

	return containers, nil
}

// RemoveContainer will itterate over all the engines in the cluster and first kill
// the container then remove it complete from the engine
func (m *ClusterManager) RemoveContainer(c *Container) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	for _, engine := range m.engines {
		if err := c.Kill(engine); err != nil {
			if err == dockerclient.ErrNotFound {
				continue
			}

			return err
		}

		return engine.client.RemoveContainer(c.Name)
	}

	return nil
}
<commit_msg>remove containers across engines<commit_after>package citadel

import (
	"errors"
	"log"
	"strconv"
	"strings"
	"sync"

	"github.com/rcrowley/go-metrics"
	"github.com/samalba/dockerclient"
)

var (
	ErrNoSchedulerForType = errors.New("no scheduler installed for container type")
	ErrUnableToSchedule   = errors.New("unable to schedule container")
	ErrSchedulerExists    = errors.New("scheduler exists for type")
)

// ClusterManager manages changes to the state of the cluster
type ClusterManager struct {
	engines         map[string]*Engine
	resourceManager *ResourceManager

	schedulers map[string]Scheduler

	// timer records the time it takes to comple a scheduling operation
	timer metrics.Timer

	logger *log.Logger
	mux    sync.Mutex
}

// NewClusterManager returns a new cluster manager initialized with the registry
// and a logger
func NewClusterManager(engines []*Engine, logger *log.Logger) *ClusterManager {
	m := &ClusterManager{
		engines:         make(map[string]*Engine),
		schedulers:      make(map[string]Scheduler),
		resourceManager: newEngineManger(logger),
		logger:          logger,
		timer:           metrics.NewTimer(),
	}

	for _, e := range engines {
		m.engines[e.ID] = e
	}

	metrics.Register("citadel-timer", m.timer)

	return m
}

// ScheduleContainer uses the schedulers registered with the cluster and finds
// a resource that is able to run the container.
//
// If not scheduling decision can be made an ErrUnableToSchedule error is returned.
func (m *ClusterManager) ScheduleContainer(c *Container) (*Transaction, error) {
	if err := ValidateContainer(c); err != nil {
		return nil, err
	}

	m.mux.Lock()

	t := newTransaction(c)
	defer func() {
		t.Close()
		m.mux.Unlock()
		m.timer.UpdateSince(t.Started)
	}()

	m.logger.Printf("task=%q image=%q cpus=%f memory=%f type=%q\n", "schedule", c.Image, c.Cpus, c.Memory, c.Type)

	// find the correct scheduler for the container's type
	scheduler := m.schedulers[c.Type]

	if scheduler == nil {
		return nil, ErrNoSchedulerForType
	}

	accepted := []*Engine{}

	for _, e := range m.engines {
		// ensure that we preload all the containers for an engine to be used in the scheduling decison
		if err := e.loadContainers(); err != nil {
			return nil, err
		}

		canrun, err := scheduler.Schedule(c, e)
		if err != nil {
			return nil, err
		}

		if canrun {
			accepted = append(accepted, e)
		}
	}

	m.logger.Printf("task=%q image=%q resource.count=%d\n", "schedule", c.Image, len(accepted))

	// check with the resource manager to ensure that the engines that the scheduler is able
	// to run the container and to place the container on the resource with the best utilization
	// score to maximize effenciency
	engine, err := m.resourceManager.PlaceContainer(c, accepted)
	if err != nil {
		return nil, err
	}

	m.logger.Printf("task=%q image=%q placement=%q\n", "schedule", c.Image, engine.Addr)

	t.place(engine)

	if err := m.runContainer(t); err != nil {
		return nil, err
	}

	return t, nil
}

// AddEngine adds a new engine to the cluster for use
func (m *ClusterManager) AddEngine(e *Engine) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	m.engines[e.ID] = e

	return nil

}

// RemoveEngine removes the engine from the cluster
func (m *ClusterManager) RemoveEngine(e *Engine) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	delete(m.engines, e.ID)

	return nil
}

// RegisterScheduler registers the scheduler for a specific container type within the
// cluster.  An ErrSchedulerExists error is returned if the cluster already has a
// scheduler registered for that specific type.
func (m *ClusterManager) RegisterScheduler(tpe string, s Scheduler) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	m.logger.Printf("task=%q type=%q", "register\n", tpe)

	if _, exists := m.schedulers[tpe]; exists {
		return ErrSchedulerExists
	}

	m.schedulers[tpe] = s

	return nil
}

func (m *ClusterManager) runContainer(t *Transaction) error {
	if err := t.Container.Run(t.Placement.Engine); err != nil {
		return err
	}

	info, err := t.Placement.Engine.client.InspectContainer(t.Container.Name)
	if err != nil {
		return err
	}

	t.Placement.InternalIP = info.NetworkSettings.IpAddress

	for pp, b := range info.NetworkSettings.Ports {
		proto := strings.Split(pp, "/")[1]
		port, err := strconv.Atoi(b[0].HostPort)
		if err != nil {
			return err
		}

		t.Placement.Ports = append(t.Placement.Ports, &Port{
			Proto: proto,
			Port:  port,
		})
	}

	return nil
}

// ListContainers returns all the running containers in the cluster
func (m *ClusterManager) ListContainers() ([]*Container, error) {
	m.mux.Lock()
	defer m.mux.Unlock()

	containers := []*Container{}

	for _, engine := range m.engines {
		if err := engine.loadContainers(); err != nil {
			return nil, err
		}

		containers = append(containers, engine.containers...)
	}

	return containers, nil
}

// RemoveContainer will iterate over all the engines in the cluster and first kill
// the container then remove it complete from the engine
func (m *ClusterManager) RemoveContainer(c *Container) error {
	m.mux.Lock()
	defer m.mux.Unlock()

	for _, engine := range m.engines {
		if err := c.Kill(engine); err != nil {
			if err == dockerclient.ErrNotFound {
				continue
			}

			return err
		}

		if err := engine.client.RemoveContainer(c.Name); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/gnickm/gobt/bencode"
	"github.com/gnickm/gobt/bt"
	"io"
	"net"
	"net/http"
	"strconv"
	"strings"
)

type AnnounceRequest struct {
	peerId      bt.PeerId
	infoHash    bt.InfoHash
	ip          net.IP
	port        int
	downloaded  int
	left        int
	uploaded    int
	eventString string
	compactMode bool
	respChan    chan bencode.BEDictionary
}

func main() {
	announceReqChan := make(chan AnnounceRequest)

	go requestProcessor(announceReqChan)

	http.HandleFunc("/announce", makeAnnounceHandler(announceReqChan))
	http.ListenAndServe(":80", nil)
}

func makeAnnounceHandler(reqChan chan AnnounceRequest) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var responseDict bencode.BEDictionary
		ar, err := parseAnnounceRequest(r)
		if err != nil {
			responseDict = bencode.BEDictionary{}
			responseDict["failure message"] = bencode.BEString(err.Error())
		} else {
			reqChan <- *ar
			responseDict = <-ar.respChan
		}

		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		io.WriteString(w, responseDict.BEncode())
	}
}

func parseAnnounceRequest(r *http.Request) (*AnnounceRequest, error) {
	query := r.URL.Query()

	// --- Handle 'info_hash' parameter
	if query.Get("info_hash") == "" {
		return nil, errors.New("Missing required parameter 'info_hash'")
	}
	infoHash := bt.InfoHash(query.Get("info_hash"))
	if !infoHash.Validate() {
		return nil, errors.New("Invalid value in 'info_hash'")
	}

	// --- Handle 'peer_id' parameter
	if query.Get("peer_id") == "" {
		return nil, errors.New("Missing required parameter 'peer_id'")
	}
	peerId := bt.PeerId(query.Get("peer_id"))
	if !peerId.Validate() {
		return nil, errors.New("Invalid value in 'peer_id'")
	}

	// --- Handle 'port' parameter
	if query.Get("port") == "" {
		return nil, errors.New("Missing required parameter 'port'")
	}
	port, err := strconv.Atoi(query.Get("port"))
	if err != nil || port < 1 || port > 65535 {
		return nil, errors.New("Invalid value in 'port'")
	}

	// --- Handle optional 'ip' parameter and find peer's IP
	var ip net.IP
	if query.Get("ip") == "" {
		// Use the request remote address if not specified
		chunks := strings.Split(":", r.RemoteAddr)
		ip = net.ParseIP(chunks[0])
	} else {
		ip = net.ParseIP(query.Get("ip"))
	}
	if ip == nil {
		return nil, errors.New("Invalid peer IP address")
	}

	downloaded, _ := strconv.Atoi(query.Get("downloaded"))
	left, _ := strconv.Atoi(query.Get("left"))
	uploaded, _ := strconv.Atoi(query.Get("uploaded"))

	ar := AnnounceRequest{
		peerId,
		infoHash,
		ip,
		port,
		downloaded,
		left,
		uploaded,
		query.Get("eventString"),
		query.Get("compact") == "1",
		make(chan bencode.BEDictionary),
	}

	return &ar, nil
}

func requestProcessor(announceReqChan chan AnnounceRequest) {
	peerMap := make(map[bt.PeerId]bt.Peer)
	for {
		var peer bt.Peer
		var ok bool

		// Block until we get an announce request
		ar := <-announceReqChan

		// Look up the peer, add it if missing
		if peer, ok = peerMap[ar.peerId]; !ok {
			peer = bt.Peer{ar.peerId, ar.ip, ar.port, []bt.InfoHash{}}
			peerMap[ar.peerId] = peer
		}
		peer.AddInfoHash(ar.infoHash)
	}
}
<commit_msg>WIP: finished requestProcessor(), next up pickPeers()<commit_after>package main

import (
	"errors"
	"github.com/gnickm/gobt/bencode"
	"github.com/gnickm/gobt/bt"
	"io"
	"net"
	"net/http"
	"strconv"
	"strings"
)

type AnnounceRequest struct {
	peerId      bt.PeerId
	infoHash    bt.InfoHash
	ip          net.IP
	port        int
	downloaded  int
	left        int
	uploaded    int
	eventString string
	compactMode bool
	respChan    chan bencode.BEDictionary
}

type PeerEntry struct {
	peer       *bt.Peer
	downloaded int
	left       int
	uploaded   int
}

func main() {
	announceReqChan := make(chan AnnounceRequest)

	go requestProcessor(announceReqChan)

	http.HandleFunc("/announce", makeAnnounceHandler(announceReqChan))
	http.ListenAndServe(":80", nil)
}

func makeAnnounceHandler(reqChan chan AnnounceRequest) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var responseDict bencode.BEDictionary
		ar, err := parseAnnounceRequest(r)
		if err != nil {
			responseDict = bencode.BEDictionary{}
			responseDict["failure message"] = bencode.BEString(err.Error())
		} else {
			reqChan <- *ar
			responseDict = <-ar.respChan
		}

		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		io.WriteString(w, responseDict.BEncode())
	}
}

func parseAnnounceRequest(r *http.Request) (*AnnounceRequest, error) {
	query := r.URL.Query()

	// --- Handle 'info_hash' parameter
	if query.Get("info_hash") == "" {
		return nil, errors.New("Missing required parameter 'info_hash'")
	}
	infoHash := bt.InfoHash(query.Get("info_hash"))
	if !infoHash.Validate() {
		return nil, errors.New("Invalid value in 'info_hash'")
	}

	// --- Handle 'peer_id' parameter
	if query.Get("peer_id") == "" {
		return nil, errors.New("Missing required parameter 'peer_id'")
	}
	peerId := bt.PeerId(query.Get("peer_id"))
	if !peerId.Validate() {
		return nil, errors.New("Invalid value in 'peer_id'")
	}

	// --- Handle 'port' parameter
	if query.Get("port") == "" {
		return nil, errors.New("Missing required parameter 'port'")
	}
	port, err := strconv.Atoi(query.Get("port"))
	if err != nil || port < 1 || port > 65535 {
		return nil, errors.New("Invalid value in 'port'")
	}

	// --- Handle optional 'ip' parameter and find peer's IP
	var ip net.IP
	if query.Get("ip") == "" {
		// Use the request remote address if not specified
		chunks := strings.Split(":", r.RemoteAddr)
		ip = net.ParseIP(chunks[0])
	} else {
		ip = net.ParseIP(query.Get("ip"))
	}
	if ip == nil {
		return nil, errors.New("Invalid peer IP address")
	}

	downloaded, _ := strconv.Atoi(query.Get("downloaded"))
	left, _ := strconv.Atoi(query.Get("left"))
	uploaded, _ := strconv.Atoi(query.Get("uploaded"))

	ar := AnnounceRequest{
		peerId,
		infoHash,
		ip,
		port,
		downloaded,
		left,
		uploaded,
		query.Get("eventString"),
		query.Get("compact") == "1",
		make(chan bencode.BEDictionary),
	}

	return &ar, nil
}

func requestProcessor(announceReqChan chan AnnounceRequest) {
	torrentMap := make(map[bt.InfoHash]map[bt.PeerId]PeerEntry)
	for {
		var peerEntry PeerEntry
		var peerMap map[bt.PeerId]PeerEntry
		var ok bool
		var announceResp bencode.BEDictionary

		// Block until we get an announce request
		ar := <-announceReqChan

		// Look up peer map for this torrent, add it if missing
		if peerMap, ok = torrentMap[ar.infoHash]; !ok {
			peerMap = make(map[bt.PeerId]PeerEntry)
		}

		// Look up the peer, add it if missing
		if peerEntry, ok = peerMap[ar.peerId]; !ok {
			peer := bt.Peer{ar.peerId, ar.ip, ar.port, []bt.InfoHash{}}
			peerEntry := PeerEntry{
				&peer,
				ar.downloaded,
				ar.left,
				ar.uploaded,
			}
			peerMap[ar.peerId] = peerEntry
		}
		peerEntry.peer.AddInfoHash(ar.infoHash)

		pickedPeerList := pickPeers(peerMap)

		if ar.compactMode {
			ar.respChan <- makeCompactAnnounceResponse(pickedPeerList)
		} else {
			ar.respChan <- makeFullAnnounceResponse(pickedPeerList)
		}
	}
}
<|endoftext|>


<commit_before>package tracking

import (
	"encoding/json"
	"net/http"
	"time"

	"gopkg.in/mgo.v2/bson"
)

// Route represents a set of coordinates to draw a path on our tracking map
type Route struct {
	Name        string    `json:"name"           bson:"name"`
	Description string    `json:"description"    bson:"description"`
	StartTime   string    `json:"startTime"      bson:"startTime"`
	EndTime     string    `json:"endTime"        bson:"endTime"`
	Enabled     bool      `json:"enabled,string" bson:"enabled"`
	Color       string    `json:"color"          bson:"color"`
	Width       int       `json:"width,string"   bson:"width"`
	Coords 		  string    `json:"coords"         bson:"coords"`
	Created     time.Time `json:"created"        bson:"created"`
	Updated     time.Time `json:"updated"        bson:"updated"`
}

// Stop indicates where a tracked object is scheduled to arrive
type Stop struct {
	Name        string    `json:"name"        bson:"name"`
	Phonetic    string    `json:"phonetic"    bson:"phonetic"`
	Description string    `json:"description" bson:"description"`
	Address     string    `json:"address"     bson:"address"`
	TimeServed  string    `json:"timeServed"  bson:"timeServed"`
	Lat         float64   `json:"lat"         bson:"lat"`
	Lng         float64   `json:"lng"         bson:"lng"`
	Enabled     bool      `json:"enabled"     bson:"enabled"`
}

// RoutesHandler finds all of the routes in the database
func (App *App) RoutesHandler(w http.ResponseWriter, r *http.Request) {
	// Find all routes in database
	var routes []Route
	err := App.Routes.Find(bson.M{}).All(&routes)
	// Handle query errors
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	// Send each route to client as JSON
	WriteJSON(w, routes)
}

// RoutesCreateHandler adds a new route to the database
func (App *App) RoutesCreateHandler(w http.ResponseWriter, r *http.Request) {
	// Create a new route object using request fields
	route := Route{}
	err := json.NewDecoder(r.Body).Decode(&route)

	// Error handling
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	// Store new route under routes collection
	err = App.Routes.Insert(&route)
	// Error handling
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
<commit_msg>Gofmt<commit_after>package tracking

import (
	"encoding/json"
	"net/http"
	"time"

	"gopkg.in/mgo.v2/bson"
)

// Route represents a set of coordinates to draw a path on our tracking map
type Route struct {
	Name        string    `json:"name"           bson:"name"`
	Description string    `json:"description"    bson:"description"`
	StartTime   string    `json:"startTime"      bson:"startTime"`
	EndTime     string    `json:"endTime"        bson:"endTime"`
	Enabled     bool      `json:"enabled,string" bson:"enabled"`
	Color       string    `json:"color"          bson:"color"`
	Width       int       `json:"width,string"   bson:"width"`
	Coords      string    `json:"coords"         bson:"coords"`
	Created     time.Time `json:"created"        bson:"created"`
	Updated     time.Time `json:"updated"        bson:"updated"`
}

// Stop indicates where a tracked object is scheduled to arrive
type Stop struct {
	Name        string  `json:"name"        bson:"name"`
	Phonetic    string  `json:"phonetic"    bson:"phonetic"`
	Description string  `json:"description" bson:"description"`
	Address     string  `json:"address"     bson:"address"`
	TimeServed  string  `json:"timeServed"  bson:"timeServed"`
	Lat         float64 `json:"lat"         bson:"lat"`
	Lng         float64 `json:"lng"         bson:"lng"`
	Enabled     bool    `json:"enabled"     bson:"enabled"`
}

// RoutesHandler finds all of the routes in the database
func (App *App) RoutesHandler(w http.ResponseWriter, r *http.Request) {
	// Find all routes in database
	var routes []Route
	err := App.Routes.Find(bson.M{}).All(&routes)
	// Handle query errors
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	// Send each route to client as JSON
	WriteJSON(w, routes)
}

// RoutesCreateHandler adds a new route to the database
func (App *App) RoutesCreateHandler(w http.ResponseWriter, r *http.Request) {
	// Create a new route object using request fields
	route := Route{}
	err := json.NewDecoder(r.Body).Decode(&route)

	// Error handling
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	// Store new route under routes collection
	err = App.Routes.Insert(&route)
	// Error handling
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>// mouseAnalyzer determines vesicle release events and latencies for our
// mouse NMJ 6 AZ model with two synaptic vesicles each according to the
// second sensor faciliation model (see Ma et al., J. Neurophys, 2014)
package main

import (
	"flag"
	"fmt"
	"github.com/haskelladdict/mbdr/libmbd"
	"log"
	"math"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	numPulses      int     // number of AP pulses
	energyModel    bool    // use the energy model
	sytEnergy      int     // energy of activated synaptotagmin toward vesicle fusion
	yEnergy        int     // energy of activated Y sites toward vesicle fusion
	numActiveSites int     // number of simultaneously active sites required for release
	isiValue       float64 // interstimulus interval
)

func init() {
	flag.IntVar(&numPulses, "p", 1, "number of AP pulses in the model")
	flag.IntVar(&sytEnergy, "s", -1, "energy of active synaptotagmin sites "+
		"(required with -e flag)")
	flag.IntVar(&yEnergy, "y", -1, "energy of active y sites "+
		"(required with -e flag")
	flag.BoolVar(&energyModel, "e", false, "use the energy model instead of "+
		"deterministic model")
	flag.IntVar(&numActiveSites, "n", 0, "number of sites required for activation"+
		"for deterministic model")
	flag.Float64Var(&isiValue, "i", -1.0, "pulse duration in [s] for analysis multi "+
		"pulse data")
}

// usage prints a brief usage information to stdout
func usage() {
	fmt.Println("usage: mouseAnalyzer [options] <binary mcell files>")
	fmt.Println("\noptions:")
	flag.PrintDefaults()
}

// extractSeed attempts to extract the seed from the filename of the provided
// binary mcell data file.
// NOTE: the following filenaming convention is assumed *.<seedIDString>.bin.(gz|bz2)
func extractSeed(fileName string) (int, error) {
	items := strings.Split(fileName, ".")
	if len(items) <= 3 {
		return -1, fmt.Errorf("incorrectly formatted fileName. " +
			"Expected *.<seedIDString>.bin.(gz|bz2)")
	}

	for i := len(items) - 1; i >= 0; i-- {
		if items[i] == "bin" && i >= 1 {
			seed, err := strconv.Atoi(items[i-1])
			if err != nil {
				return -1, err
			}
			return seed, nil
		}
	}
	return -1, fmt.Errorf("Unable to extract seed id from filename ", fileName)
}

// printReleases prints a summary statistic for all released vesicle for a
// given seed
func printReleases(data *libmbd.MCellData, seed int, rel []*ReleaseEvent) {
	fmt.Println("mouseAnalyzer ran on", time.Now())
	if host, err := os.Hostname(); err == nil {
		fmt.Println("on ", host)
	}
	fmt.Println("\n-------------- parameters --------------")
	fmt.Println("number of pulses       : ", numPulses)
	fmt.Println("ISI                    : ", isiValue)
	if energyModel {
		fmt.Println("model                  : energy model")
		fmt.Println("syt energy             : ", sytEnergy)
		fmt.Println("y energy               : ", yEnergy)
	} else {
		fmt.Println("model                  : deterministic model")
		fmt.Println("number of active sites : ", numActiveSites)
	}
	fmt.Println("------------- data --------------------\n")
	timeStep := data.StepSize()
	for _, r := range rel {

		channels, err := determineCaChanContrib(data, r)
		if err != nil {
			log.Fatal(err)
		}

		eventTime := float64(r.eventIter) * timeStep
		pulseID := int(math.Floor(eventTime/isiValue)) + 1

		fmt.Printf("seed : %d   AZ : %d   ves : %d   time : %e   pulseID : %d", seed,
			r.azId+1, r.vesicleID+1, eventTime, pulseID)
		fmt.Printf("  sensors: [")
		for _, s := range r.sensors {
			fmt.Printf("%d ", s)
		}
		fmt.Printf("]")
		fmt.Printf("  channels: [")
		for n, c := range channels {
			fmt.Printf("%s : %d  ", n, int(c))
		}
		fmt.Printf("]")
		fmt.Printf("\n")
	}
}

// determineCaContrib determines which Ca channels contributed to the release
// of a particular vesicle.
// NOTE: We try to be as agnostic as we can in terms of the particular
// nomenclature used for naming the channels. However, the expectation is
// that data files tracking Ca binding to vesicles are named
// vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func determineCaChanContrib(data *libmbd.MCellData, rel *ReleaseEvent) (map[string]float64, error) {
	channels := make(map[string]float64)
	// the az/channel counting is 1 based whereas our internal counting is 0 based
	regexString := fmt.Sprintf("vesicle_%d_%d_ca_.*", rel.azId+1, rel.vesicleID+1)
	counts, err := data.BlockDataByRegex(regexString)
	if err != nil {
		return nil, err
	}
	for k, c := range counts {
		if len(c.Col) != 1 {
			return nil, fmt.Errorf("data set %s has more than the expected 1 column",
				k)
		}
		if c.Col[0][rel.eventIter] > 0 {
			// need to subtract 2 from regexString due to the extra ".*"
			caString, err := extractCaChanName(k, len(regexString)-2)
			if err != nil {
				return nil, err
			}
			channels[caString] = c.Col[0][rel.eventIter]
		}
	}

	return channels, nil
}

// extractCaChanName attempts to extract the name of the calcium channel based
// on the expected data name pattern vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func extractCaChanName(name string, prefixLength int) (string, error) {
	caName := name[prefixLength:]
	items := strings.Split(caName, ".")
	if len(items) == 0 {
		return "", fmt.Errorf("Could not determine Ca channel name from data set %s", name)
	}
	return items[0], nil
}

// main entry point
func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		usage()
		return
	}

	// some sanity checks
	if energyModel && (sytEnergy < 0 || yEnergy < 0) {
		log.Fatal("Please provide a non-negative synaptotagmin and y site energy")
	}

	if !energyModel && numActiveSites == 0 {
		log.Fatal("Please provide a positive count for the number of required active sites")
	}

	if numPulses > 1 && isiValue <= 0 {
		log.Fatal("Analysis multi-pulse data requires a non-zero ISI value.")
	}

	// loop over all provided data sets
	for _, fileName := range flag.Args() {

		seed, err := extractSeed(fileName)
		if err != nil {
			log.Fatal(err)
		}

		data, err := libmbd.Read(fileName)
		if err != nil {
			log.Fatal(err)
		}

		err = analyze(data, energyModel, seed, numPulses, numActiveSites, sytEnergy,
			yEnergy)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Reorganized header printing routine.<commit_after>// mouseAnalyzer determines vesicle release events and latencies for our
// mouse NMJ 6 AZ model with two synaptic vesicles each according to the
// second sensor faciliation model (see Ma et al., J. Neurophys, 2014)
package main

import (
	"flag"
	"fmt"
	"github.com/haskelladdict/mbdr/libmbd"
	"log"
	"math"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	numPulses      int     // number of AP pulses
	energyModel    bool    // use the energy model
	sytEnergy      int     // energy of activated synaptotagmin toward vesicle fusion
	yEnergy        int     // energy of activated Y sites toward vesicle fusion
	numActiveSites int     // number of simultaneously active sites required for release
	isiValue       float64 // interstimulus interval

)

func init() {
	flag.IntVar(&numPulses, "p", 1, "number of AP pulses in the model")
	flag.IntVar(&sytEnergy, "s", -1, "energy of active synaptotagmin sites "+
		"(required with -e flag)")
	flag.IntVar(&yEnergy, "y", -1, "energy of active y sites "+
		"(required with -e flag")
	flag.BoolVar(&energyModel, "e", false, "use the energy model instead of "+
		"deterministic model")
	flag.IntVar(&numActiveSites, "n", 0, "number of sites required for activation"+
		"for deterministic model")
	flag.Float64Var(&isiValue, "i", -1.0, "pulse duration in [s] for analysis multi "+
		"pulse data")
}

// usage prints a brief usage information to stdout
func usage() {
	fmt.Println("usage: mouseAnalyzer [options] <binary mcell files>")
	fmt.Println("\noptions:")
	flag.PrintDefaults()
}

// extractSeed attempts to extract the seed from the filename of the provided
// binary mcell data file.
// NOTE: the following filenaming convention is assumed *.<seedIDString>.bin.(gz|bz2)
func extractSeed(fileName string) (int, error) {
	items := strings.Split(fileName, ".")
	if len(items) <= 3 {
		return -1, fmt.Errorf("incorrectly formatted fileName. " +
			"Expected *.<seedIDString>.bin.(gz|bz2)")
	}

	for i := len(items) - 1; i >= 0; i-- {
		if items[i] == "bin" && i >= 1 {
			seed, err := strconv.Atoi(items[i-1])
			if err != nil {
				return -1, err
			}
			return seed, nil
		}
	}
	return -1, fmt.Errorf("Unable to extract seed id from filename ", fileName)
}

// printHeader prints and informative header file with date and commandline
// options requested for analysis
func printHeader() {
	fmt.Println("mouseAnalyzer ran on", time.Now())
	if host, err := os.Hostname(); err == nil {
		fmt.Println("on ", host)
	}
	fmt.Println("\n-------------- parameters --------------")
	fmt.Println("number of pulses       :", numPulses)
	if numPulses > 1 {
		fmt.Println("ISI                    :", isiValue)
	}
	if energyModel {
		fmt.Println("model                  : energy model")
		fmt.Println("syt energy             :", sytEnergy)
		fmt.Println("y energy               :", yEnergy)
	} else {
		fmt.Println("model                  : deterministic model")
		fmt.Println("number of active sites :", numActiveSites)
	}
	fmt.Println("-------------- data --------------------\n")
}

// printReleases prints a summary statistic for all released vesicle for a
// given seed
func printReleases(data *libmbd.MCellData, seed int, rel []*ReleaseEvent) {
	timeStep := data.StepSize()
	for _, r := range rel {

		channels, err := determineCaChanContrib(data, r)
		if err != nil {
			log.Fatal(err)
		}

		eventTime := float64(r.eventIter) * timeStep
		pulseID := int(math.Floor(eventTime/isiValue)) + 1

		fmt.Printf("seed : %d   AZ : %d   ves : %d   time : %e   pulseID : %d", seed,
			r.azId+1, r.vesicleID+1, eventTime, pulseID)
		fmt.Printf("  sensors: [")
		for _, s := range r.sensors {
			fmt.Printf("%d ", s)
		}
		fmt.Printf("]")
		fmt.Printf("  channels: [")
		for n, c := range channels {
			fmt.Printf("%s : %d  ", n, int(c))
		}
		fmt.Printf("]")
		fmt.Printf("\n")
	}
}

// determineCaContrib determines which Ca channels contributed to the release
// of a particular vesicle.
// NOTE: We try to be as agnostic as we can in terms of the particular
// nomenclature used for naming the channels. However, the expectation is
// that data files tracking Ca binding to vesicles are named
// vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func determineCaChanContrib(data *libmbd.MCellData, rel *ReleaseEvent) (map[string]float64, error) {
	channels := make(map[string]float64)
	// the az/channel counting is 1 based whereas our internal counting is 0 based
	regexString := fmt.Sprintf("vesicle_%d_%d_ca_.*", rel.azId+1, rel.vesicleID+1)
	counts, err := data.BlockDataByRegex(regexString)
	if err != nil {
		return nil, err
	}
	for k, c := range counts {
		if len(c.Col) != 1 {
			return nil, fmt.Errorf("data set %s has more than the expected 1 column",
				k)
		}
		if c.Col[0][rel.eventIter] > 0 {
			// need to subtract 2 from regexString due to the extra ".*"
			caString, err := extractCaChanName(k, len(regexString)-2)
			if err != nil {
				return nil, err
			}
			channels[caString] = c.Col[0][rel.eventIter]
		}
	}

	return channels, nil
}

// extractCaChanName attempts to extract the name of the calcium channel based
// on the expected data name pattern vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func extractCaChanName(name string, prefixLength int) (string, error) {
	caName := name[prefixLength:]
	items := strings.Split(caName, ".")
	if len(items) == 0 {
		return "", fmt.Errorf("Could not determine Ca channel name from data set %s", name)
	}
	return items[0], nil
}

// main entry point
func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		usage()
		return
	}

	// some sanity checks
	if energyModel && (sytEnergy < 0 || yEnergy < 0) {
		log.Fatal("Please provide a non-negative synaptotagmin and y site energy")
	}

	if !energyModel && numActiveSites == 0 {
		log.Fatal("Please provide a positive count for the number of required active sites")
	}

	if numPulses > 1 && isiValue <= 0 {
		log.Fatal("Analysis multi-pulse data requires a non-zero ISI value.")
	}

	printHeader()
	// loop over all provided data sets
	for _, fileName := range flag.Args() {

		seed, err := extractSeed(fileName)
		if err != nil {
			log.Fatal(err)
		}

		data, err := libmbd.Read(fileName)
		if err != nil {
			log.Fatal(err)
		}

		err = analyze(data, energyModel, seed, numPulses, numActiveSites, sytEnergy,
			yEnergy)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package dbo

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/byuoitav/authmiddleware/bearertoken"
	"github.com/byuoitav/configuration-database-microservice/accessors"
)

// GetData will run a get on the url, and attempt to fill the interface provided from the returned JSON.
func GetData(url string, structToFill interface{}) error {
	log.Printf("Getting data from URL: %s...", url)
	// Make an HTTP client so we can add custom headers (currently used for adding in the Bearer token for inter-microservice communication)

	client := &http.Client{}
	req, _ := http.NewRequest("GET", url, nil)

	err := setToken(req)
	if err != nil {
		return err
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(b, structToFill)
	if err != nil {
		return err
	}
	log.Printf("Done.")
	return nil
}

//PostData hits POST endpoints
func PostData(url string, structToAdd interface{}) ([]byte, error) {
	log.Printf("Posting data to URL: %s...", url)

	body, err := json.Marshal(structToAdd)
	if err != nil {
		return nil, err
	}
	client := &http.Client{}
	req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body))

	err = setToken(req)
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(response.Body)

}

func setToken(request *http.Request) error {
	if len(os.Getenv("LOCAL_ENVIRONMENT")) == 0 {

		log.Printf("Adding the bearer token for inter-service communication")

		token, err := bearertoken.GetToken()
		if err != nil {
			return err
		}

		request.Header.Set("Authorization", "Bearer "+token.Token)

	}

	return nil
}

// GetAllRawCommands retrieves all the commands
func GetAllRawCommands() (commands []accessors.RawCommand, err error) {
	log.Printf("Getting all commands.")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/commands"
	err = GetData(url, &commands)

	if err != nil {
		log.Printf("Error: %s", err.Error())
		return
	}

	log.Printf("Done.")
	return
}

// GetRoomByInfo simply retrieves a device's information from the databse.
func GetRoomByInfo(buildingName string, roomName string) (toReturn accessors.Room, err error) {
	log.Printf("Getting room %s in building %s...", roomName, buildingName)
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName, &toReturn)
	return
}

// GetDeviceByName simply retrieves a device's information from the databse.
func GetDeviceByName(buildingName string, roomName string, deviceName string) (toReturn accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices/"+deviceName, &toReturn)
	return
}

// GetDevicesByRoom will jut get the devices based on the room.
func GetDevicesByRoom(buildingName string, roomName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices", &toReturn)
	return
}

// GetDevicesByBuildingAndRoomAndRole will get the devices with the given role from the DB
func GetDevicesByBuildingAndRoomAndRole(building string, room string, roleName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+building+"/rooms/"+room+"/devices/roles/"+roleName, &toReturn)
	return
}

// SetAudioInDB will set the audio levels in the database
func SetAudioInDB(building string, room string, device accessors.Device) error {
	log.Printf("Updating audio levels in DB.")

	if device.Volume != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/volume/" + strconv.Itoa(*device.Volume)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	if device.Muted != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/muted/" + strconv.FormatBool(*device.Muted)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	return nil
}

// GetBuildings will return all buildings
func GetBuildings() ([]accessors.Building, error) {
	log.Printf("getting all buildings...")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings"
	var buildings []accessors.Building
	err := GetData(url, &buildings)

	return buildings, err
}

// GetRooms returns all the rooms in a given building
func GetRoomsByBuilding(building string) ([]accessors.Room, error) {
	log.Printf("getting all rooms from %v ...", building)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms"
	var rooms []accessors.Room
	err := GetData(url, &rooms)
	return rooms, err
}

// AddBuilding monsters
func AddBuilding(buildingToAdd accessors.Building) (accessors.Building, error) {
	log.Printf("adding building %v to database", buildingToAdd.Shortname)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + buildingToAdd.Shortname

	result, err := PostData(url, buildingToAdd)
	if err != nil {
		return buildingToAdd, err
	}

	var building accessors.Building
	err = json.Unmarshal(result, &building)
	if err != nil {
		return building, err
	}

	return building, nil

}
<commit_msg>added capability to query buildings on shortname<commit_after>package dbo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/byuoitav/authmiddleware/bearertoken"
	"github.com/byuoitav/configuration-database-microservice/accessors"
)

// GetData will run a get on the url, and attempt to fill the interface provided from the returned JSON.
func GetData(url string, structToFill interface{}) error {
	log.Printf("Getting data from URL: %s...", url)
	// Make an HTTP client so we can add custom headers (currently used for adding in the Bearer token for inter-microservice communication)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}

	err = setToken(req)
	if err != nil {
		return err
	}

	if req == nil {
		fmt.Printf("Alert! req is nil!")
	}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		errorString, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return errors.New(string(errorString))
	}

	err = json.Unmarshal(b, structToFill)
	if err != nil {
		return err
	}
	log.Printf("Done.")
	return nil
}

//PostData hits POST endpoints
func PostData(url string, structToAdd interface{}) ([]byte, error) {
	log.Printf("Posting data to URL: %s...", url)

	body, err := json.Marshal(structToAdd)
	if err != nil {
		return nil, err
	}
	client := &http.Client{}
	req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body))

	req.Header.Set("Content-Type", "application/json")

	err = setToken(req)
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if response.StatusCode != http.StatusOK {
		errorString, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return []byte{}, err
		}
		return []byte{}, errors.New(string(errorString))
	}

	return ioutil.ReadAll(response.Body)
}

func setToken(request *http.Request) error {
	fmt.Printf("Calling setToken on %v", request)

	if len(os.Getenv("LOCAL_ENVIRONMENT")) == 0 {

		log.Printf("Adding the bearer token for inter-service communication")

		token, err := bearertoken.GetToken()
		if err != nil {
			return err
		}

		request.Header.Set("Authorization", "Bearer "+token.Token)

	}

	return nil
}

// GetAllRawCommands retrieves all the commands
func GetAllRawCommands() (commands []accessors.RawCommand, err error) {
	log.Printf("Getting all commands.")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/commands"
	err = GetData(url, &commands)

	if err != nil {
		log.Printf("Error: %s", err.Error())
		return
	}

	log.Printf("Done.")
	return
}

// GetRoomByInfo simply retrieves a device's information from the databse.
func GetRoomByInfo(buildingName string, roomName string) (toReturn accessors.Room, err error) {
	log.Printf("Getting room %s in building %s...", roomName, buildingName)
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName, &toReturn)
	return
}

// GetDeviceByName simply retrieves a device's information from the databse.
func GetDeviceByName(buildingName string, roomName string, deviceName string) (toReturn accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices/"+deviceName, &toReturn)
	return
}

// GetDevicesByRoom will jut get the devices based on the room.
func GetDevicesByRoom(buildingName string, roomName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices", &toReturn)
	return
}

// GetDevicesByBuildingAndRoomAndRole will get the devices with the given role from the DB
func GetDevicesByBuildingAndRoomAndRole(building string, room string, roleName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+building+"/rooms/"+room+"/devices/roles/"+roleName, &toReturn)
	return
}

// SetAudioInDB will set the audio levels in the database
func SetAudioInDB(building string, room string, device accessors.Device) error {
	log.Printf("Updating audio levels in DB.")

	if device.Volume != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/volume/" + strconv.Itoa(*device.Volume)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	if device.Muted != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/muted/" + strconv.FormatBool(*device.Muted)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	return nil
}

// GetBuildings will return all buildings
func GetBuildings() ([]accessors.Building, error) {
	log.Printf("getting all buildings...")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings"
	var buildings []accessors.Building
	err := GetData(url, &buildings)

	return buildings, err
}

// GetRooms returns all the rooms in a given building
func GetRoomsByBuilding(building string) ([]accessors.Room, error) {
	log.Printf("getting all rooms from %v ...", building)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms"
	var rooms []accessors.Room
	err := GetData(url, &rooms)
	return rooms, err
}

// GetBuildingByShortname returns a building with a given shortname
func GetBuildingByShortname(building string) (accessors.Building, error) {
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/shortname/" + building
	var output accessors.Building
	err := GetData(url, output)
	if err != nil {
		return output, err
	}
	return output, nil
}

// AddBuilding monsters
func AddBuilding(buildingToAdd accessors.Building) (accessors.Building, error) {
	log.Printf("adding building %v to database", buildingToAdd.Shortname)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + buildingToAdd.Shortname

	result, err := PostData(url, buildingToAdd)
	if err != nil {
		return buildingToAdd, err
	}

	var building accessors.Building
	err = json.Unmarshal(result, &building)
	if err != nil {
		return building, err
	}

	return building, nil

}
<|endoftext|>