<commit_before>package v2actions_test

import (
	"errors"

	. "code.cloudfoundry.org/cli/actors/v2actions"
	"code.cloudfoundry.org/cli/actors/v2actions/v2actionsfakes"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("Route Actions", func() {
	var (
		actor                     Actor
		fakeCloudControllerClient *v2actionsfakes.FakeCloudControllerClient
	)

	BeforeEach(func() {
		fakeCloudControllerClient = new(v2actionsfakes.FakeCloudControllerClient)
		actor = NewActor(fakeCloudControllerClient)
	})

	Describe("GetOrphanedRoutesBySpace", func() {
		BeforeEach(func() {
			fakeCloudControllerClient.GetRouteApplicationsStub = func(routeGUID string) ([]ccv2.Application, ccv2.Warnings, error) {
				switch routeGUID {
				case "orphaned-route-guid-1":
					return []ccv2.Application{}, nil, nil
				case "orphaned-route-guid-2":
					return []ccv2.Application{}, nil, nil
				case "not-orphaned-route-guid-3":
					return []ccv2.Application{
						{GUID: "app-guid"},
					}, nil, nil
				}
				Fail("Unexpected route-guid")
				return []ccv2.Application{}, nil, nil
			}
		})

		Context("when there are orphaned routes", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					{
						GUID:         "orphaned-route-guid-1",
						DomainFields: ccv2.Domain{GUID: "some-domain-guid", Name: " some-domain.com"},
					},
					{
						GUID:         "orphaned-route-guid-2",
						DomainFields: ccv2.Domain{GUID: "some-other-domain-guid", Name: "some-other-domain.com"},
					},
					{
						GUID: "not-orphaned-route-guid-3",
					},
				}, nil, nil)
				fakeCloudControllerClient.GetSharedDomainStub = func(domainGUID string) (ccv2.Domain, ccv2.Warnings, error) {
					switch domainGUID {
					case "some-domain-guid":
						return ccv2.Domain{
							GUID: "some-domain-guid",
							Name: "some-domain.com",
						}, nil, nil
					case "some-other-domain-guid":
						return ccv2.Domain{
							GUID: "some-other-domain-guid",
							Name: "some-other-domain.com",
						}, nil, nil
					}

					return ccv2.Domain{}, nil, errors.New("Unexpected domain GUID")
				}
			})

			It("returns the orphaned routes with the domain names", func() {
				orphanedRoutes, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).NotTo(HaveOccurred())
				Expect(orphanedRoutes).To(ConsistOf([]Route{
					{
						GUID:   "orphaned-route-guid-1",
						Domain: "some-domain.com",
					},
					{
						GUID:   "orphaned-route-guid-2",
						Domain: "some-other-domain.com",
					},
				}))

				Expect(fakeCloudControllerClient.GetSpaceRoutesCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.GetSpaceRoutesArgsForCall(0)).To(Equal("space-guid"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsCallCount()).To(Equal(3))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(0)).To(Equal("orphaned-route-guid-1"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(1)).To(Equal("orphaned-route-guid-2"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(2)).To(Equal("not-orphaned-route-guid-3"))
			})
		})

		Context("when there are no orphaned routes", func() {
			var expectedErr OrphanedRoutesNotFoundError

			BeforeEach(func() {
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					ccv2.Route{GUID: "not-orphaned-route-guid-3"},
				}, nil, nil)
			})

			It("returns an OrphanedRoutesNotFoundError", func() {
				orphanedRoutes, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).To(MatchError(expectedErr))
				Expect(orphanedRoutes).To(BeNil())

				Expect(fakeCloudControllerClient.GetSpaceRoutesCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.GetSpaceRoutesArgsForCall(0)).To(Equal("space-guid"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(0)).To(Equal("not-orphaned-route-guid-3"))
			})
		})

		Context("when there are warnings", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					ccv2.Route{GUID: "route-guid-1"},
					ccv2.Route{GUID: "route-guid-2"},
				}, ccv2.Warnings{"get-routes-warning"}, nil)
				fakeCloudControllerClient.GetRouteApplicationsReturns(nil, ccv2.Warnings{"get-applications-warning"}, nil)
				fakeCloudControllerClient.GetSharedDomainReturns(ccv2.Domain{GUID: "some-guid"}, ccv2.Warnings{"get-shared-domain-warning"}, nil)
			})

			It("returns all the warnings", func() {
				_, warnings, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).NotTo(HaveOccurred())
				Expect(warnings).To(ConsistOf("get-routes-warning", "get-applications-warning", "get-shared-domain-warning", "get-applications-warning", "get-shared-domain-warning"))
			})
		})

		Context("when the spaces routes API request returns an error", func() {
			var expectedErr error

			BeforeEach(func() {
				expectedErr = errors.New("spaces routes error")
				fakeCloudControllerClient.GetSpaceRoutesReturns(nil, nil, expectedErr)
			})

			It("returns the error", func() {
				_, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).To(Equal(expectedErr))
			})
		})

		Context("when a route's applications API request returns an error", func() {
			var expectedErr error

			BeforeEach(func() {
				expectedErr = errors.New("application error")
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					ccv2.Route{GUID: "route-guid"},
				}, nil, nil)
				fakeCloudControllerClient.GetRouteApplicationsReturns(nil, nil, expectedErr)
			})

			It("returns the error", func() {
				_, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).To(Equal(expectedErr))
			})
		})
	})

	Describe("DeleteRouteByGUID", func() {
		Context("when the route exists", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.DeleteRouteReturns(nil, nil)
			})

			It("deletes the route", func() {
				_, err := actor.DeleteRouteByGUID("some-route-guid")
				Expect(err).NotTo(HaveOccurred())

				Expect(fakeCloudControllerClient.DeleteRouteCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.DeleteRouteArgsForCall(0)).To(Equal("some-route-guid"))
			})
		})

		Context("when the API returns both warnings and an error", func() {
			var expectedErr error

			BeforeEach(func() {
				expectedErr = errors.New("bananahammock")
				fakeCloudControllerClient.DeleteRouteReturns(ccv2.Warnings{"foo", "bar"}, expectedErr)
			})

			It("returns both the warnings and the error", func() {
				warnings, err := actor.DeleteRouteByGUID("some-route-guid")
				Expect(err).To(MatchError(expectedErr))
				Expect(warnings).To(ConsistOf("foo", "bar"))
			})
		})
	})

	Describe("Route", func() {
		DescribeTable("String", func(host string, domain string, path string, port int, expectedValue string) {
			route := Route{
				Host:   host,
				Domain: domain,
				Path:   path,
				Port:   port,
			}
			actualValue := route.String()

			Expect(actualValue).To(Equal(expectedValue))
		},

			Entry("has domain", "", "domain.com", "", 0, "domain.com"),
			Entry("has host, domain", "host", "domain.com", "", 0, "host.domain.com"),
			Entry("has domain, path", "", "domain.com", "path", 0, "domain.com/path"),
			Entry("has host, domain, path", "host", "domain.com", "path", 0, "host.domain.com/path"),
			Entry("has domain, port", "", "domain.com", "", 3333, "domain.com:3333"),
		)
	})
})
<commit_msg>additional test for String method on Route<commit_after>package v2actions_test

import (
	"errors"

	. "code.cloudfoundry.org/cli/actors/v2actions"
	"code.cloudfoundry.org/cli/actors/v2actions/v2actionsfakes"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("Route Actions", func() {
	var (
		actor                     Actor
		fakeCloudControllerClient *v2actionsfakes.FakeCloudControllerClient
	)

	BeforeEach(func() {
		fakeCloudControllerClient = new(v2actionsfakes.FakeCloudControllerClient)
		actor = NewActor(fakeCloudControllerClient)
	})

	Describe("GetOrphanedRoutesBySpace", func() {
		BeforeEach(func() {
			fakeCloudControllerClient.GetRouteApplicationsStub = func(routeGUID string) ([]ccv2.Application, ccv2.Warnings, error) {
				switch routeGUID {
				case "orphaned-route-guid-1":
					return []ccv2.Application{}, nil, nil
				case "orphaned-route-guid-2":
					return []ccv2.Application{}, nil, nil
				case "not-orphaned-route-guid-3":
					return []ccv2.Application{
						{GUID: "app-guid"},
					}, nil, nil
				}
				Fail("Unexpected route-guid")
				return []ccv2.Application{}, nil, nil
			}
		})

		Context("when there are orphaned routes", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					{
						GUID:         "orphaned-route-guid-1",
						DomainFields: ccv2.Domain{GUID: "some-domain-guid", Name: " some-domain.com"},
					},
					{
						GUID:         "orphaned-route-guid-2",
						DomainFields: ccv2.Domain{GUID: "some-other-domain-guid", Name: "some-other-domain.com"},
					},
					{
						GUID: "not-orphaned-route-guid-3",
					},
				}, nil, nil)
				fakeCloudControllerClient.GetSharedDomainStub = func(domainGUID string) (ccv2.Domain, ccv2.Warnings, error) {
					switch domainGUID {
					case "some-domain-guid":
						return ccv2.Domain{
							GUID: "some-domain-guid",
							Name: "some-domain.com",
						}, nil, nil
					case "some-other-domain-guid":
						return ccv2.Domain{
							GUID: "some-other-domain-guid",
							Name: "some-other-domain.com",
						}, nil, nil
					}

					return ccv2.Domain{}, nil, errors.New("Unexpected domain GUID")
				}
			})

			It("returns the orphaned routes with the domain names", func() {
				orphanedRoutes, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).NotTo(HaveOccurred())
				Expect(orphanedRoutes).To(ConsistOf([]Route{
					{
						GUID:   "orphaned-route-guid-1",
						Domain: "some-domain.com",
					},
					{
						GUID:   "orphaned-route-guid-2",
						Domain: "some-other-domain.com",
					},
				}))

				Expect(fakeCloudControllerClient.GetSpaceRoutesCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.GetSpaceRoutesArgsForCall(0)).To(Equal("space-guid"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsCallCount()).To(Equal(3))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(0)).To(Equal("orphaned-route-guid-1"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(1)).To(Equal("orphaned-route-guid-2"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(2)).To(Equal("not-orphaned-route-guid-3"))
			})
		})

		Context("when there are no orphaned routes", func() {
			var expectedErr OrphanedRoutesNotFoundError

			BeforeEach(func() {
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					ccv2.Route{GUID: "not-orphaned-route-guid-3"},
				}, nil, nil)
			})

			It("returns an OrphanedRoutesNotFoundError", func() {
				orphanedRoutes, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).To(MatchError(expectedErr))
				Expect(orphanedRoutes).To(BeNil())

				Expect(fakeCloudControllerClient.GetSpaceRoutesCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.GetSpaceRoutesArgsForCall(0)).To(Equal("space-guid"))
				Expect(fakeCloudControllerClient.GetRouteApplicationsCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.GetRouteApplicationsArgsForCall(0)).To(Equal("not-orphaned-route-guid-3"))
			})
		})

		Context("when there are warnings", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					ccv2.Route{GUID: "route-guid-1"},
					ccv2.Route{GUID: "route-guid-2"},
				}, ccv2.Warnings{"get-routes-warning"}, nil)
				fakeCloudControllerClient.GetRouteApplicationsReturns(nil, ccv2.Warnings{"get-applications-warning"}, nil)
				fakeCloudControllerClient.GetSharedDomainReturns(ccv2.Domain{GUID: "some-guid"}, ccv2.Warnings{"get-shared-domain-warning"}, nil)
			})

			It("returns all the warnings", func() {
				_, warnings, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).NotTo(HaveOccurred())
				Expect(warnings).To(ConsistOf("get-routes-warning", "get-applications-warning", "get-shared-domain-warning", "get-applications-warning", "get-shared-domain-warning"))
			})
		})

		Context("when the spaces routes API request returns an error", func() {
			var expectedErr error

			BeforeEach(func() {
				expectedErr = errors.New("spaces routes error")
				fakeCloudControllerClient.GetSpaceRoutesReturns(nil, nil, expectedErr)
			})

			It("returns the error", func() {
				_, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).To(Equal(expectedErr))
			})
		})

		Context("when a route's applications API request returns an error", func() {
			var expectedErr error

			BeforeEach(func() {
				expectedErr = errors.New("application error")
				fakeCloudControllerClient.GetSpaceRoutesReturns([]ccv2.Route{
					ccv2.Route{GUID: "route-guid"},
				}, nil, nil)
				fakeCloudControllerClient.GetRouteApplicationsReturns(nil, nil, expectedErr)
			})

			It("returns the error", func() {
				_, _, err := actor.GetOrphanedRoutesBySpace("space-guid")
				Expect(err).To(Equal(expectedErr))
			})
		})
	})

	Describe("DeleteRouteByGUID", func() {
		Context("when the route exists", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.DeleteRouteReturns(nil, nil)
			})

			It("deletes the route", func() {
				_, err := actor.DeleteRouteByGUID("some-route-guid")
				Expect(err).NotTo(HaveOccurred())

				Expect(fakeCloudControllerClient.DeleteRouteCallCount()).To(Equal(1))
				Expect(fakeCloudControllerClient.DeleteRouteArgsForCall(0)).To(Equal("some-route-guid"))
			})
		})

		Context("when the API returns both warnings and an error", func() {
			var expectedErr error

			BeforeEach(func() {
				expectedErr = errors.New("bananahammock")
				fakeCloudControllerClient.DeleteRouteReturns(ccv2.Warnings{"foo", "bar"}, expectedErr)
			})

			It("returns both the warnings and the error", func() {
				warnings, err := actor.DeleteRouteByGUID("some-route-guid")
				Expect(err).To(MatchError(expectedErr))
				Expect(warnings).To(ConsistOf("foo", "bar"))
			})
		})
	})

	Describe("Route", func() {
		DescribeTable("String", func(host string, domain string, path string, port int, expectedValue string) {
			route := Route{
				Host:   host,
				Domain: domain,
				Path:   path,
				Port:   port,
			}
			Expect(route.String()).To(Equal(expectedValue))
		},

			Entry("has domain", "", "domain.com", "", 0, "domain.com"),
			Entry("has host, domain", "host", "domain.com", "", 0, "host.domain.com"),
			Entry("has domain, path", "", "domain.com", "path", 0, "domain.com/path"),
			Entry("has host, domain, path", "host", "domain.com", "path", 0, "host.domain.com/path"),
			Entry("has domain, port", "", "domain.com", "", 3333, "domain.com:3333"),
			Entry("has host, domain, path, port", "host", "domain.com", "path", 3333, "domain.com:3333"),
		)
	})
})
<|endoftext|>


<commit_before>package core

import (
	"log"
	"net/http"
	"strconv"
	"strings"

	"github.com/mdlayher/wavepipe/api"
	"github.com/mdlayher/wavepipe/api/auth"
	"github.com/mdlayher/wavepipe/config"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/gzip"
	"github.com/martini-contrib/render"
)

// apiRouter sets up the instance of martini
func apiRouter(apiKillChan chan struct{}) {
	log.Println("api: starting...")

	// Initialize martini
	m := martini.New()

	// Set up middleware
	// GZIP all requests to drastically reduce size
	m.Use(gzip.All())
	m.Use(render.Renderer(render.Options{
		// Output human-readable JSON. GZIP will essentially negate the size increase, and this
		// makes the API much more developer-friendly
		IndentJSON: true,
	}))

	// Enable graceful shutdown when triggered by manager
	stopAPI := false
	m.Use(func(r render.Render) {
		// If API is stopping, render a HTTP 503
		if stopAPI {
			r.JSON(503, api.Error{
				Code:    503,
				Message: "service is shutting down",
			})
			return
		}
	})

	// Authenticate all API calls
	m.Use(func(req *http.Request, res http.ResponseWriter, c martini.Context, r render.Render) {
		// Set a different authentication method depending on endpoint
		var authMethod auth.AuthMethod

		// For login, use the bcrypt authenticator to generate a new session
		path := strings.TrimRight(req.URL.Path, "/")
		if path == "/api/v0/login" {
			authMethod = new(auth.BcryptAuth)
		} else {
			// For other API methods, use the HMAC-SHA1 authenticator
			authMethod = new(auth.HMACAuth)
		}

		// Attempt authentication
		user, clientErr, serverErr := authMethod.Authenticate(req)

		// Check for client error
		if clientErr != nil {
			// If no username or password, send a WWW-Authenticate header to prompt request
			// This allows for manual exploration of the API if needed
			if clientErr == auth.ErrNoUsername || clientErr == auth.ErrNoPassword {
				res.Header().Set("WWW-Authenticate", "Basic")
			}

			r.JSON(401, api.Error{
				Code:    401,
				Message: "authentication failed: " + clientErr.Error(),
			})
			return
		}

		// Check for server error
		if serverErr != nil {
			log.Println(serverErr)

			r.JSON(500, api.Error{
				Code:    500,
				Message: "server error",
			})
			return
		}

		// Successful login, map session user to martini context
		c.Map(user)

		// Print information about this API call
		log.Printf("api: [%s] %s", req.RemoteAddr, req.URL.Path)
	})

	// Set up API routes
	r := martini.NewRouter()

	// Set up API information route
	r.Get("/api", api.APIInfo)

	// Set up API group routes, with API version parameter
	r.Group("/api/:version", func(r martini.Router) {
		// Root API, containing information and help
		r.Get("", api.APIInfo)

		// Albums API
		r.Get("/albums", api.GetAlbums)
		r.Get("/albums/:id", api.GetAlbums)

		// Artists API
		r.Get("/artists", api.GetArtists)
		r.Get("/artists/:id", api.GetArtists)

		// Login API
		r.Get("/login", api.GetLogin)

		// Songs API
		r.Get("/songs", api.GetSongs)
		r.Get("/songs/:id", api.GetSongs)

		// Stream API
		r.Get("/stream", api.GetStream)
		r.Get("/stream/:id", api.GetStream)
	})

	// Add router action, start server
	m.Action(r.Handle)
	go func() {
		// Load config
		conf, err := config.C.Load()
		if err != nil {
			log.Println(err)
			return
		}

		// Start server
		log.Println("api: listening on port", conf.Port)
		if err := http.ListenAndServe(":"+strconv.Itoa(conf.Port), m); err != nil {
			log.Println(err)
		}
	}()

	// Trigger events via channel
	for {
		select {
		// Stop API
		case <-apiKillChan:
			// Stop serving requests
			stopAPI = true

			// Inform manager that shutdown is complete
			log.Println("api: stopped!")
			apiKillChan <- struct{}{}
			return
		}
	}
}
<commit_msg>Add logout API functionality<commit_after>package core

import (
	"log"
	"net/http"
	"strconv"
	"strings"

	"github.com/mdlayher/wavepipe/api"
	"github.com/mdlayher/wavepipe/api/auth"
	"github.com/mdlayher/wavepipe/config"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/gzip"
	"github.com/martini-contrib/render"
)

// apiRouter sets up the instance of martini
func apiRouter(apiKillChan chan struct{}) {
	log.Println("api: starting...")

	// Initialize martini
	m := martini.New()

	// Set up middleware
	// GZIP all requests to drastically reduce size
	m.Use(gzip.All())
	m.Use(render.Renderer(render.Options{
		// Output human-readable JSON. GZIP will essentially negate the size increase, and this
		// makes the API much more developer-friendly
		IndentJSON: true,
	}))

	// Enable graceful shutdown when triggered by manager
	stopAPI := false
	m.Use(func(r render.Render) {
		// If API is stopping, render a HTTP 503
		if stopAPI {
			r.JSON(503, api.Error{
				Code:    503,
				Message: "service is shutting down",
			})
			return
		}
	})

	// Authenticate all API calls
	m.Use(func(req *http.Request, res http.ResponseWriter, c martini.Context, r render.Render) {
		// Set a different authentication method depending on endpoint
		var authMethod auth.AuthMethod

		// For login, use the bcrypt authenticator to generate a new session
		path := strings.TrimRight(req.URL.Path, "/")
		if path == "/api/v0/login" {
			authMethod = new(auth.BcryptAuth)
		} else {
			// For other API methods, use the HMAC-SHA1 authenticator
			authMethod = new(auth.HMACAuth)
		}

		// Attempt authentication
		user, session, clientErr, serverErr := authMethod.Authenticate(req)

		// Check for client error
		if clientErr != nil {
			// If no username or password, send a WWW-Authenticate header to prompt request
			// This allows for manual exploration of the API if needed
			if clientErr == auth.ErrNoUsername || clientErr == auth.ErrNoPassword {
				res.Header().Set("WWW-Authenticate", "Basic")
			}

			r.JSON(401, api.Error{
				Code:    401,
				Message: "authentication failed: " + clientErr.Error(),
			})
			return
		}

		// Check for server error
		if serverErr != nil {
			log.Println(serverErr)

			r.JSON(500, api.Error{
				Code:    500,
				Message: "server error",
			})
			return
		}

		// Successful login, map session user and session to martini context
		c.Map(user)
		c.Map(session)

		// Print information about this API call
		log.Printf("api: [%s] %s", req.RemoteAddr, req.URL.Path)
	})

	// Set up API routes
	r := martini.NewRouter()

	// Set up API information route
	r.Get("/api", api.APIInfo)

	// Set up API group routes, with API version parameter
	r.Group("/api/:version", func(r martini.Router) {
		// Root API, containing information and help
		r.Get("", api.APIInfo)

		// Albums API
		r.Get("/albums", api.GetAlbums)
		r.Get("/albums/:id", api.GetAlbums)

		// Artists API
		r.Get("/artists", api.GetArtists)
		r.Get("/artists/:id", api.GetArtists)

		// Login API
		r.Get("/login", api.GetLogin)

		// Logout API
		r.Get("/logout", api.GetLogout)

		// Songs API
		r.Get("/songs", api.GetSongs)
		r.Get("/songs/:id", api.GetSongs)

		// Stream API
		r.Get("/stream", api.GetStream)
		r.Get("/stream/:id", api.GetStream)
	})

	// Add router action, start server
	m.Action(r.Handle)
	go func() {
		// Load config
		conf, err := config.C.Load()
		if err != nil {
			log.Println(err)
			return
		}

		// Start server
		log.Println("api: listening on port", conf.Port)
		if err := http.ListenAndServe(":"+strconv.Itoa(conf.Port), m); err != nil {
			log.Println(err)
		}
	}()

	// Trigger events via channel
	for {
		select {
		// Stop API
		case <-apiKillChan:
			// Stop serving requests
			stopAPI = true

			// Inform manager that shutdown is complete
			log.Println("api: stopped!")
			apiKillChan <- struct{}{}
			return
		}
	}
}
<|endoftext|>


<commit_before>package tunnel

import (
	"os"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/proxy/mucp"
	"github.com/micro/go-micro/registry/memory"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/server"
	tun "github.com/micro/go-micro/tunnel"
	"github.com/micro/go-micro/tunnel/transport"
	"github.com/micro/go-micro/util/log"
)

var (
	// Name of the router microservice
	Name = "go.micro.tunnel"
	// Address is the tunnel microservice bind address
	Address = ":9095"
	// Tunnel is the tunnel bind address
	Tunnel = ":9096"
	// Router is the router gossip bind address
	Router = ":9093"
	// Network is the network id
	Network = "local"
)

// run runs the micro server
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("network_address")) > 0 {
		Network = ctx.String("network")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("tunnel_address")) > 0 {
		Tunnel = ctx.String("tunnel")
	}
	// default gateway address
	var gateway string
	if len(ctx.String("gateway_address")) > 0 {
		gateway = ctx.String("gateway")
	}

	// Initialise service
	service := micro.NewService(
		micro.Name(Name),
		micro.Address(Address),
		micro.RegisterTTL(time.Duration(ctx.GlobalInt("register_ttl"))*time.Second),
		micro.RegisterInterval(time.Duration(ctx.GlobalInt("register_interval"))*time.Second),
	)

	// local tunnel router
	r := router.NewRouter(
		router.Id(service.Server().Options().Id),
		router.Registry(service.Client().Options().Registry),
		router.Address(Router),
		router.Network(Network),
		router.Gateway(gateway),
	)

	// create a tunnel
	t := tun.NewTunnel(
		tun.Address(Tunnel),
	)

	// create tunnel client with tunnel transport
	tunTransport := transport.NewTransport(
		transport.WithTunnel(t),
	)

	// local server client talks to tunnel
	localSrvClient := client.NewClient(
		client.Transport(tunTransport),
	)

	// local proxy
	localProxy := mucp.NewProxy(
		options.WithValue("proxy.router", r),
		options.WithValue("proxy.client", localSrvClient),
		options.WithValue("proxy.endpoint", Tunnel),
	)

	// init server
	service.Server().Init(
		server.WithRouter(localProxy),
	)

	// local transport client
	tunSrvClient := client.NewClient(
		client.Transport(service.Options().Transport),
	)

	// local proxy
	tunProxy := mucp.NewProxy(
		options.WithValue("proxy.client", tunSrvClient),
	)

	// create memory registry
	memRegistry := memory.NewRegistry()

	// local server
	tunSrv := server.NewServer(
		server.Address(Tunnel),
		server.Transport(tunTransport),
		server.WithRouter(tunProxy),
		server.Registry(memRegistry),
	)

	if err := tunSrv.Start(); err != nil {
		log.Logf("[tunnel] error starting tunnel server: %v", err)
		os.Exit(1)
	}

	if err := service.Run(); err != nil {
		log.Log("[tunnel] %s failed: %v", Name, err)
	}

	// stop the router
	if err := r.Stop(); err != nil {
		log.Logf("[tunnel] error stopping tunnel router: %v", err)
	}

	// stop the server
	if err := tunSrv.Stop(); err != nil {
		log.Logf("[tunnel] error stopping tunnel server: %v", err)
	}

	if err := t.Connect(); err != nil {
		log.Logf("[tunnel] error stopping tunnel: %v", err)
	}

	log.Logf("[tunnel] stopped")
}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "tunnel",
		Usage: "Run the micro network tunnel",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "tunnel_address",
				Usage:  "Set the micro tunnel address :9096",
				EnvVar: "MICRO_TUNNEL_ADDRESS",
			},
			cli.StringFlag{
				Name:   "network_address",
				Usage:  "Set the micro network address: local",
				EnvVar: "MICRO_NETWORK_ADDRESS",
			},
			cli.StringFlag{
				Name:   "gateway_address",
				Usage:  "Set the micro default gateway address :9094",
				EnvVar: "MICRO_GATEWAY_ADDRESS",
			},
		},
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<commit_msg>Provide tunnel nodes via tunnel_nodes cli flag<commit_after>package tunnel

import (
	"os"
	"strings"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/proxy/mucp"
	"github.com/micro/go-micro/registry/memory"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/server"
	tun "github.com/micro/go-micro/tunnel"
	"github.com/micro/go-micro/tunnel/transport"
	"github.com/micro/go-micro/util/log"
)

var (
	// Name of the router microservice
	Name = "go.micro.tunnel"
	// Address is the tunnel microservice bind address
	Address = ":9095"
	// Tunnel is the tunnel bind address
	Tunnel = ":9096"
	// Router is the router gossip bind address
	Router = ":9093"
	// Network is the network id
	Network = "local"
)

// run runs the micro server
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("network_address")) > 0 {
		Network = ctx.String("network")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("tunnel_address")) > 0 {
		Tunnel = ctx.String("tunnel")
	}
	var nodes []string
	if len(ctx.String("tunnel_nodes")) > 0 {
		nodes = strings.Split(ctx.String("tunnel_nodes"), ",")
	}
	// default gateway address
	var gateway string
	if len(ctx.String("gateway_address")) > 0 {
		gateway = ctx.String("gateway")
	}

	// Initialise service
	service := micro.NewService(
		micro.Name(Name),
		micro.Address(Address),
		micro.RegisterTTL(time.Duration(ctx.GlobalInt("register_ttl"))*time.Second),
		micro.RegisterInterval(time.Duration(ctx.GlobalInt("register_interval"))*time.Second),
	)

	// local tunnel router
	r := router.NewRouter(
		router.Id(service.Server().Options().Id),
		router.Registry(service.Client().Options().Registry),
		router.Address(Router),
		router.Network(Network),
		router.Gateway(gateway),
	)

	// create a tunnel
	t := tun.NewTunnel(
		tun.Address(Tunnel),
		tun.Nodes(nodes...),
	)

	// create tunnel client with tunnel transport
	tunTransport := transport.NewTransport(
		transport.WithTunnel(t),
	)

	// local server client talks to tunnel
	localSrvClient := client.NewClient(
		client.Transport(tunTransport),
	)

	// local proxy
	localProxy := mucp.NewProxy(
		options.WithValue("proxy.router", r),
		options.WithValue("proxy.client", localSrvClient),
		options.WithValue("proxy.endpoint", Tunnel),
	)

	// init server
	service.Server().Init(
		server.WithRouter(localProxy),
	)

	// local transport client
	tunSrvClient := client.NewClient(
		client.Transport(service.Options().Transport),
	)

	// local proxy
	tunProxy := mucp.NewProxy(
		options.WithValue("proxy.client", tunSrvClient),
	)

	// create memory registry
	memRegistry := memory.NewRegistry()

	// local server
	tunSrv := server.NewServer(
		server.Address(Tunnel),
		server.Transport(tunTransport),
		server.WithRouter(tunProxy),
		server.Registry(memRegistry),
	)

	if err := tunSrv.Start(); err != nil {
		log.Logf("[tunnel] error starting tunnel server: %v", err)
		os.Exit(1)
	}

	if err := service.Run(); err != nil {
		log.Log("[tunnel] %s failed: %v", Name, err)
	}

	// stop the router
	if err := r.Stop(); err != nil {
		log.Logf("[tunnel] error stopping tunnel router: %v", err)
	}

	// stop the server
	if err := tunSrv.Stop(); err != nil {
		log.Logf("[tunnel] error stopping tunnel server: %v", err)
	}

	if err := t.Connect(); err != nil {
		log.Logf("[tunnel] error stopping tunnel: %v", err)
	}

	log.Logf("[tunnel] stopped")
}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "tunnel",
		Usage: "Run the micro network tunnel",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "tunnel_address",
				Usage:  "Set the micro tunnel address :9096",
				EnvVar: "MICRO_TUNNEL_ADDRESS",
			},
			cli.StringFlag{
				Name:   "network_address",
				Usage:  "Set the micro network address: local",
				EnvVar: "MICRO_NETWORK_ADDRESS",
			},
			cli.StringFlag{
				Name:   "gateway_address",
				Usage:  "Set the micro default gateway address :9094",
				EnvVar: "MICRO_GATEWAY_ADDRESS",
			},
			cli.StringFlag{
				Name:   "tunnel_nodes",
				Usage:  "Set the micro tunnel nodes",
				EnvVar: "MICRO_TUNNEL_NODES",
			},
		},
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<|endoftext|>


<commit_before>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/pkg/errors"
)

type TunnelRoute struct {
	Network    string    `json:"network"`
	TunnelId   string    `json:"tunnel_id"`
	TunnelName string    `json:"tunnel_name"`
	Comment    string    `json:"comment"`
	CreatedAt  time.Time `json:"created_at"`
	DeletedAt  time.Time `json:"deleted_at"`
}

// TunnelRouteListResponse is the API response for listing tunnel routes.
type TunnelRouteListResponse struct {
	Response
	Result []TunnelRoute `json:"result"`
}

func (api *API) TunnelRoutes(ctx context.Context) ([]TunnelRoute, error) {
	uri := fmt.Sprintf("/%s/%s/teamnet/routes", AccountRouteRoot, api.AccountID)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)

	if err != nil {
		return []TunnelRoute{}, err
	}

	var resp TunnelRouteListResponse
	err = json.Unmarshal(res, &resp)
	if err != nil {
		return []TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return resp.Result, nil
}
<commit_msg>that struct doesn't need to be public since it's never actually returned anywhere.<commit_after>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/pkg/errors"
)

type TunnelRoute struct {
	Network    string    `json:"network"`
	TunnelId   string    `json:"tunnel_id"`
	TunnelName string    `json:"tunnel_name"`
	Comment    string    `json:"comment"`
	CreatedAt  time.Time `json:"created_at"`
	DeletedAt  time.Time `json:"deleted_at"`
}

// tunnelRouteListResponse is the API response for listing tunnel routes.
type tunnelRouteListResponse struct {
	Response
	Result []TunnelRoute `json:"result"`
}

func (api *API) TunnelRoutes(ctx context.Context) ([]TunnelRoute, error) {
	uri := fmt.Sprintf("/%s/%s/teamnet/routes", AccountRouteRoot, api.AccountID)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)

	if err != nil {
		return []TunnelRoute{}, err
	}

	var resp tunnelRouteListResponse
	err = json.Unmarshal(res, &resp)
	if err != nil {
		return []TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return resp.Result, nil
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"github.com/toorop/tmail/message"
	"github.com/toorop/tmail/scope"
	"github.com/toorop/tmail/store"
	"github.com/jinzhu/gorm"
	"strings"
	//"github.com/bitly/go-nsq"
	"errors"
	"io"
	"net/mail"
	"sync"
	"time"
)

type QMessage struct {
	sync.Mutex
	Id                      int64
	Key                     string // identifier  -> store.Get(key)
	MailFrom                string
	AuthUser                string // Si il y a eu authetification SMTP contier le login/user sert pour le routage
	ReturnPath              string
	RcptTo                  string
	Host                    string
	AddedAt                 time.Time
	NextDeliveryScheduledAt time.Time
	Status                  uint32 // 0 delivery in progress, 1 to be discarded, 2 scheduled, 3 to be bounced
	DeliveryFailedCount     uint32
}

// Delete delete message from queue
func (q *QMessage) Delete() error {
	q.Lock()
	defer q.Unlock()
	var err error
	// remove from DB
	if err = scope.DB.Delete(q).Error; err != nil {
		return err
	}

	// If there is no other reference in DB, remove raw message from store
	var c uint
	if err = scope.DB.Model(QMessage{}).Where("'key' = ?", q.Key).Count(&c).Error; err != nil {
		return err
	}
	if c != 0 {
		return nil
	}
	qStore, err := store.New(scope.Cfg.GetStoreDriver(), scope.Cfg.GetStoreSource())
	if err != nil {
		return err
	}
	err = qStore.Del(q.Key)
	// Si le fichier n'existe pas ce n'est pas une véritable erreur
	if err != nil && strings.Contains(err.Error(), "no such file") {
		err = nil
	}
	return err
}

// UpdateFromDb update message from DB
func (q *QMessage) UpdateFromDb() error {
	q.Lock()
	defer q.Unlock()
	return scope.DB.Find(q).Error
}

// SaveInDb save qMessage in DB
func (q *QMessage) SaveInDb() error {
	q.Lock()
	defer q.Unlock()
	return scope.DB.Save(q).Error
}

// Discard mark message as being discarded on next delivery attemp
func (q *QMessage) Discard() error {
	if q.Status == 0 {
		return errors.New("delivery in progress, message status can't be changed")
	}
	q.Lock()
	q.Status = 1
	q.Unlock()
	return q.SaveInDb()
}

// Bounce mark message as being bounced on next delivery attemp
func (q *QMessage) Bounce() error {
	if q.Status == 0 {
		return errors.New("delivery in progress, message status can't be changed")
	}
	q.Lock()
	q.Status = 3
	q.Unlock()
	return q.SaveInDb()
}

// GetMessageByKey return a message from is key
func QueueGetMessageById(id int64) (msg *QMessage, err error) {
	msg = &QMessage{}
	err = scope.DB.Where("id = ?", id).First(msg).Error
	if err != nil && err == gorm.RecordNotFound {
		err = errors.New("not found")
	}
	return
}

// Add add a new mail in queue
func QueueAddMessage(msg *message.Message, envelope message.Envelope, authUser string) (key string, err error) {
	qStore, err := store.New(scope.Cfg.GetStoreDriver(), scope.Cfg.GetStoreSource())
	if err != nil {
		return
	}
	rawMess, err := msg.GetRaw()
	if err != nil {
		return
	}

	// Retun Path
	returnPath := ""
	// Exist ?
	if msg.HaveHeader("return-path") {
		t, err := mail.ParseAddress(msg.GetHeader("return-path"))
		if err != nil {
			return "", err
		}
		returnPath = t.Address
	} else {
		returnPath = envelope.MailFrom

	}

	// generate key
	hasher := sha1.New()
	if _, err = io.Copy(hasher, bytes.NewReader(rawMess)); err != nil {
		return
	}
	key = fmt.Sprintf("%x", hasher.Sum(nil))
	err = qStore.Put(key, bytes.NewReader(rawMess))
	if err != nil {
		return
	}

	// init new producer
	/*var producer *nsq.Producer
	nsqCfg := nsq.NewConfig()
	nsqCfg.UserAgent = "tmail.smtpd"

	producer, err = nsq.NewProducer("127.0.0.1:4150", nsqCfg)
	if err != nil {
		return
	}*/
	//defer producer.Stop()

	cloop := 0
	for _, rcptTo := range envelope.RcptTo {
		qm := QMessage{
			Key:                 key,
			AuthUser:            authUser,
			MailFrom:            envelope.MailFrom,
			ReturnPath:          returnPath,
			RcptTo:              rcptTo,
			Host:                message.GetHostFromAddress(rcptTo),
			AddedAt:             time.Now(),
			Status:              0,
			DeliveryFailedCount: 0,
		}

		// create record in db
		err = scope.DB.Create(&qm).Error
		if err != nil {
			// Rollback on storage
			if cloop == 0 {
				qStore.Del(key)
			}
			return
		}

		// publish
		var jMsg []byte
		jMsg, err = json.Marshal(qm)
		if err != nil {
			if cloop == 0 {
				qStore.Del(key)
			}
			scope.DB.Delete(&qm)
			return
		}
		// queue local  | queue remote
		err = scope.NsqQueueProducer.Publish("todeliver", jMsg)
		if err != nil {
			if cloop == 0 {
				qStore.Del(key)
			}
			scope.DB.Delete(&qm)
			return
		}
		cloop++
	}
	return
}

// ListMessage return all message in queue
func QueueListMessages() ([]QMessage, error) {
	messages := []QMessage{}
	err := scope.DB.Find(&messages).Error
	return messages, err
}
<commit_msg>bugfix count msg in queue<commit_after>package core

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"github.com/jinzhu/gorm"
	"github.com/toorop/tmail/message"
	"github.com/toorop/tmail/scope"
	"github.com/toorop/tmail/store"
	"strings"
	//"github.com/bitly/go-nsq"
	"errors"
	"io"
	"net/mail"
	"sync"
	"time"
)

type QMessage struct {
	sync.Mutex
	Id                      int64
	Key                     string // identifier  -> store.Get(key)
	MailFrom                string
	AuthUser                string // Si il y a eu authetification SMTP contier le login/user sert pour le routage
	ReturnPath              string
	RcptTo                  string
	Host                    string
	AddedAt                 time.Time
	NextDeliveryScheduledAt time.Time
	Status                  uint32 // 0 delivery in progress, 1 to be discarded, 2 scheduled, 3 to be bounced
	DeliveryFailedCount     uint32
}

// Delete delete message from queue
func (q *QMessage) Delete() error {
	q.Lock()
	defer q.Unlock()
	var err error
	// remove from DB
	if err = scope.DB.Delete(q).Error; err != nil {
		return err
	}

	// If there is no other reference in DB, remove raw message from store
	var c uint
	if err = scope.DB.Model(QMessage{}).Where("key = ?", q.Key).Count(&c).Error; err != nil {
		return err
	}
	if c != 0 {
		return nil
	}
	qStore, err := store.New(scope.Cfg.GetStoreDriver(), scope.Cfg.GetStoreSource())
	if err != nil {
		return err
	}
	err = qStore.Del(q.Key)
	// Si le fichier n'existe pas ce n'est pas une véritable erreur
	if err != nil && strings.Contains(err.Error(), "no such file") {
		err = nil
	}
	return err
}

// UpdateFromDb update message from DB
func (q *QMessage) UpdateFromDb() error {
	q.Lock()
	defer q.Unlock()
	return scope.DB.Find(q).Error
}

// SaveInDb save qMessage in DB
func (q *QMessage) SaveInDb() error {
	q.Lock()
	defer q.Unlock()
	return scope.DB.Save(q).Error
}

// Discard mark message as being discarded on next delivery attemp
func (q *QMessage) Discard() error {
	if q.Status == 0 {
		return errors.New("delivery in progress, message status can't be changed")
	}
	q.Lock()
	q.Status = 1
	q.Unlock()
	return q.SaveInDb()
}

// Bounce mark message as being bounced on next delivery attemp
func (q *QMessage) Bounce() error {
	if q.Status == 0 {
		return errors.New("delivery in progress, message status can't be changed")
	}
	q.Lock()
	q.Status = 3
	q.Unlock()
	return q.SaveInDb()
}

// GetMessageByKey return a message from is key
func QueueGetMessageById(id int64) (msg *QMessage, err error) {
	msg = &QMessage{}
	err = scope.DB.Where("id = ?", id).First(msg).Error
	if err != nil && err == gorm.RecordNotFound {
		err = errors.New("not found")
	}
	return
}

// Add add a new mail in queue
func QueueAddMessage(msg *message.Message, envelope message.Envelope, authUser string) (key string, err error) {
	qStore, err := store.New(scope.Cfg.GetStoreDriver(), scope.Cfg.GetStoreSource())
	if err != nil {
		return
	}
	rawMess, err := msg.GetRaw()
	if err != nil {
		return
	}

	// Retun Path
	returnPath := ""
	// Exist ?
	if msg.HaveHeader("return-path") {
		t, err := mail.ParseAddress(msg.GetHeader("return-path"))
		if err != nil {
			return "", err
		}
		returnPath = t.Address
	} else {
		returnPath = envelope.MailFrom

	}

	// generate key
	hasher := sha1.New()
	if _, err = io.Copy(hasher, bytes.NewReader(rawMess)); err != nil {
		return
	}
	key = fmt.Sprintf("%x", hasher.Sum(nil))
	err = qStore.Put(key, bytes.NewReader(rawMess))
	if err != nil {
		return
	}

	// init new producer
	/*var producer *nsq.Producer
	nsqCfg := nsq.NewConfig()
	nsqCfg.UserAgent = "tmail.smtpd"

	producer, err = nsq.NewProducer("127.0.0.1:4150", nsqCfg)
	if err != nil {
		return
	}*/
	//defer producer.Stop()

	cloop := 0
	qmessages := []QMessage{}
	for _, rcptTo := range envelope.RcptTo {
		qm := QMessage{
			Key:                 key,
			AuthUser:            authUser,
			MailFrom:            envelope.MailFrom,
			ReturnPath:          returnPath,
			RcptTo:              rcptTo,
			Host:                message.GetHostFromAddress(rcptTo),
			AddedAt:             time.Now(),
			Status:              0,
			DeliveryFailedCount: 0,
		}

		// create record in db
		err = scope.DB.Create(&qm).Error
		if err != nil {
			// Rollback on storage
			if cloop == 0 {
				qStore.Del(key)
			}
			return
		}
		cloop++
		qmessages = append(qmessages, qm)
	}

	for _, qmsg := range qmessages {
		// publish
		var jMsg []byte
		jMsg, err = json.Marshal(qmsg)
		if err != nil {
			if cloop == 1 {
				qStore.Del(key)
			}
			scope.DB.Delete(&qmsg)
			return
		}
		// queue local  | queue remote
		err = scope.NsqQueueProducer.Publish("todeliver", jMsg)
		if err != nil {
			if cloop == 1 {
				qStore.Del(key)
			}
			scope.DB.Delete(&qmsg)
			return
		}
	}
	return
}

// ListMessage return all message in queue
func QueueListMessages() ([]QMessage, error) {
	messages := []QMessage{}
	err := scope.DB.Find(&messages).Error
	return messages, err
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package metricsdebug

import (
	"fmt"
	"strings"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"gopkg.in/juju/charm.v6-unstable"
	"gopkg.in/juju/names.v2"

	"github.com/juju/juju/api"
	actionapi "github.com/juju/juju/api/action"
	"github.com/juju/juju/api/application"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/juju/action"
	"github.com/juju/juju/cmd/modelcmd"
)

// TODO(bogdanteleaga): update this once querying for actions by name is implemented.
const collectMetricsDoc = `
Trigger metrics collection

This command waits for the metric collection to finish before returning.
You may abort this command and it will continue to run asynchronously.
Results may be checked  by 'juju show-action-status'.
`

const (
	// commandTimeout represents the timeout for executing the command itself
	commandTimeout = 3 * time.Second
)

var logger = loggo.GetLogger("juju.cmd.juju.collect-metrics")

// collectMetricsCommand retrieves metrics stored in the juju controller.
type collectMetricsCommand struct {
	modelcmd.ModelCommandBase
	unit    string
	service string
	entity  string
}

// NewCollectMetricsCommand creates a new collectMetricsCommand.
func NewCollectMetricsCommand() cmd.Command {
	return modelcmd.Wrap(&collectMetricsCommand{})
}

// Info implements Command.Info.
func (c *collectMetricsCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "collect-metrics",
		Args:    "[application or unit]",
		Purpose: "Collect metrics on the given unit/application.",
		Doc:     collectMetricsDoc,
	}
}

// Init reads and verifies the cli arguments for the collectMetricsCommand
func (c *collectMetricsCommand) Init(args []string) error {
	if len(args) == 0 {
		return errors.New("you need to specify a unit or application.")
	}
	c.entity = args[0]
	if names.IsValidUnit(c.entity) {
		c.unit = c.entity
	} else if names.IsValidApplication(args[0]) {
		c.service = c.entity
	} else {
		return errors.Errorf("%q is not a valid unit or application", args[0])
	}
	if err := cmd.CheckEmpty(args[1:]); err != nil {
		return errors.Errorf("unknown command line arguments: " + strings.Join(args, ","))
	}
	return nil
}

type runClient interface {
	action.APIClient
	Run(run params.RunParams) ([]params.ActionResult, error)
}

var newRunClient = func(conn api.Connection) runClient {
	return actionapi.NewClient(conn)
}

func parseRunOutput(result params.ActionResult) (string, string, error) {
	if result.Error != nil {
		return "", "", result.Error
	}
	stdout, ok := result.Output["Stdout"].(string)
	if !ok {
		return "", "", errors.New("could not read stdout")
	}
	stderr, ok := result.Output["Stderr"].(string)
	if !ok {
		return "", "", errors.New("could not read stderr")
	}
	return strings.Trim(stdout, " \t\n"), strings.Trim(stderr, " \t\n"), nil
}

func parseActionResult(result params.ActionResult) (string, error) {
	if result.Action != nil {
		logger.Infof("ran action id %v", result.Action.Tag)
	}
	_, stderr, err := parseRunOutput(result)
	if err != nil {
		return "", errors.Trace(err)
	}
	tag, err := names.ParseUnitTag(result.Action.Receiver)
	if err != nil {
		return "", errors.Trace(err)
	}
	if strings.Contains(stderr, "nc: unix connect failed: No such file or directory") {
		return "", errors.New("no collect application listening: does application support metric collection?")
	}
	return tag.Id(), nil
}

type serviceClient interface {
	GetCharmURL(service string) (*charm.URL, error)
}

var newServiceClient = func(root api.Connection) serviceClient {
	return application.NewClient(root)
}

func isLocalCharmURL(conn api.Connection, entity string) (bool, error) {
	serviceName := entity
	var err error
	if names.IsValidUnit(entity) {
		serviceName, err = names.UnitApplication(entity)
		if err != nil {
			return false, errors.Trace(err)
		}
	}

	client := newServiceClient(conn)
	// TODO (mattyw, anastasiamac) The storage work might lead to an api
	// allowing us to query charm url for a unit.
	// When that api exists we should use that here.
	url, err := client.GetCharmURL(serviceName)
	if err != nil {
		return false, errors.Trace(err)
	}
	return url.Schema == "local", nil
}

var newAPIConn = func(cmd modelcmd.ModelCommandBase) (api.Connection, error) {
	return cmd.NewAPIRoot()
}

// Run implements Command.Run.
func (c *collectMetricsCommand) Run(ctx *cmd.Context) error {
	root, err := newAPIConn(c.ModelCommandBase)
	if err != nil {
		return errors.Trace(err)
	}
	runnerClient := newRunClient(root)
	defer runnerClient.Close()

	islocal, err := isLocalCharmURL(root, c.entity)
	if err != nil {
		return errors.Annotate(err, "failed to find charmURL for entity")
	}
	if !islocal {
		return errors.Errorf("%q is not a local charm", c.entity)
	}

	units := []string{}
	services := []string{}
	if c.unit != "" {
		units = []string{c.unit}
	}
	if c.service != "" {
		services = []string{c.service}
	}
	runParams := params.RunParams{
		Timeout:      commandTimeout,
		Units:        units,
		Applications: services,
		Commands:     "nc -U ../metrics-collect.socket",
	}

	// trigger metrics collection
	runResults, err := runnerClient.Run(runParams)
	if err != nil {
		return errors.Trace(err)
	}

	// We want to wait for the action results indefinitely.  Discard the tick.
	wait := time.NewTimer(0 * time.Second)
	_ = <-wait.C
	// trigger sending metrics in parallel
	resultChannel := make(chan string, len(runResults))
	for _, result := range runResults {
		r := result
		if r.Error != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		tag, err := names.ParseActionTag(r.Action.Tag)
		if err != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		actionResult, err := getActionResult(runnerClient, tag.Id(), wait)
		if err != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		unitId, err := parseActionResult(actionResult)
		if err != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		go func() {
			defer func() {
				resultChannel <- unitId
			}()
			sendParams := params.RunParams{
				Timeout:  commandTimeout,
				Units:    []string{unitId},
				Commands: "nc -U ../metrics-send.socket",
			}
			sendResults, err := runnerClient.Run(sendParams)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			if len(sendResults) != 1 {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v\n", unitId)
				return
			}
			if sendResults[0].Error != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, sendResults[0].Error)
				return
			}
			tag, err := names.ParseActionTag(sendResults[0].Action.Tag)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			actionResult, err := getActionResult(runnerClient, tag.Id(), wait)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			stdout, stderr, err := parseRunOutput(actionResult)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			if stdout != "ok" {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, errors.New(stderr))
			}
		}()
	}

	for _ = range runResults {
		// The default is to wait forever for the command to finish.
		select {
		case <-resultChannel:
		}
	}
	return nil
}

// getActionResult abstracts over the action CLI function that we use here to fetch results
var getActionResult = func(c runClient, actionId string, wait *time.Timer) (params.ActionResult, error) {
	return action.GetActionResult(c, actionId, wait)
}
<commit_msg>remove space<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package metricsdebug

import (
	"fmt"
	"strings"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"gopkg.in/juju/charm.v6-unstable"
	"gopkg.in/juju/names.v2"

	"github.com/juju/juju/api"
	actionapi "github.com/juju/juju/api/action"
	"github.com/juju/juju/api/application"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/juju/action"
	"github.com/juju/juju/cmd/modelcmd"
)

// TODO(bogdanteleaga): update this once querying for actions by name is implemented.
const collectMetricsDoc = `
Trigger metrics collection

This command waits for the metric collection to finish before returning.
You may abort this command and it will continue to run asynchronously.
Results may be checked by 'juju show-action-status'.
`

const (
	// commandTimeout represents the timeout for executing the command itself
	commandTimeout = 3 * time.Second
)

var logger = loggo.GetLogger("juju.cmd.juju.collect-metrics")

// collectMetricsCommand retrieves metrics stored in the juju controller.
type collectMetricsCommand struct {
	modelcmd.ModelCommandBase
	unit    string
	service string
	entity  string
}

// NewCollectMetricsCommand creates a new collectMetricsCommand.
func NewCollectMetricsCommand() cmd.Command {
	return modelcmd.Wrap(&collectMetricsCommand{})
}

// Info implements Command.Info.
func (c *collectMetricsCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "collect-metrics",
		Args:    "[application or unit]",
		Purpose: "Collect metrics on the given unit/application.",
		Doc:     collectMetricsDoc,
	}
}

// Init reads and verifies the cli arguments for the collectMetricsCommand
func (c *collectMetricsCommand) Init(args []string) error {
	if len(args) == 0 {
		return errors.New("you need to specify a unit or application.")
	}
	c.entity = args[0]
	if names.IsValidUnit(c.entity) {
		c.unit = c.entity
	} else if names.IsValidApplication(args[0]) {
		c.service = c.entity
	} else {
		return errors.Errorf("%q is not a valid unit or application", args[0])
	}
	if err := cmd.CheckEmpty(args[1:]); err != nil {
		return errors.Errorf("unknown command line arguments: " + strings.Join(args, ","))
	}
	return nil
}

type runClient interface {
	action.APIClient
	Run(run params.RunParams) ([]params.ActionResult, error)
}

var newRunClient = func(conn api.Connection) runClient {
	return actionapi.NewClient(conn)
}

func parseRunOutput(result params.ActionResult) (string, string, error) {
	if result.Error != nil {
		return "", "", result.Error
	}
	stdout, ok := result.Output["Stdout"].(string)
	if !ok {
		return "", "", errors.New("could not read stdout")
	}
	stderr, ok := result.Output["Stderr"].(string)
	if !ok {
		return "", "", errors.New("could not read stderr")
	}
	return strings.Trim(stdout, " \t\n"), strings.Trim(stderr, " \t\n"), nil
}

func parseActionResult(result params.ActionResult) (string, error) {
	if result.Action != nil {
		logger.Infof("ran action id %v", result.Action.Tag)
	}
	_, stderr, err := parseRunOutput(result)
	if err != nil {
		return "", errors.Trace(err)
	}
	tag, err := names.ParseUnitTag(result.Action.Receiver)
	if err != nil {
		return "", errors.Trace(err)
	}
	if strings.Contains(stderr, "nc: unix connect failed: No such file or directory") {
		return "", errors.New("no collect application listening: does application support metric collection?")
	}
	return tag.Id(), nil
}

type serviceClient interface {
	GetCharmURL(service string) (*charm.URL, error)
}

var newServiceClient = func(root api.Connection) serviceClient {
	return application.NewClient(root)
}

func isLocalCharmURL(conn api.Connection, entity string) (bool, error) {
	serviceName := entity
	var err error
	if names.IsValidUnit(entity) {
		serviceName, err = names.UnitApplication(entity)
		if err != nil {
			return false, errors.Trace(err)
		}
	}

	client := newServiceClient(conn)
	// TODO (mattyw, anastasiamac) The storage work might lead to an api
	// allowing us to query charm url for a unit.
	// When that api exists we should use that here.
	url, err := client.GetCharmURL(serviceName)
	if err != nil {
		return false, errors.Trace(err)
	}
	return url.Schema == "local", nil
}

var newAPIConn = func(cmd modelcmd.ModelCommandBase) (api.Connection, error) {
	return cmd.NewAPIRoot()
}

// Run implements Command.Run.
func (c *collectMetricsCommand) Run(ctx *cmd.Context) error {
	root, err := newAPIConn(c.ModelCommandBase)
	if err != nil {
		return errors.Trace(err)
	}
	runnerClient := newRunClient(root)
	defer runnerClient.Close()

	islocal, err := isLocalCharmURL(root, c.entity)
	if err != nil {
		return errors.Annotate(err, "failed to find charmURL for entity")
	}
	if !islocal {
		return errors.Errorf("%q is not a local charm", c.entity)
	}

	units := []string{}
	services := []string{}
	if c.unit != "" {
		units = []string{c.unit}
	}
	if c.service != "" {
		services = []string{c.service}
	}
	runParams := params.RunParams{
		Timeout:      commandTimeout,
		Units:        units,
		Applications: services,
		Commands:     "nc -U ../metrics-collect.socket",
	}

	// trigger metrics collection
	runResults, err := runnerClient.Run(runParams)
	if err != nil {
		return errors.Trace(err)
	}

	// We want to wait for the action results indefinitely.  Discard the tick.
	wait := time.NewTimer(0 * time.Second)
	_ = <-wait.C
	// trigger sending metrics in parallel
	resultChannel := make(chan string, len(runResults))
	for _, result := range runResults {
		r := result
		if r.Error != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		tag, err := names.ParseActionTag(r.Action.Tag)
		if err != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		actionResult, err := getActionResult(runnerClient, tag.Id(), wait)
		if err != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		unitId, err := parseActionResult(actionResult)
		if err != nil {
			fmt.Fprintf(ctx.Stdout, "failed to collect metrics: %v\n", err)
			resultChannel <- "invalid id"
			continue
		}
		go func() {
			defer func() {
				resultChannel <- unitId
			}()
			sendParams := params.RunParams{
				Timeout:  commandTimeout,
				Units:    []string{unitId},
				Commands: "nc -U ../metrics-send.socket",
			}
			sendResults, err := runnerClient.Run(sendParams)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			if len(sendResults) != 1 {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v\n", unitId)
				return
			}
			if sendResults[0].Error != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, sendResults[0].Error)
				return
			}
			tag, err := names.ParseActionTag(sendResults[0].Action.Tag)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			actionResult, err := getActionResult(runnerClient, tag.Id(), wait)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			stdout, stderr, err := parseRunOutput(actionResult)
			if err != nil {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, err)
				return
			}
			if stdout != "ok" {
				fmt.Fprintf(ctx.Stdout, "failed to send metrics for unit %v: %v\n", unitId, errors.New(stderr))
			}
		}()
	}

	for _ = range runResults {
		// The default is to wait forever for the command to finish.
		select {
		case <-resultChannel:
		}
	}
	return nil
}

// getActionResult abstracts over the action CLI function that we use here to fetch results
var getActionResult = func(c runClient, actionId string, wait *time.Timer) (params.ActionResult, error) {
	return action.GetActionResult(c, actionId, wait)
}
<|endoftext|>


<commit_before>package gateway

import (
	"io"
	"net/http"
	"strconv"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/httprouter"
	log "github.com/funkygao/log4go"
)

// PUT /v1/msgs/:appid/:topic/:ver?group=xx&q=<dead|retry>
// FIXME X-Bury and param q is duplicated
func (this *subServer) buryHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		group      string
		rawTopic   string
		shadow     string
		bury       string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64 = -1
		err        error
	)

	query := r.URL.Query()
	group = query.Get("group")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	bury = r.Header.Get(HttpHeaderMsgBury)
	if !sla.ValidateShadowName(bury) {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} illegal bury: %s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), bury)

		writeBadRequest(w, "illegal bury")
		return
	}

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} %v",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	partition = r.Header.Get(HttpHeaderPartition)
	offset = r.Header.Get(HttpHeaderOffset)
	if partition == "" || offset == "" {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} empty offset or partition",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"))

		writeBadRequest(w, "empty offset or partition")
		return
	}

	offsetN, err = strconv.ParseInt(offset, 10, 64)
	if err != nil || offsetN < 0 {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} illegal offset:%s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), offset)

		writeBadRequest(w, "bad offset")
		return
	}
	partitionN, err = strconv.Atoi(partition)
	if err != nil || partitionN < 0 {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} illegal partition:%s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), partition)

		writeBadRequest(w, "bad partition")
		return
	}

	shadow = query.Get("q")

	log.Debug("bury[%s/%s] %s(%s): {%s.%s.%s bury:%s shadow=%s partition:%s offset:%s UA:%s}",
		myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, bury, shadow,
		partition, offset, r.Header.Get("User-Agent"))

	msgLen := int(r.ContentLength)
	msg := make([]byte, msgLen)
	if _, err = io.ReadAtLeast(r.Body, msg, msgLen); err != nil {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} %v",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} invalid appid:%s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), hisAppid)

		writeBadRequest(w, "invalid appid")
		return
	}

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("bury[%s/%s] %s(%s): {%s.%s.%s q:%s UA:%s} invalid shadow name",
				myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("bury[%s/%s] %s(%s): {%s.%s.%s q:%s UA:%s} not a shadowed topic",
				myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, realIp, "", Options.PermitStandbySub)
	if err != nil {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} %v",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// step1: pub
	shadowTopic := manager.Default.ShadowTopic(bury, myAppid, hisAppid, topic, ver, group)
	_, _, err = store.DefaultPubStore.SyncPub(cluster, shadowTopic, nil, msg)
	if err != nil {
		log.Error("bury[%s/%s] %s(%s): %s.%s.%s %v",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, err)

		writeServerError(w, err.Error())
		return
	}

	// step2: skip this message in the master topic TODO atomic with step1
	if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
		Topic:     rawTopic, // FIXME it's wrong!!!
		Partition: int32(partitionN),
		Offset:    offsetN,
	}); err != nil {
		log.Error("bury[%s/%s] %s(%s): %s.%s.%s %v",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, err)

		writeServerError(w, err.Error())
		return
	}

	w.Write(ResponseOk)
}
<commit_msg>WIP bury<commit_after>package gateway

import (
	"io"
	"net/http"
	"strconv"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/httprouter"
	log "github.com/funkygao/log4go"
)

// PUT /v1/msgs/:appid/:topic/:ver?group=xx&q=<dead|retry>
// q=retry&X-Bury=dead means bury from retry queue to dead queue
func (this *subServer) buryHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		group      string
		rawTopic   string
		shadow     string
		bury       string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64 = -1
		err        error
	)

	query := r.URL.Query()
	group = query.Get("group")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	bury = r.Header.Get(HttpHeaderMsgBury)
	if !sla.ValidateShadowName(bury) {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} illegal bury: %s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), bury)

		writeBadRequest(w, "illegal bury")
		return
	}

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} %v",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	partition = r.Header.Get(HttpHeaderPartition)
	offset = r.Header.Get(HttpHeaderOffset)
	if partition == "" || offset == "" {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} empty offset or partition",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"))

		writeBadRequest(w, "empty offset or partition")
		return
	}

	offsetN, err = strconv.ParseInt(offset, 10, 64)
	if err != nil || offsetN < 0 {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} illegal offset:%s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), offset)

		writeBadRequest(w, "bad offset")
		return
	}
	partitionN, err = strconv.Atoi(partition)
	if err != nil || partitionN < 0 {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} illegal partition:%s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), partition)

		writeBadRequest(w, "bad partition")
		return
	}

	shadow = query.Get("q")

	log.Debug("bury[%s/%s] %s(%s): {%s.%s.%s bury:%s shadow=%s partition:%s offset:%s UA:%s}",
		myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, bury, shadow, partition, offset, r.Header.Get("User-Agent"))

	msgLen := int(r.ContentLength)
	msg := make([]byte, msgLen)
	if _, err = io.ReadAtLeast(r.Body, msg, msgLen); err != nil {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} %v",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("bury[%s/%s] %s(%s): {%s.%s.%s UA:%s} invalid appid:%s",
			myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, r.Header.Get("User-Agent"), hisAppid)

		writeBadRequest(w, "invalid appid")
		return
	}

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("bury[%s/%s] %s(%s): {%s.%s.%s q:%s UA:%s} invalid shadow name",
				myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("bury[%s/%s] %s(%s): {%s.%s.%s q:%s UA:%s} not a shadowed topic",
				myAppid, group, r.RemoteAddr, realIp, hisAppid, topic, ver, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, realIp, "", Options.PermitStandbySub)
	if err != nil {
		log.Error("bury[%s/%s] %s(%s): {%s UA:%s} %v",
			myAppid, group, r.RemoteAddr, realIp, rawTopic, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// step1: pub
	shadowTopic := manager.Default.ShadowTopic(bury, myAppid, hisAppid, topic, ver, group)
	_, _, err = store.DefaultPubStore.SyncPub(cluster, shadowTopic, nil, msg)
	if err != nil {
		log.Error("bury[%s/%s] %s(%s): %s %v", myAppid, group, r.RemoteAddr, realIp, shadowTopic, err)

		writeServerError(w, err.Error())
		return
	}

	// step2: skip this message in the master topic TODO atomic with step1
	if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
		Topic:     rawTopic,
		Partition: int32(partitionN),
		Offset:    offsetN,
	}); err != nil {
		log.Error("bury[%s/%s] %s(%s): %s %v", myAppid, group, r.RemoteAddr, realIp, rawTopic, err)

		writeServerError(w, err.Error())
		return
	}

	w.Write(ResponseOk)
}
<|endoftext|>


<commit_before>// Package gs implements utility for accessing Skia perf data in Google Storage.
package gs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"regexp"
	"strings"
	"time"
)

import (
	"code.google.com/p/google-api-go-client/storage/v1"
	"github.com/golang/glog"
)

import (
	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/types"
)

var (
	// dirMap maps dataset name to a slice with Google Storage subdirectory and file prefix.
	dirMap = map[string][]string{
		"skps":  {"pics-json-v2", "bench_"},
		"micro": {"stats-json-v2", "microbench2_"},
	}

	trybotDataPath = regexp.MustCompile(`^[a-z]*[/]?([0-9]{4}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9a-zA-Z-]+-Trybot/[0-9]+/[0-9]+)$`)
)

const (
	GS_PROJECT_BUCKET = "chromium-skia-gm"
)

// GetStorageService returns a Cloud Storage service.
func GetStorageService() (*storage.Service, error) {
	return storage.New(http.DefaultClient)
}

// lastDate takes a year and month, and returns the last day of the month.
//
// This is done by going to the first day 0:00 of the next month, subtracting an
// hour, then returning the date.
func lastDate(year int, month time.Month) int {
	return time.Date(year, month+1, 1, 0, 0, 0, 0, time.UTC).Add(-time.Hour).Day()
}

// GetLatestGSDirs gets the appropriate directory names in which data
// would be stored between the given timestamp range.
//
// The returning directories cover the range till the date of startTS, and may
// be precise to the hour.
func GetLatestGSDirs(startTS int64, endTS int64, bsSubdir string) []string {
	startTime := time.Unix(startTS, 0).UTC()
	startYear, startMonth, startDay := startTime.Date()
	glog.Infoln("GS dir start time: ", startTime)
	endTime := time.Unix(endTS, 0).UTC()
	lastAddedTime := startTime
	results := make([]string, 0)
	newYear, newMonth, newDay := endTime.Date()
	newHour := endTime.Hour()
	lastYear, lastMonth, _ := lastAddedTime.Date()
	if lastYear != newYear {
		for i := lastYear; i < newYear; i++ {
			if i != startYear {
				results = append(results, fmt.Sprintf("%04d", i))
			} else {
				for j := startMonth; j <= time.December; j++ {
					if j == startMonth && startDay > 1 {
						for k := startDay; k <= lastDate(i, j); k++ {
							results = append(results, fmt.Sprintf("%04d/%02d/%02d", i, j, k))
						}
					} else {
						results = append(results, fmt.Sprintf("%04d/%02d", i, j))
					}
				}
			}
		}
		lastAddedTime = time.Date(newYear, time.January, 1, 0, 0, 0, 0, time.UTC)
	}
	lastYear, lastMonth, _ = lastAddedTime.Date()
	if lastMonth != newMonth {
		for i := lastMonth; i < newMonth; i++ {
			if i != startMonth {
				results = append(results, fmt.Sprintf("%04d/%02d", lastYear, i))
			} else {
				for j := startDay; j <= lastDate(lastYear, i); j++ {
					results = append(results, fmt.Sprintf("%04d/%02d/%02d", lastYear, i, j))
				}
			}
		}
		lastAddedTime = time.Date(newYear, newMonth, 1, 0, 0, 0, 0, time.UTC)
	}
	lastYear, lastMonth, lastDay := lastAddedTime.Date()
	if lastDay != newDay {
		for i := lastDay; i < newDay; i++ {
			results = append(results, fmt.Sprintf("%04d/%02d/%02d", lastYear, lastMonth, i))
		}
		lastAddedTime = time.Date(newYear, newMonth, newDay, 0, 0, 0, 0, time.UTC)
	}
	lastYear, lastMonth, lastDay = lastAddedTime.Date()
	lastHour := lastAddedTime.Hour()
	for i := lastHour; i < newHour+1; i++ {
		results = append(results, fmt.Sprintf("%04d/%02d/%02d/%02d", lastYear, lastMonth, lastDay, i))
	}
	for i := range results {
		results[i] = fmt.Sprintf("%s/%s", bsSubdir, results[i])
	}
	return results
}

// RunInfo stores trybot run result info for a requester.
//
// Issues maps a string representing Reitveld issue info to a slice of dirs
// containing its try results. A sample dir looks like:
// "2014/07/31/18/Perf-Win7-ShuttleA-HD2000-x86-Release-Trybot/75/423413006"
type RunInfo struct {
	Requester string              `json:"requester"`
	Issues    map[string][]string `json:"issues"`
}

// TryInfo stores try result information on Google Storage bench files.
type TryInfo struct {
	Results []*RunInfo `json:"results"`
}

// IssueInfo stores information on a specific issue.
//
// Information is read from the Reitveld JSON api, for instance,
// https://codereview.chromium.org/api/427903003
// Only information we care about is stored.
type IssueInfo struct {
	Owner   string `json:"owner"`
	Subject string `json:"subject"`
}

// JSONPerfInput stores the input JSON data that we care about. Currently this
// includes "key" and "value" fields in perf/server/(microbench|skpbench).json.
type JSONPerfInput struct {
	Value  float64                `json:"value"`
	Params map[string]interface{} `json:"params"`
}

// RequestForStorageURL returns an http.Request for a given Cloud Storage URL.
// This is workaround of a known issue: embedded slashes in URLs require use of
// URL.Opaque property
func RequestForStorageURL(url string) (*http.Request, error) {
	r, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("HTTP new request error: %s", err)
	}
	schemePos := strings.Index(url, ":")
	queryPos := strings.Index(url, "?")
	if queryPos == -1 {
		queryPos = len(url)
	}
	r.URL.Opaque = url[schemePos+1 : queryPos]
	return r, nil
}

// getTryData takes a prefix path and dataset, and returns the trybot JSON data stored in
// Google Storage under the prefix.
//
// The given prefix path is the path to a trybot build result, such as:
// "trybots/micro/2014/07/16/01/Perf-Win7-ShuttleA-HD2000-x86-Release-Trybot/57/423413006"
//
// Currently it takes in JSON format that's used for BigQuery ingestion, and
// outputs in the TileGUI format defined in src/types. Only the Traces fields
// are populated in the TileGUI, with data containing [[0, <value>]] for just
// one data point per key.
//
// TODO(bensong) adjust input/output formats as needed by the inputs and the
// frontend.
func getTryData(prefix string, dataset config.DatasetName) ([]byte, error) {
	gs, err := GetStorageService()
	if err != nil {
		return nil, fmt.Errorf("Unable to get GS service: %s", nil)
	}
	t := types.NewTileGUI(-1, -1) // Tile level/number don't matter here.
	req := gs.Objects.List(GS_PROJECT_BUCKET).Prefix(prefix)
	for req != nil {
		resp, err := req.Do()
		if err != nil {
			return nil, fmt.Errorf("Google Storage request error: %s", err)
		}
		for _, result := range resp.Items {
			// Use ingester.BenchFile here.
			r, err := RequestForStorageURL(result.MediaLink)
			if err != nil {
				return nil, fmt.Errorf("Google Storage MediaLink request error: %s", err)
			}
			res, err := http.DefaultClient.Do(r)
			defer res.Body.Close()
			if err != nil {
				return nil, fmt.Errorf("GET error: %s", err)
			}
			body, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return nil, fmt.Errorf("Read body error: %s", err)
			}
			i := JSONPerfInput{}
			for _, j := range bytes.Split(body, []byte("\n")) {
				if len(j) == 0 {
					continue
				}
				if err := json.Unmarshal(j, &i); err != nil {
					return nil, fmt.Errorf("JSON unmarshal error: %s", err)
				}
				newData := make([][2]float64, 0)
				newData = append(newData, [2]float64{
					0.0, // Commit timestamp is unused.
					i.Value,
				})
				if _, exists := i.Params["builderName"]; !exists {
					continue
				}
				// Remove the -Trybot prefix so the trybot keys
				// and normal keys match.
				i.Params["builderName"] = strings.Replace(fmt.Sprint(i.Params["builderName"]), "-Trybot", "", 1)
				/*
					t.Traces = append(t.Traces, types.TraceGUI{
						Data: newData,
						Label:  string(types.MakeTraceKey(i.Params)),
					})
				*/
			}
		}
		if len(resp.NextPageToken) > 0 {
			req.PageToken(resp.NextPageToken)
		} else {
			req = nil
		}
	}
	d, err := json.Marshal(t)
	if err != nil {
		return nil, fmt.Errorf("JSON marshal error: %s", err)
	}
	return d, nil
}

// GetTryResults takes a string of path info, an end timestamp, and the
// number of days to look back, and returns corresponding trybot results from
// Google Storage.
//
// When a full bench path is not given (only the optional dataset info), returns
// a JSON bytes marshalled from TryInfo.
//
// If a valid urlpath is given for a specific try run, returns JSON from
// getTryData() above.
//
// TODO(bensong): add metrics for GS roundtrip time and failure rates.
func GetTryResults(urlpath string, endTS int64, daysback int) ([]byte, error) {
	dirParts := strings.Split(urlpath, "/")
	datasetName := config.DATASET_SKP
	dataset := "pics-json-v2"
	dataFilePrefix := "bench_"
	if k, ok := dirMap[dirParts[0]]; ok {
		datasetName = config.DatasetName(dirParts[0])
		dataset = k[0]
		dataFilePrefix = k[1]
	}
	if len(dirParts) == 1 { // Tries to return list of try result dirs.
		results := &TryInfo{
			Results: []*RunInfo{},
		}
		dirs := GetLatestGSDirs(time.Unix(endTS, 0).UTC().AddDate(0, 0, 0-daysback).Unix(), endTS, "trybot/"+dataset)
		if len(dirs) == 0 {
			return json.Marshal(results)
		}
		gs, err := GetStorageService()
		if err != nil {
			return nil, fmt.Errorf("Unable to get GS service: %s", nil)
		}
		m := make(map[string]bool)
		for _, dir := range dirs {
			req := gs.Objects.List(GS_PROJECT_BUCKET).Prefix(dir)
			for req != nil {
				resp, err := req.Do()
				if err != nil {
					return nil, fmt.Errorf("Google Storage request error: %s", err)
				}
				for _, result := range resp.Items {
					// Extracts the useful parts.
					dirPath := path.Dir(result.Name)
					// Removes "trybot" and dataset
					toReturn := strings.Split(dirPath, "/")[2:]
					m[strings.Join(toReturn, "/")] = true
				}
				if len(resp.NextPageToken) > 0 {
					req.PageToken(resp.NextPageToken)
				} else {
					req = nil
				}
			}
		}
		requesterIssues := map[string][]string{}
		issueDirs := map[string][]string{}
		issueDescription := map[string]string{}

		for k := range m {
			if match := trybotDataPath.FindStringSubmatch(k); match == nil {
				glog.Infoln("Unexpected try path, skipping: ", k)
				continue
			}
			s := strings.Split(k, "/")
			issue := s[len(s)-1]
			if _, ok := issueDirs[issue]; !ok {
				issueDirs[issue] = []string{}
			}
			issueDirs[issue] = append(issueDirs[issue], k)

			if _, ok := issueDescription[issue]; ok {
				continue
			}
			resp, err := http.Get("https://codereview.chromium.org/api/" + issue)
			defer resp.Body.Close()
			owner := "unknown"
			description := "unknown"
			if err != nil {
				glog.Warningln("Could not get Reitveld info, use unknown: ", err)
			} else {
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					glog.Warningln("Could not read Reitveld info, use unknown: ", err)
				}
				i := IssueInfo{}
				if err := json.Unmarshal(body, &i); err != nil {
					glog.Warningln("Could not unmarshal Reitveld info, use unknown: ", err)
				} else {
					owner = i.Owner
					description = i.Subject
				}
			}
			issueDescription[issue] = description
			if _, ok := requesterIssues[owner]; !ok {
				requesterIssues[owner] = []string{}
			}
			requesterIssues[owner] = append(requesterIssues[owner], issue)
		}
		for k, v := range requesterIssues {
			issues := map[string][]string{}
			for _, i := range v {
				issues[i+": "+issueDescription[i]] = issueDirs[i]
			}
			r := &RunInfo{
				Requester: k,
				Issues:    issues,
			}
			results.Results = append(results.Results, r)
		}
		return json.Marshal(results)
	} else { // Tries to read stats from the given dir.
		if !trybotDataPath.MatchString(urlpath) {
			return nil, fmt.Errorf("Wrong URL path format for trybot stats: %s\n", urlpath)
		}
		trymatch := trybotDataPath.FindStringSubmatch(urlpath)
		if trymatch == nil { // This should never happen after the check above?
			return nil, fmt.Errorf("Cannot find trybot path in regexp for: %s\n", urlpath)
		}
		return getTryData(path.Join("trybot", dataset, trymatch[1], dataFilePrefix), datasetName)
	}
}
<commit_msg>Remove unused code, trybots are now done through ingestion.<commit_after>// Package gs implements utility for accessing Skia perf data in Google Storage.
package gs

import (
	"fmt"
	"net/http"
	"regexp"
	"strings"
	"time"
)

import (
	"code.google.com/p/google-api-go-client/storage/v1"
	"github.com/golang/glog"
)

var (
	// dirMap maps dataset name to a slice with Google Storage subdirectory and file prefix.
	dirMap = map[string][]string{
		"skps":  {"pics-json-v2", "bench_"},
		"micro": {"stats-json-v2", "microbench2_"},
	}

	trybotDataPath = regexp.MustCompile(`^[a-z]*[/]?([0-9]{4}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9a-zA-Z-]+-Trybot/[0-9]+/[0-9]+)$`)
)

const (
	GS_PROJECT_BUCKET = "chromium-skia-gm"
)

// GetStorageService returns a Cloud Storage service.
func GetStorageService() (*storage.Service, error) {
	return storage.New(http.DefaultClient)
}

// lastDate takes a year and month, and returns the last day of the month.
//
// This is done by going to the first day 0:00 of the next month, subtracting an
// hour, then returning the date.
func lastDate(year int, month time.Month) int {
	return time.Date(year, month+1, 1, 0, 0, 0, 0, time.UTC).Add(-time.Hour).Day()
}

// GetLatestGSDirs gets the appropriate directory names in which data
// would be stored between the given timestamp range.
//
// The returning directories cover the range till the date of startTS, and may
// be precise to the hour.
func GetLatestGSDirs(startTS int64, endTS int64, bsSubdir string) []string {
	startTime := time.Unix(startTS, 0).UTC()
	startYear, startMonth, startDay := startTime.Date()
	glog.Infoln("GS dir start time: ", startTime)
	endTime := time.Unix(endTS, 0).UTC()
	lastAddedTime := startTime
	results := make([]string, 0)
	newYear, newMonth, newDay := endTime.Date()
	newHour := endTime.Hour()
	lastYear, lastMonth, _ := lastAddedTime.Date()
	if lastYear != newYear {
		for i := lastYear; i < newYear; i++ {
			if i != startYear {
				results = append(results, fmt.Sprintf("%04d", i))
			} else {
				for j := startMonth; j <= time.December; j++ {
					if j == startMonth && startDay > 1 {
						for k := startDay; k <= lastDate(i, j); k++ {
							results = append(results, fmt.Sprintf("%04d/%02d/%02d", i, j, k))
						}
					} else {
						results = append(results, fmt.Sprintf("%04d/%02d", i, j))
					}
				}
			}
		}
		lastAddedTime = time.Date(newYear, time.January, 1, 0, 0, 0, 0, time.UTC)
	}
	lastYear, lastMonth, _ = lastAddedTime.Date()
	if lastMonth != newMonth {
		for i := lastMonth; i < newMonth; i++ {
			if i != startMonth {
				results = append(results, fmt.Sprintf("%04d/%02d", lastYear, i))
			} else {
				for j := startDay; j <= lastDate(lastYear, i); j++ {
					results = append(results, fmt.Sprintf("%04d/%02d/%02d", lastYear, i, j))
				}
			}
		}
		lastAddedTime = time.Date(newYear, newMonth, 1, 0, 0, 0, 0, time.UTC)
	}
	lastYear, lastMonth, lastDay := lastAddedTime.Date()
	if lastDay != newDay {
		for i := lastDay; i < newDay; i++ {
			results = append(results, fmt.Sprintf("%04d/%02d/%02d", lastYear, lastMonth, i))
		}
		lastAddedTime = time.Date(newYear, newMonth, newDay, 0, 0, 0, 0, time.UTC)
	}
	lastYear, lastMonth, lastDay = lastAddedTime.Date()
	lastHour := lastAddedTime.Hour()
	for i := lastHour; i < newHour+1; i++ {
		results = append(results, fmt.Sprintf("%04d/%02d/%02d/%02d", lastYear, lastMonth, lastDay, i))
	}
	for i := range results {
		results[i] = fmt.Sprintf("%s/%s", bsSubdir, results[i])
	}
	return results
}

// RequestForStorageURL returns an http.Request for a given Cloud Storage URL.
// This is workaround of a known issue: embedded slashes in URLs require use of
// URL.Opaque property
func RequestForStorageURL(url string) (*http.Request, error) {
	r, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("HTTP new request error: %s", err)
	}
	schemePos := strings.Index(url, ":")
	queryPos := strings.Index(url, "?")
	if queryPos == -1 {
		queryPos = len(url)
	}
	r.URL.Opaque = url[schemePos+1 : queryPos]
	return r, nil
}
<|endoftext|>


<commit_before>package redis

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/go-redis/redis"
	"github.com/mainflux/mainflux/logger"
	"github.com/mainflux/mainflux/lora"
)

const (
	protocol = "lora"

	group  = "mainflux.lora"
	stream = "mainflux.things"

	thingPrefix = "thing."
	thingCreate = thingPrefix + "create"
	thingUpdate = thingPrefix + "update"
	thingRemove = thingPrefix + "remove"

	channelPrefix = "channel."
	channelCreate = channelPrefix + "create"
	channelUpdate = channelPrefix + "update"
	channelRemove = channelPrefix + "remove"
)

var (
	errMetadataType = errors.New("metadatada is not of type lora")

	errMetadataAppID = errors.New("application ID not found in channel metadatada")

	errMetadataDevEUI = errors.New("device EUI not found in thing metadatada")
)

// EventStore represents event source for things and channels provisioning.
type EventStore interface {
	// Subscribes to geven subject and receives events.
	Subscribe(string)
}

type thingLoraMetadata struct {
	Type   string `json:"type"`
	DevEUI string `json:"devEUI"`
}

type channelLoraMetadata struct {
	Type  string `json:"type"`
	AppID string `json:"appID"`
}

type eventStore struct {
	svc      lora.Service
	client   *redis.Client
	consumer string
	logger   logger.Logger
}

// NewEventStore returns new event store instance.
func NewEventStore(svc lora.Service, client *redis.Client, consumer string, log logger.Logger) EventStore {
	return eventStore{
		svc:      svc,
		client:   client,
		consumer: consumer,
		logger:   log,
	}
}

func (es eventStore) Subscribe(subject string) {
	es.client.XGroupCreate(stream, group, "$").Err()
	for {
		streams, err := es.client.XReadGroup(&redis.XReadGroupArgs{
			Group:    group,
			Consumer: es.consumer,
			Streams:  []string{stream, ">"},
			Count:    100,
		}).Result()
		if err != nil || len(streams) == 0 {
			continue
		}

		for _, msg := range streams[0].Messages {
			event := msg.Values

			var err error
			switch event["operation"] {
			case thingCreate:
				cte := decodeCreateThing(event)
				err = es.handleCreateThing(cte)
			case thingUpdate:
				ute := decodeUpdateThing(event)
				err = es.handleUpdateThing(ute)
			case thingRemove:
				rte := decodeRemoveThing(event)
				err = es.handleRemoveThing(rte)
			case channelCreate:
				cce := decodeCreateChannel(event)
				err = es.handleCreateChannel(cce)
			case channelUpdate:
				uce := decodeUpdateChannel(event)
				err = es.handleUpdateChannel(uce)
			case channelRemove:
				rce := decodeRemoveChannel(event)
				err = es.handleRemoveChannel(rce)
			}
			if err != nil {
				es.logger.Warn(fmt.Sprintf("Failed to handle event sourcing: %s", err.Error()))
				break
			}
			es.client.XAck(stream, group, msg.ID)
		}
	}
}

func decodeCreateThing(event map[string]interface{}) createThingEvent {
	return createThingEvent{
		id:       read(event, "id", ""),
		kind:     read(event, "type", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeUpdateThing(event map[string]interface{}) updateThingEvent {
	return updateThingEvent{
		id:       read(event, "id", ""),
		kind:     read(event, "type", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeRemoveThing(event map[string]interface{}) removeThingEvent {
	return removeThingEvent{
		id: read(event, "id", ""),
	}
}

func decodeCreateChannel(event map[string]interface{}) createChannelEvent {
	return createChannelEvent{
		id:       read(event, "id", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeUpdateChannel(event map[string]interface{}) updateChannelEvent {
	return updateChannelEvent{
		id:       read(event, "id", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeRemoveChannel(event map[string]interface{}) removeChannelEvent {
	return removeChannelEvent{
		id: read(event, "id", ""),
	}
}

func (es eventStore) handleCreateThing(cte createThingEvent) error {
	em := thingLoraMetadata{}
	if err := json.Unmarshal([]byte(cte.metadata), &em); err != nil {
		return err
	}

	if em.Type != protocol {
		return errMetadataType
	}
	if em.DevEUI != "" {
		return errMetadataDevEUI
	}

	return es.svc.CreateThing(cte.id, em.DevEUI)
}

func (es eventStore) handleUpdateThing(ute updateThingEvent) error {
	em := thingLoraMetadata{}
	if err := json.Unmarshal([]byte(ute.metadata), &em); err != nil {
		return err
	}

	if em.Type != protocol {
		return errMetadataType
	}
	if em.DevEUI != "" {
		return errMetadataDevEUI
	}

	return es.svc.CreateThing(ute.id, em.DevEUI)
}

func (es eventStore) handleRemoveThing(rte removeThingEvent) error {
	return es.svc.RemoveThing(rte.id)
}

func (es eventStore) handleCreateChannel(cce createChannelEvent) error {
	cm := channelLoraMetadata{}
	if err := json.Unmarshal([]byte(cce.metadata), &cm); err != nil {
		return err
	}

	if cm.Type != protocol {
		return errMetadataType
	}
	if cm.AppID != "" {
		return errMetadataAppID
	}

	return es.svc.CreateChannel(cce.id, cm.AppID)
}

func (es eventStore) handleUpdateChannel(uce updateChannelEvent) error {
	cm := channelLoraMetadata{}
	if err := json.Unmarshal([]byte(uce.metadata), &cm); err != nil {
		return err
	}

	if cm.Type != protocol {
		return errMetadataType
	}
	if cm.AppID != "" {
		return errMetadataAppID
	}

	return es.svc.UpdateChannel(uce.id, cm.AppID)
}

func (es eventStore) handleRemoveChannel(rce removeChannelEvent) error {
	return es.svc.RemoveChannel(rce.id)
}

func read(event map[string]interface{}, key, def string) string {
	val, ok := event[key].(string)
	if !ok {
		return def
	}

	return val
}
<commit_msg>Fix lora-adapter event store handlers (#492)<commit_after>package redis

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/go-redis/redis"
	"github.com/mainflux/mainflux/logger"
	"github.com/mainflux/mainflux/lora"
)

const (
	protocol = "lora"

	group  = "mainflux.lora"
	stream = "mainflux.things"

	thingPrefix = "thing."
	thingCreate = thingPrefix + "create"
	thingUpdate = thingPrefix + "update"
	thingRemove = thingPrefix + "remove"

	channelPrefix = "channel."
	channelCreate = channelPrefix + "create"
	channelUpdate = channelPrefix + "update"
	channelRemove = channelPrefix + "remove"
)

var (
	errMetadataType = errors.New("metadatada is not of type lora")

	errMetadataAppID = errors.New("application ID not found in channel metadatada")

	errMetadataDevEUI = errors.New("device EUI not found in thing metadatada")
)

// EventStore represents event source for things and channels provisioning.
type EventStore interface {
	// Subscribes to geven subject and receives events.
	Subscribe(string)
}

type thingLoraMetadata struct {
	Type   string `json:"type"`
	DevEUI string `json:"devEUI"`
}

type channelLoraMetadata struct {
	Type  string `json:"type"`
	AppID string `json:"appID"`
}

type eventStore struct {
	svc      lora.Service
	client   *redis.Client
	consumer string
	logger   logger.Logger
}

// NewEventStore returns new event store instance.
func NewEventStore(svc lora.Service, client *redis.Client, consumer string, log logger.Logger) EventStore {
	return eventStore{
		svc:      svc,
		client:   client,
		consumer: consumer,
		logger:   log,
	}
}

func (es eventStore) Subscribe(subject string) {
	es.client.XGroupCreate(stream, group, "$").Err()
	for {
		streams, err := es.client.XReadGroup(&redis.XReadGroupArgs{
			Group:    group,
			Consumer: es.consumer,
			Streams:  []string{stream, ">"},
			Count:    100,
		}).Result()
		if err != nil || len(streams) == 0 {
			continue
		}

		for _, msg := range streams[0].Messages {
			event := msg.Values

			var err error
			switch event["operation"] {
			case thingCreate:
				cte := decodeCreateThing(event)
				err = es.handleCreateThing(cte)
			case thingUpdate:
				ute := decodeUpdateThing(event)
				err = es.handleUpdateThing(ute)
			case thingRemove:
				rte := decodeRemoveThing(event)
				err = es.handleRemoveThing(rte)
			case channelCreate:
				cce := decodeCreateChannel(event)
				err = es.handleCreateChannel(cce)
			case channelUpdate:
				uce := decodeUpdateChannel(event)
				err = es.handleUpdateChannel(uce)
			case channelRemove:
				rce := decodeRemoveChannel(event)
				err = es.handleRemoveChannel(rce)
			}
			if err != nil {
				es.logger.Warn(fmt.Sprintf("Failed to handle event sourcing: %s", err.Error()))
				break
			}
			es.client.XAck(stream, group, msg.ID)
		}
	}
}

func decodeCreateThing(event map[string]interface{}) createThingEvent {
	return createThingEvent{
		id:       read(event, "id", ""),
		kind:     read(event, "type", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeUpdateThing(event map[string]interface{}) updateThingEvent {
	return updateThingEvent{
		id:       read(event, "id", ""),
		kind:     read(event, "type", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeRemoveThing(event map[string]interface{}) removeThingEvent {
	return removeThingEvent{
		id: read(event, "id", ""),
	}
}

func decodeCreateChannel(event map[string]interface{}) createChannelEvent {
	return createChannelEvent{
		id:       read(event, "id", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeUpdateChannel(event map[string]interface{}) updateChannelEvent {
	return updateChannelEvent{
		id:       read(event, "id", ""),
		metadata: read(event, "metadata", ""),
	}
}

func decodeRemoveChannel(event map[string]interface{}) removeChannelEvent {
	return removeChannelEvent{
		id: read(event, "id", ""),
	}
}

func (es eventStore) handleCreateThing(cte createThingEvent) error {
	em := thingLoraMetadata{}
	if err := json.Unmarshal([]byte(cte.metadata), &em); err != nil {
		return err
	}

	if em.Type != protocol {
		return errMetadataType
	}
	if em.DevEUI == "" {
		return errMetadataDevEUI
	}

	return es.svc.CreateThing(cte.id, em.DevEUI)
}

func (es eventStore) handleUpdateThing(ute updateThingEvent) error {
	em := thingLoraMetadata{}
	if err := json.Unmarshal([]byte(ute.metadata), &em); err != nil {
		return err
	}

	if em.Type != protocol {
		return errMetadataType
	}
	if em.DevEUI == "" {
		return errMetadataDevEUI
	}

	return es.svc.CreateThing(ute.id, em.DevEUI)
}

func (es eventStore) handleRemoveThing(rte removeThingEvent) error {
	return es.svc.RemoveThing(rte.id)
}

func (es eventStore) handleCreateChannel(cce createChannelEvent) error {
	cm := channelLoraMetadata{}
	if err := json.Unmarshal([]byte(cce.metadata), &cm); err != nil {
		return err
	}

	if cm.Type != protocol {
		return errMetadataType
	}
	if cm.AppID == "" {
		return errMetadataAppID
	}

	return es.svc.CreateChannel(cce.id, cm.AppID)
}

func (es eventStore) handleUpdateChannel(uce updateChannelEvent) error {
	cm := channelLoraMetadata{}
	if err := json.Unmarshal([]byte(uce.metadata), &cm); err != nil {
		return err
	}

	if cm.Type != protocol {
		return errMetadataType
	}
	if cm.AppID == "" {
		return errMetadataAppID
	}

	return es.svc.UpdateChannel(uce.id, cm.AppID)
}

func (es eventStore) handleRemoveChannel(rce removeChannelEvent) error {
	return es.svc.RemoveChannel(rce.id)
}

func read(event map[string]interface{}, key, def string) string {
	val, ok := event[key].(string)
	if !ok {
		return def
	}

	return val
}
<|endoftext|>