<commit_before>package agent

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
)

func TestHTTP_AllocsList(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Directly manipulate the state
		state := s.Agent.server.State()
		alloc1 := mock.Alloc()
		alloc2 := mock.Alloc()
		state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID))
		state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))
		err := state.UpsertAllocs(1000,
			[]*structs.Allocation{alloc1, alloc2})
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/allocations", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		obj, err := s.Server.AllocsRequest(respW, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Check for the index
		if respW.HeaderMap.Get("X-Nomad-Index") == "" {
			t.Fatalf("missing index")
		}
		if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
			t.Fatalf("missing known leader")
		}
		if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
			t.Fatalf("missing last contact")
		}

		// Check the alloc
		n := obj.([]*structs.AllocListStub)
		if len(n) != 2 {
			t.Fatalf("bad: %#v", n)
		}
	})
}

func TestHTTP_AllocsPrefixList(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Directly manipulate the state
		state := s.Agent.server.State()

		alloc1 := mock.Alloc()
		alloc1.ID = "aaaaaaaa-e8f7-fd38-c855-ab94ceb89706"
		alloc2 := mock.Alloc()
		alloc2.ID = "aaabbbbb-e8f7-fd38-c855-ab94ceb89706"
		summary1 := mock.JobSummary(alloc1.JobID)
		summary2 := mock.JobSummary(alloc2.JobID)
		if err := state.UpsertJobSummary(998, summary1); err != nil {
			t.Fatal(err)
		}
		if err := state.UpsertJobSummary(999, summary2); err != nil {
			t.Fatal(err)
		}
		if err := state.UpsertAllocs(1000,
			[]*structs.Allocation{alloc1, alloc2}); err != nil {
			t.Fatalf("err: %v", err)
		}

		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/allocations?prefix=aaab", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		obj, err := s.Server.AllocsRequest(respW, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Check for the index
		if respW.HeaderMap.Get("X-Nomad-Index") == "" {
			t.Fatalf("missing index")
		}
		if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
			t.Fatalf("missing known leader")
		}
		if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
			t.Fatalf("missing last contact")
		}

		// Check the alloc
		n := obj.([]*structs.AllocListStub)
		if len(n) != 1 {
			t.Fatalf("bad: %#v", n)
		}

		// Check the identifier
		if n[0].ID != alloc2.ID {
			t.Fatalf("expected alloc ID: %v, Actual: %v", alloc2.ID, n[0].ID)
		}
	})
}

func TestHTTP_AllocQuery(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Directly manipulate the state
		state := s.Agent.server.State()
		alloc := mock.Alloc()
		if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil {
			t.Fatal(err)
		}
		err := state.UpsertAllocs(1000,
			[]*structs.Allocation{alloc})
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		obj, err := s.Server.AllocSpecificRequest(respW, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Check for the index
		if respW.HeaderMap.Get("X-Nomad-Index") == "" {
			t.Fatalf("missing index")
		}
		if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
			t.Fatalf("missing known leader")
		}
		if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
			t.Fatalf("missing last contact")
		}

		// Check the job
		a := obj.(*structs.Allocation)
		if a.ID != alloc.ID {
			t.Fatalf("bad: %#v", a)
		}
	})
}

func TestHTTP_AllocStats(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/client/allocation/123/foo", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		_, err = s.Server.ClientAllocRequest(respW, req)
		if !strings.Contains(err.Error(), resourceNotFoundErr) {
			t.Fatalf("err: %v", err)
		}
	})
}

func TestHTTP_AllocSnapshot(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		_, err = s.Server.ClientAllocRequest(respW, req)
		if !strings.Contains(err.Error(), allocNotFoundErr) {
			t.Fatalf("err: %v", err)
		}
	})
}
<commit_msg>Added two tests<commit_after>package agent

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
)

func TestHTTP_AllocsList(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Directly manipulate the state
		state := s.Agent.server.State()
		alloc1 := mock.Alloc()
		alloc2 := mock.Alloc()
		state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID))
		state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))
		err := state.UpsertAllocs(1000,
			[]*structs.Allocation{alloc1, alloc2})
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/allocations", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		obj, err := s.Server.AllocsRequest(respW, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Check for the index
		if respW.HeaderMap.Get("X-Nomad-Index") == "" {
			t.Fatalf("missing index")
		}
		if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
			t.Fatalf("missing known leader")
		}
		if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
			t.Fatalf("missing last contact")
		}

		// Check the alloc
		n := obj.([]*structs.AllocListStub)
		if len(n) != 2 {
			t.Fatalf("bad: %#v", n)
		}
	})
}

func TestHTTP_AllocsPrefixList(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Directly manipulate the state
		state := s.Agent.server.State()

		alloc1 := mock.Alloc()
		alloc1.ID = "aaaaaaaa-e8f7-fd38-c855-ab94ceb89706"
		alloc2 := mock.Alloc()
		alloc2.ID = "aaabbbbb-e8f7-fd38-c855-ab94ceb89706"
		summary1 := mock.JobSummary(alloc1.JobID)
		summary2 := mock.JobSummary(alloc2.JobID)
		if err := state.UpsertJobSummary(998, summary1); err != nil {
			t.Fatal(err)
		}
		if err := state.UpsertJobSummary(999, summary2); err != nil {
			t.Fatal(err)
		}
		if err := state.UpsertAllocs(1000,
			[]*structs.Allocation{alloc1, alloc2}); err != nil {
			t.Fatalf("err: %v", err)
		}

		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/allocations?prefix=aaab", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		obj, err := s.Server.AllocsRequest(respW, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Check for the index
		if respW.HeaderMap.Get("X-Nomad-Index") == "" {
			t.Fatalf("missing index")
		}
		if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
			t.Fatalf("missing known leader")
		}
		if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
			t.Fatalf("missing last contact")
		}

		// Check the alloc
		n := obj.([]*structs.AllocListStub)
		if len(n) != 1 {
			t.Fatalf("bad: %#v", n)
		}

		// Check the identifier
		if n[0].ID != alloc2.ID {
			t.Fatalf("expected alloc ID: %v, Actual: %v", alloc2.ID, n[0].ID)
		}
	})
}

func TestHTTP_AllocQuery(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Directly manipulate the state
		state := s.Agent.server.State()
		alloc := mock.Alloc()
		if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil {
			t.Fatal(err)
		}
		err := state.UpsertAllocs(1000,
			[]*structs.Allocation{alloc})
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		obj, err := s.Server.AllocSpecificRequest(respW, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		// Check for the index
		if respW.HeaderMap.Get("X-Nomad-Index") == "" {
			t.Fatalf("missing index")
		}
		if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
			t.Fatalf("missing known leader")
		}
		if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
			t.Fatalf("missing last contact")
		}

		// Check the job
		a := obj.(*structs.Allocation)
		if a.ID != alloc.ID {
			t.Fatalf("bad: %#v", a)
		}
	})
}

func TestHTTP_AllocStats(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/client/allocation/123/foo", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		_, err = s.Server.ClientAllocRequest(respW, req)
		if !strings.Contains(err.Error(), resourceNotFoundErr) {
			t.Fatalf("err: %v", err)
		}
	})
}

func TestHTTP_AllocSnapshot(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		_, err = s.Server.ClientAllocRequest(respW, req)
		if !strings.Contains(err.Error(), allocNotFoundErr) {
			t.Fatalf("err: %v", err)
		}
	})
}

func TestHTTP_AllocGC(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/client/allocation/123/gc", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		_, err = s.Server.ClientAllocRequest(respW, req)
		if !strings.Contains(err.Error(), "unable to collect allocation") {
			t.Fatalf("err: %v", err)
		}
	})
}

func TestHTTP_AllocAllGC(t *testing.T) {
	httpTest(t, nil, func(s *TestServer) {
		// Make the HTTP request
		req, err := http.NewRequest("GET", "/v1/client/gc", nil)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		respW := httptest.NewRecorder()

		// Make the request
		_, err = s.Server.ClientGCRequest(respW, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
	})

}
<|endoftext|>


<commit_before>// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package registry

import (
	"encoding/json"
	"regexp"
	"strings"
	"time"

	"github.com/goharbor/harbor/src/common/dao"
	clairdao "github.com/goharbor/harbor/src/common/dao/clair"
	"github.com/goharbor/harbor/src/common/models"
	"github.com/goharbor/harbor/src/common/utils"
	"github.com/goharbor/harbor/src/common/utils/log"
	"github.com/goharbor/harbor/src/core/api"
	"github.com/goharbor/harbor/src/core/config"
	"github.com/goharbor/harbor/src/core/notifier"
	coreutils "github.com/goharbor/harbor/src/core/utils"
	rep_notification "github.com/goharbor/harbor/src/replication/event/notification"
	"github.com/goharbor/harbor/src/replication/event/topic"
)

// NotificationHandler handles request on /service/notifications/, which listens to registry's events.
type NotificationHandler struct {
	api.BaseController
}

const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+(json|prettyjws)`
const vicPrefix = "vic/"

// Post handles POST request, and records audit log or refreshes cache based on event.
func (n *NotificationHandler) Post() {
	var notification models.Notification
	err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), &notification)

	if err != nil {
		log.Errorf("failed to decode notification: %v", err)
		return
	}

	events, err := filterEvents(&notification)
	if err != nil {
		log.Errorf("failed to filter events: %v", err)
		return
	}

	for _, event := range events {
		repository := event.Target.Repository
		project, _ := utils.ParseRepository(repository)
		tag := event.Target.Tag
		action := event.Action

		user := event.Actor.Name
		if len(user) == 0 {
			user = "anonymous"
		}

		pro, err := config.GlobalProjectMgr.Get(project)
		if err != nil {
			log.Errorf("failed to get project by name %s: %v", project, err)
			return
		}
		if pro == nil {
			log.Warningf("project %s not found", project)
			continue
		}

		go func() {
			if err := dao.AddAccessLog(models.AccessLog{
				Username:  user,
				ProjectID: pro.ProjectID,
				RepoName:  repository,
				RepoTag:   tag,
				Operation: action,
				OpTime:    time.Now(),
			}); err != nil {
				log.Errorf("failed to add access log: %v", err)
			}
		}()

		if action == "push" {
			go func() {
				exist := dao.RepositoryExists(repository)
				if exist {
					return
				}
				log.Debugf("Add repository %s into DB.", repository)
				repoRecord := models.RepoRecord{
					Name:      repository,
					ProjectID: pro.ProjectID,
				}
				if err := dao.AddRepository(repoRecord); err != nil {
					log.Errorf("Error happens when adding repository: %v", err)
				}
			}()

			go func() {
				image := repository + ":" + tag
				err := notifier.Publish(topic.ReplicationEventTopicOnPush, rep_notification.OnPushNotification{
					Image: image,
				})
				if err != nil {
					log.Errorf("failed to publish on push topic for resource %s: %v", image, err)
					return
				}
				log.Debugf("the on push topic for resource %s published", image)
			}()

			if autoScanEnabled(pro) {
				last, err := clairdao.GetLastUpdate()
				if err != nil {
					log.Errorf("Failed to get last update from Clair DB, error: %v, the auto scan will be skipped.", err)
				} else if last == 0 {
					log.Infof("The Vulnerability data is not ready in Clair DB, the auto scan will be skipped.", err)
				} else if err := coreutils.TriggerImageScan(repository, tag); err != nil {
					log.Warningf("Failed to scan image, repository: %s, tag: %s, error: %v", repository, tag, err)
				}
			}
		}
		if action == "pull" {
			go func() {
				log.Debugf("Increase the repository %s pull count.", repository)
				if err := dao.IncreasePullCount(repository); err != nil {
					log.Errorf("Error happens when increasing pull count: %v", repository)
				}
			}()
		}
	}
}

func filterEvents(notification *models.Notification) ([]*models.Event, error) {
	events := []*models.Event{}

	for _, event := range notification.Events {
		log.Debugf("receive an event: \n----ID: %s \n----target: %s:%s \n----digest: %s \n----action: %s \n----mediatype: %s \n----user-agent: %s", event.ID, event.Target.Repository,
			event.Target.Tag, event.Target.Digest, event.Action, event.Target.MediaType, event.Request.UserAgent)

		isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType)
		if err != nil {
			log.Errorf("failed to match the media type against pattern: %v", err)
			continue
		}

		if !isManifest {
			continue
		}

		// pull and push manifest by docker-client or vic
		if (strings.HasPrefix(event.Request.UserAgent, "docker") || strings.HasPrefix(event.Request.UserAgent, vicPrefix) ||
	                strings.HasPrefix(event.Request.UserAgent, "jib")) && (event.Action == "pull" || event.Action == "push") {
			events = append(events, &event)
			log.Debugf("add event to collect: %s", event.ID)
			continue
		}

		// push manifest by docker-client or job-service
		if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" {
			events = append(events, &event)
			log.Debugf("add event to collect: %s", event.ID)
			continue
		}
	}

	return events, nil
}

func autoScanEnabled(project *models.Project) bool {
	if !config.WithClair() {
		log.Debugf("Auto Scan disabled because Harbor is not deployed with Clair")
		return false
	}

	return project.AutoScan()
}

// Render returns nil as it won't render any template.
func (n *NotificationHandler) Render() error {
	return nil
}
<commit_msg>add support jib<commit_after>// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package registry

import (
	"encoding/json"
	"regexp"
	"strings"
	"time"

	"github.com/goharbor/harbor/src/common/dao"
	clairdao "github.com/goharbor/harbor/src/common/dao/clair"
	"github.com/goharbor/harbor/src/common/models"
	"github.com/goharbor/harbor/src/common/utils"
	"github.com/goharbor/harbor/src/common/utils/log"
	"github.com/goharbor/harbor/src/core/api"
	"github.com/goharbor/harbor/src/core/config"
	"github.com/goharbor/harbor/src/core/notifier"
	coreutils "github.com/goharbor/harbor/src/core/utils"
	rep_notification "github.com/goharbor/harbor/src/replication/event/notification"
	"github.com/goharbor/harbor/src/replication/event/topic"
)

// NotificationHandler handles request on /service/notifications/, which listens to registry's events.
type NotificationHandler struct {
	api.BaseController
}

const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+(json|prettyjws)`
const vicPrefix = "vic/"

// Post handles POST request, and records audit log or refreshes cache based on event.
func (n *NotificationHandler) Post() {
	var notification models.Notification
	err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), &notification)

	if err != nil {
		log.Errorf("failed to decode notification: %v", err)
		return
	}

	events, err := filterEvents(&notification)
	if err != nil {
		log.Errorf("failed to filter events: %v", err)
		return
	}

	for _, event := range events {
		repository := event.Target.Repository
		project, _ := utils.ParseRepository(repository)
		tag := event.Target.Tag
		action := event.Action

		user := event.Actor.Name
		if len(user) == 0 {
			user = "anonymous"
		}

		pro, err := config.GlobalProjectMgr.Get(project)
		if err != nil {
			log.Errorf("failed to get project by name %s: %v", project, err)
			return
		}
		if pro == nil {
			log.Warningf("project %s not found", project)
			continue
		}

		go func() {
			if err := dao.AddAccessLog(models.AccessLog{
				Username:  user,
				ProjectID: pro.ProjectID,
				RepoName:  repository,
				RepoTag:   tag,
				Operation: action,
				OpTime:    time.Now(),
			}); err != nil {
				log.Errorf("failed to add access log: %v", err)
			}
		}()

		if action == "push" {
			go func() {
				exist := dao.RepositoryExists(repository)
				if exist {
					return
				}
				log.Debugf("Add repository %s into DB.", repository)
				repoRecord := models.RepoRecord{
					Name:      repository,
					ProjectID: pro.ProjectID,
				}
				if err := dao.AddRepository(repoRecord); err != nil {
					log.Errorf("Error happens when adding repository: %v", err)
				}
			}()

			go func() {
				image := repository + ":" + tag
				err := notifier.Publish(topic.ReplicationEventTopicOnPush, rep_notification.OnPushNotification{
					Image: image,
				})
				if err != nil {
					log.Errorf("failed to publish on push topic for resource %s: %v", image, err)
					return
				}
				log.Debugf("the on push topic for resource %s published", image)
			}()

			if autoScanEnabled(pro) {
				last, err := clairdao.GetLastUpdate()
				if err != nil {
					log.Errorf("Failed to get last update from Clair DB, error: %v, the auto scan will be skipped.", err)
				} else if last == 0 {
					log.Infof("The Vulnerability data is not ready in Clair DB, the auto scan will be skipped.", err)
				} else if err := coreutils.TriggerImageScan(repository, tag); err != nil {
					log.Warningf("Failed to scan image, repository: %s, tag: %s, error: %v", repository, tag, err)
				}
			}
		}
		if action == "pull" {
			go func() {
				log.Debugf("Increase the repository %s pull count.", repository)
				if err := dao.IncreasePullCount(repository); err != nil {
					log.Errorf("Error happens when increasing pull count: %v", repository)
				}
			}()
		}
	}
}

func filterEvents(notification *models.Notification) ([]*models.Event, error) {
	events := []*models.Event{}

	for _, event := range notification.Events {
		log.Debugf("receive an event: \n----ID: %s \n----target: %s:%s \n----digest: %s \n----action: %s \n----mediatype: %s \n----user-agent: %s", event.ID, event.Target.Repository,
			event.Target.Tag, event.Target.Digest, event.Action, event.Target.MediaType, event.Request.UserAgent)

		isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType)
		if err != nil {
			log.Errorf("failed to match the media type against pattern: %v", err)
			continue
		}

		if !isManifest {
			continue
		}

		if checkEvent(&event) {
			events = append(events, &event)
			log.Debugf("add event to collection: %s", event.ID)
			continue
		}
	}

	return events, nil
}

func checkEvent(event *models.Event) bool {
	// pull and push manifest by docker-client or vic or jib
	if (strings.HasPrefix(event.Request.UserAgent, "docker") || strings.HasPrefix(event.Request.UserAgent, vicPrefix) || strings.HasPrefix(event.Request.UserAgent, "jib")) &&
		(event.Action == "pull" || event.Action == "push") {
		return true
	}
	// push manifest by docker-client or job-service
	if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" {
		return true
	}
	return false
}

func autoScanEnabled(project *models.Project) bool {
	if !config.WithClair() {
		log.Debugf("Auto Scan disabled because Harbor is not deployed with Clair")
		return false
	}

	return project.AutoScan()
}

// Render returns nil as it won't render any template.
func (n *NotificationHandler) Render() error {
	return nil
}
<|endoftext|>


<commit_before>// +build integration,!no-etcd

/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"bufio"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	prometheuspb "github.com/prometheus/client_model/go"
)

const scrapeRequestHeader = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text"

func init() {
	requireEtcd()
}

func scrapeMetrics(s *httptest.Server) ([]*prometheuspb.MetricFamily, error) {
	req, err := http.NewRequest("GET", s.URL+"/metrics", nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to create http request: %v", err)
	}
	// Ask the prometheus exporter for its text protocol buffer format, since it's
	// much easier to parse than its plain-text format. Don't use the serialized
	// proto representation since it uses a non-standard varint delimiter between
	// metric families.
	req.Header.Add("Accept", scrapeRequestHeader)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Unable to contact metrics endpoint of master: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Non-200 response trying to scrape metrics from master: %v", resp)
	}

	// Each line in the response body should contain all the data for a single metric.
	var metrics []*prometheuspb.MetricFamily
	scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
		var metric prometheuspb.MetricFamily
		if err := proto.UnmarshalText(scanner.Text(), &metric); err != nil {
			return nil, fmt.Errorf("Failed to unmarshal line of metrics response: %v", err)
		}
		glog.Infof("Got metric %q", metric.GetName())
		metrics = append(metrics, &metric)
	}
	return metrics, nil
}

func checkForExpectedMetrics(t *testing.T, metrics []*prometheuspb.MetricFamily, expectedMetrics []string) {
	foundMetrics := make(map[string]bool)
	for _, metric := range metrics {
		foundMetrics[metric.GetName()] = true
	}
	for _, expected := range expectedMetrics {
		if _, found := foundMetrics[expected]; !found {
			t.Errorf("Master metrics did not include expected metric %q", expected)
		}
	}
}

func TestMasterProcessMetrics(t *testing.T) {
	_, s := runAMaster(t)
	defer s.Close()

	metrics, err := scrapeMetrics(s)
	if err != nil {
		t.Fatal(err)
	}
	checkForExpectedMetrics(t, metrics, []string{
		"process_start_time_seconds",
		"process_cpu_seconds_total",
		"process_goroutines",
		"process_open_fds",
		"process_resident_memory_bytes",
	})
}

func TestApiserverMetrics(t *testing.T) {
	_, s := runAMaster(t)
	defer s.Close()

	// Make a request to the apiserver to ensure there's at least one data point
	// for the metrics we're expecting -- otherwise, they won't be exported.
	client := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Version()})
	if _, err := client.Pods(api.NamespaceDefault).List(labels.Everything(), fields.Everything()); err != nil {
		t.Fatalf("unexpected error getting pods: %v", err)
	}

	metrics, err := scrapeMetrics(s)
	if err != nil {
		t.Fatal(err)
	}
	checkForExpectedMetrics(t, metrics, []string{
		"apiserver_request_count",
		"apiserver_request_latencies",
	})
}
<commit_msg>Metrics tests can't run on Mac<commit_after>// +build integration,!no-etcd,linux

/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"bufio"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	prometheuspb "github.com/prometheus/client_model/go"
)

const scrapeRequestHeader = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text"

func init() {
	requireEtcd()
}

func scrapeMetrics(s *httptest.Server) ([]*prometheuspb.MetricFamily, error) {
	req, err := http.NewRequest("GET", s.URL+"/metrics", nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to create http request: %v", err)
	}
	// Ask the prometheus exporter for its text protocol buffer format, since it's
	// much easier to parse than its plain-text format. Don't use the serialized
	// proto representation since it uses a non-standard varint delimiter between
	// metric families.
	req.Header.Add("Accept", scrapeRequestHeader)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Unable to contact metrics endpoint of master: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Non-200 response trying to scrape metrics from master: %v", resp)
	}

	// Each line in the response body should contain all the data for a single metric.
	var metrics []*prometheuspb.MetricFamily
	scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
		var metric prometheuspb.MetricFamily
		if err := proto.UnmarshalText(scanner.Text(), &metric); err != nil {
			return nil, fmt.Errorf("Failed to unmarshal line of metrics response: %v", err)
		}
		glog.Infof("Got metric %q", metric.GetName())
		metrics = append(metrics, &metric)
	}
	return metrics, nil
}

func checkForExpectedMetrics(t *testing.T, metrics []*prometheuspb.MetricFamily, expectedMetrics []string) {
	foundMetrics := make(map[string]bool)
	for _, metric := range metrics {
		foundMetrics[metric.GetName()] = true
	}
	for _, expected := range expectedMetrics {
		if _, found := foundMetrics[expected]; !found {
			t.Errorf("Master metrics did not include expected metric %q", expected)
		}
	}
}

func TestMasterProcessMetrics(t *testing.T) {
	_, s := runAMaster(t)
	defer s.Close()

	metrics, err := scrapeMetrics(s)
	if err != nil {
		t.Fatal(err)
	}
	checkForExpectedMetrics(t, metrics, []string{
		"process_start_time_seconds",
		"process_cpu_seconds_total",
		"process_goroutines",
		"process_open_fds",
		"process_resident_memory_bytes",
	})
}

func TestApiserverMetrics(t *testing.T) {
	_, s := runAMaster(t)
	defer s.Close()

	// Make a request to the apiserver to ensure there's at least one data point
	// for the metrics we're expecting -- otherwise, they won't be exported.
	client := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Version()})
	if _, err := client.Pods(api.NamespaceDefault).List(labels.Everything(), fields.Everything()); err != nil {
		t.Fatalf("unexpected error getting pods: %v", err)
	}

	metrics, err := scrapeMetrics(s)
	if err != nil {
		t.Fatal(err)
	}
	checkForExpectedMetrics(t, metrics, []string{
		"apiserver_request_count",
		"apiserver_request_latencies",
	})
}
<|endoftext|>


<commit_before>package workflow

import (
	"bytes"
	"context"
	"encoding/base64"
	"io"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflowtemplate"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/exportentities"
)

// Export a workflow
func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	ctx, end := observability.Span(ctx, "workflow.Export")
	defer end()

	wf, errload := Load(ctx, db, cache, proj, name, u, LoadOptions{})
	if errload != nil {
		return 0, sdk.WrapError(errload, "workflow.Export> Cannot load workflow %s", name)
	}

	// If repo is from as-code do not export WorkflowSkipIfOnlyOneRepoWebhook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}

	return exportWorkflow(*wf, f, w, opts...)
}

func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	e, err := exportentities.NewWorkflow(wf, opts...)
	if err != nil {
		return 0, sdk.WrapError(err, "exportWorkflow")
	}

	// Useful to not display history_length in yaml or json if it's his default value
	if e.HistoryLength != nil && *e.HistoryLength == sdk.DefaultHistoryLength {
		e.HistoryLength = nil
	}

	// Marshal to the desired format
	b, err := exportentities.Marshal(e, f)
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	return w.Write(b)
}

// Pull a workflow with all it dependencies; it writes a tar buffer in the writer
func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format,
	encryptFunc sdk.EncryptFunc, u *sdk.User, opts ...exportentities.WorkflowOptions) (exportentities.WorkflowPulled, error) {
	ctx, end := observability.Span(ctx, "workflow.Pull")
	defer end()

	var wp exportentities.WorkflowPulled

	options := LoadOptions{
		DeepPipeline: true,
	}
	wf, errload := Load(ctx, db, cache, proj, name, u, options)
	if errload != nil {
		return wp, sdk.WrapError(errload, "cannot load workflow %s", name)
	}

	i, err := workflowtemplate.GetInstanceByWorkflowID(db, wf.ID)
	if err != nil {
		return wp, err
	}
	if i != nil {
		wf.Template, err = workflowtemplate.GetByID(db, i.WorkflowTemplateID)
		if err != nil {
			return wp, err
		}
		if err := workflowtemplate.AggregateOnWorkflowTemplate(db, wf.Template); err != nil {
			return wp, err
		}
	}

	apps := wf.GetApplications()
	envs := wf.GetEnvironments()
	pips := wf.GetPipelines()

	//Reload app to retrieve secrets
	for i := range apps {
		app := &apps[i]
		vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name)
		}
		app.Variable = vars

		if err := application.LoadAllDecryptedKeys(db, app); err != nil {
			return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name)
		}
	}

	//Reload env to retrieve secrets
	for i := range envs {
		env := &envs[i]
		vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name)
		}
		env.Variable = vars

		if err := environment.LoadAllDecryptedKeys(db, env); err != nil {
			return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name)
		}
	}

	buffw := new(bytes.Buffer)
	if _, err := exportWorkflow(*wf, f, buffw, opts...); err != nil {
		return wp, sdk.WrapError(err, "unable to export workflow")
	}
	wp.Workflow.Name = wf.Name
	wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes())

	wp.Applications = make([]exportentities.WorkflowPulledItem, len(apps))
	for i, a := range apps {
		buff := new(bytes.Buffer)
		if _, err := application.ExportApplication(db, a, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export app %s", a.Name)
		}
		wp.Applications[i].Name = a.Name
		wp.Applications[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Environments = make([]exportentities.WorkflowPulledItem, len(envs))
	for i, e := range envs {
		buff := new(bytes.Buffer)
		if _, err := environment.ExportEnvironment(db, e, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export env %s", e.Name)
		}
		wp.Environments[i].Name = e.Name
		wp.Environments[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Pipelines = make([]exportentities.WorkflowPulledItem, len(pips))
	for i, p := range pips {
		buff := new(bytes.Buffer)
		if _, err := pipeline.ExportPipeline(p, f, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export pipeline %s", p.Name)
		}
		wp.Pipelines[i].Name = p.Name
		wp.Pipelines[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	return wp, nil
}
<commit_msg>fix(api): do not export hook in workflow as code (#4052)<commit_after>package workflow

import (
	"bytes"
	"context"
	"encoding/base64"
	"io"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflowtemplate"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/exportentities"
)

// Export a workflow
func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	ctx, end := observability.Span(ctx, "workflow.Export")
	defer end()

	wf, errload := Load(ctx, db, cache, proj, name, u, LoadOptions{})
	if errload != nil {
		return 0, sdk.WrapError(errload, "workflow.Export> Cannot load workflow %s", name)
	}

	// If repo is from as-code do not export WorkflowSkipIfOnlyOneRepoWebhook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}

	return exportWorkflow(*wf, f, w, opts...)
}

func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	e, err := exportentities.NewWorkflow(wf, opts...)
	if err != nil {
		return 0, sdk.WrapError(err, "exportWorkflow")
	}

	// Useful to not display history_length in yaml or json if it's his default value
	if e.HistoryLength != nil && *e.HistoryLength == sdk.DefaultHistoryLength {
		e.HistoryLength = nil
	}

	// Marshal to the desired format
	b, err := exportentities.Marshal(e, f)
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	return w.Write(b)
}

// Pull a workflow with all it dependencies; it writes a tar buffer in the writer
func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format,
	encryptFunc sdk.EncryptFunc, u *sdk.User, opts ...exportentities.WorkflowOptions) (exportentities.WorkflowPulled, error) {
	ctx, end := observability.Span(ctx, "workflow.Pull")
	defer end()

	var wp exportentities.WorkflowPulled

	options := LoadOptions{
		DeepPipeline: true,
	}
	wf, errload := Load(ctx, db, cache, proj, name, u, options)
	if errload != nil {
		return wp, sdk.WrapError(errload, "cannot load workflow %s", name)
	}

	i, err := workflowtemplate.GetInstanceByWorkflowID(db, wf.ID)
	if err != nil {
		return wp, err
	}
	if i != nil {
		wf.Template, err = workflowtemplate.GetByID(db, i.WorkflowTemplateID)
		if err != nil {
			return wp, err
		}
		if err := workflowtemplate.AggregateOnWorkflowTemplate(db, wf.Template); err != nil {
			return wp, err
		}
	}

	apps := wf.GetApplications()
	envs := wf.GetEnvironments()
	pips := wf.GetPipelines()

	//Reload app to retrieve secrets
	for i := range apps {
		app := &apps[i]
		vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name)
		}
		app.Variable = vars

		if err := application.LoadAllDecryptedKeys(db, app); err != nil {
			return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name)
		}
	}

	//Reload env to retrieve secrets
	for i := range envs {
		env := &envs[i]
		vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name)
		}
		env.Variable = vars

		if err := environment.LoadAllDecryptedKeys(db, env); err != nil {
			return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name)
		}
	}

	buffw := new(bytes.Buffer)
	// If the repository is "as-code", hide the hook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}
	if _, err := exportWorkflow(*wf, f, buffw, opts...); err != nil {
		return wp, sdk.WrapError(err, "unable to export workflow")
	}
	wp.Workflow.Name = wf.Name
	wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes())

	wp.Applications = make([]exportentities.WorkflowPulledItem, len(apps))
	for i, a := range apps {
		buff := new(bytes.Buffer)
		if _, err := application.ExportApplication(db, a, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export app %s", a.Name)
		}
		wp.Applications[i].Name = a.Name
		wp.Applications[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Environments = make([]exportentities.WorkflowPulledItem, len(envs))
	for i, e := range envs {
		buff := new(bytes.Buffer)
		if _, err := environment.ExportEnvironment(db, e, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export env %s", e.Name)
		}
		wp.Environments[i].Name = e.Name
		wp.Environments[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Pipelines = make([]exportentities.WorkflowPulledItem, len(pips))
	for i, p := range pips {
		buff := new(bytes.Buffer)
		if _, err := pipeline.ExportPipeline(p, f, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export pipeline %s", p.Name)
		}
		wp.Pipelines[i].Name = p.Name
		wp.Pipelines[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	return wp, nil
}
<|endoftext|>


<commit_before>package persister

import (
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/Sirupsen/logrus"
	whisper "github.com/lomik/go-whisper"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
)

const storeMutexCount = 32768

type StoreFunc func(p *Whisper, values *points.Points)

// Whisper write data to *.wsp files
type Whisper struct {
	helper.Stoppable
	updateOperations    uint32
	committedPoints     uint32
	recv                func(chan bool) *points.Points
	schemas             WhisperSchemas
	aggregation         *WhisperAggregation
	workersCount        int
	rootPath            string
	created             uint32 // counter
	sparse              bool
	maxUpdatesPerSecond int
	throttleTicker      *ThrottleTicker
	storeMutex          [storeMutexCount]sync.Mutex
	mockStore           func() (StoreFunc, func())
}

// NewWhisper create instance of Whisper
func NewWhisper(rootPath string, schemas WhisperSchemas, aggregation *WhisperAggregation, recv func(chan bool) *points.Points) *Whisper {
	return &Whisper{
		recv:                recv,
		schemas:             schemas,
		aggregation:         aggregation,
		workersCount:        1,
		rootPath:            rootPath,
		maxUpdatesPerSecond: 0,
	}
}

// SetMaxUpdatesPerSecond enable throttling
func (p *Whisper) SetMaxUpdatesPerSecond(maxUpdatesPerSecond int) {
	p.maxUpdatesPerSecond = maxUpdatesPerSecond
}

// GetMaxUpdatesPerSecond returns current throttling speed
func (p *Whisper) GetMaxUpdatesPerSecond() int {
	return p.maxUpdatesPerSecond
}

// SetWorkers count
func (p *Whisper) SetWorkers(count int) {
	p.workersCount = count
}

// SetSparse creation
func (p *Whisper) SetSparse(sparse bool) {
	p.sparse = sparse
}

func (p *Whisper) SetMockStore(fn func() (StoreFunc, func())) {
	p.mockStore = fn
}

func fnv32(key string) uint32 {
	hash := uint32(2166136261)
	const prime32 = uint32(16777619)
	for i := 0; i < len(key); i++ {
		hash *= prime32
		hash ^= uint32(key[i])
	}
	return hash
}

func store(p *Whisper, values *points.Points) {
	// avoid concurrent store same metric
	// @TODO: may be flock?
	mutexIndex := fnv32(values.Metric) % storeMutexCount
	p.storeMutex[mutexIndex].Lock()
	defer p.storeMutex[mutexIndex].Unlock()

	path := filepath.Join(p.rootPath, strings.Replace(values.Metric, ".", "/", -1)+".wsp")

	w, err := whisper.Open(path)
	if err != nil {
		// create new whisper if file not exists
		if !os.IsNotExist(err) {
			logrus.Errorf("[persister] Failed to open whisper file %s: %s", path, err.Error())
			return
		}

		schema, ok := p.schemas.Match(values.Metric)
		if !ok {
			logrus.Errorf("[persister] No storage schema defined for %s", values.Metric)
			return
		}

		aggr := p.aggregation.match(values.Metric)
		if aggr == nil {
			logrus.Errorf("[persister] No storage aggregation defined for %s", values.Metric)
			return
		}

		logrus.WithFields(logrus.Fields{
			"retention":    schema.RetentionStr,
			"schema":       schema.Name,
			"aggregation":  aggr.name,
			"xFilesFactor": aggr.xFilesFactor,
			"method":       aggr.aggregationMethodStr,
		}).Debugf("[persister] Creating %s", path)

		if err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
			logrus.Error(err)
			return
		}

		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.aggregationMethod, float32(aggr.xFilesFactor), &whisper.Options{
			Sparse: p.sparse,
		})
		if err != nil {
			logrus.Errorf("[persister] Failed to create new whisper file %s: %s", path, err.Error())
			return
		}

		atomic.AddUint32(&p.created, 1)
	}

	points := make([]*whisper.TimeSeriesPoint, len(values.Data))
	for i, r := range values.Data {
		points[i] = &whisper.TimeSeriesPoint{Time: int(r.Timestamp), Value: r.Value}
	}

	atomic.AddUint32(&p.committedPoints, uint32(len(values.Data)))
	atomic.AddUint32(&p.updateOperations, 1)

	defer w.Close()

	defer func() {
		if r := recover(); r != nil {
			logrus.Errorf("[persister] UpdateMany %s recovered: %s", path, r)
		}
	}()
	w.UpdateMany(points)
}

func (p *Whisper) worker(recv func(chan bool) *points.Points, exit chan bool) {
	storeFunc := store
	var doneCb func()
	if p.mockStore != nil {
		storeFunc, doneCb = p.mockStore()
	}

LOOP:
	for {
		select {
		case <-p.throttleTicker.C:
			// pass
		case <-exit:
			return
		}

		points := recv(exit)
		if points == nil {
			// exit closed
			break LOOP
		}
		storeFunc(p, points)
		if doneCb != nil {
			doneCb()
		}
	}
}

// Stat callback
func (p *Whisper) Stat(send helper.StatCallback) {
	updateOperations := atomic.LoadUint32(&p.updateOperations)
	committedPoints := atomic.LoadUint32(&p.committedPoints)
	atomic.AddUint32(&p.updateOperations, -updateOperations)
	atomic.AddUint32(&p.committedPoints, -committedPoints)

	created := atomic.LoadUint32(&p.created)
	atomic.AddUint32(&p.created, -created)

	send("updateOperations", float64(updateOperations))
	send("committedPoints", float64(committedPoints))
	if updateOperations > 0 {
		send("pointsPerUpdate", float64(committedPoints)/float64(updateOperations))
	} else {
		send("pointsPerUpdate", 0.0)
	}

	send("created", float64(created))

}

// Start worker
func (p *Whisper) Start() error {
	return p.StartFunc(func() error {
		p.WithExit(func(exitChan chan bool) {
			p.throttleTicker = NewThrottleTicker(p.maxUpdatesPerSecond)

			p.Go(func(exit chan bool) {
				p.worker(p.recv, exit)
			})

		})

		return nil
	})
}

func (p *Whisper) Stop() {
	p.StopFunc(func() {
		p.throttleTicker.Stop()
	})
}
<commit_msg>missed multiworker persister<commit_after>package persister

import (
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/Sirupsen/logrus"
	whisper "github.com/lomik/go-whisper"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
)

const storeMutexCount = 32768

type StoreFunc func(p *Whisper, values *points.Points)

// Whisper write data to *.wsp files
type Whisper struct {
	helper.Stoppable
	updateOperations    uint32
	committedPoints     uint32
	recv                func(chan bool) *points.Points
	schemas             WhisperSchemas
	aggregation         *WhisperAggregation
	workersCount        int
	rootPath            string
	created             uint32 // counter
	sparse              bool
	maxUpdatesPerSecond int
	throttleTicker      *ThrottleTicker
	storeMutex          [storeMutexCount]sync.Mutex
	mockStore           func() (StoreFunc, func())
	// blockThrottleNs        uint64 // sum ns counter
	// blockQueueGetNs        uint64 // sum ns counter
	// blockAvoidConcurrentNs uint64 // sum ns counter
	// blockUpdateManyNs      uint64 // sum ns counter
}

// NewWhisper create instance of Whisper
func NewWhisper(rootPath string, schemas WhisperSchemas, aggregation *WhisperAggregation, recv func(chan bool) *points.Points) *Whisper {
	return &Whisper{
		recv:                recv,
		schemas:             schemas,
		aggregation:         aggregation,
		workersCount:        1,
		rootPath:            rootPath,
		maxUpdatesPerSecond: 0,
	}
}

// SetMaxUpdatesPerSecond enable throttling
func (p *Whisper) SetMaxUpdatesPerSecond(maxUpdatesPerSecond int) {
	p.maxUpdatesPerSecond = maxUpdatesPerSecond
}

// GetMaxUpdatesPerSecond returns current throttling speed
func (p *Whisper) GetMaxUpdatesPerSecond() int {
	return p.maxUpdatesPerSecond
}

// SetWorkers count
func (p *Whisper) SetWorkers(count int) {
	if count >= 1 {
		p.workersCount = count
	} else {
		p.workersCount = 1
	}
}

// SetSparse creation
func (p *Whisper) SetSparse(sparse bool) {
	p.sparse = sparse
}

func (p *Whisper) SetMockStore(fn func() (StoreFunc, func())) {
	p.mockStore = fn
}

func fnv32(key string) uint32 {
	hash := uint32(2166136261)
	const prime32 = uint32(16777619)
	for i := 0; i < len(key); i++ {
		hash *= prime32
		hash ^= uint32(key[i])
	}
	return hash
}

func store(p *Whisper, values *points.Points) {
	// avoid concurrent store same metric
	// @TODO: may be flock?
	// start := time.Now()
	mutexIndex := fnv32(values.Metric) % storeMutexCount
	p.storeMutex[mutexIndex].Lock()
	// atomic.AddUint64(&p.blockAvoidConcurrentNs, uint64(time.Since(start).Nanoseconds()))
	defer p.storeMutex[mutexIndex].Unlock()

	path := filepath.Join(p.rootPath, strings.Replace(values.Metric, ".", "/", -1)+".wsp")

	w, err := whisper.Open(path)
	if err != nil {
		// create new whisper if file not exists
		if !os.IsNotExist(err) {
			logrus.Errorf("[persister] Failed to open whisper file %s: %s", path, err.Error())
			return
		}

		schema, ok := p.schemas.Match(values.Metric)
		if !ok {
			logrus.Errorf("[persister] No storage schema defined for %s", values.Metric)
			return
		}

		aggr := p.aggregation.match(values.Metric)
		if aggr == nil {
			logrus.Errorf("[persister] No storage aggregation defined for %s", values.Metric)
			return
		}

		logrus.WithFields(logrus.Fields{
			"retention":    schema.RetentionStr,
			"schema":       schema.Name,
			"aggregation":  aggr.name,
			"xFilesFactor": aggr.xFilesFactor,
			"method":       aggr.aggregationMethodStr,
		}).Debugf("[persister] Creating %s", path)

		if err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
			logrus.Error(err)
			return
		}

		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.aggregationMethod, float32(aggr.xFilesFactor), &whisper.Options{
			Sparse: p.sparse,
		})
		if err != nil {
			logrus.Errorf("[persister] Failed to create new whisper file %s: %s", path, err.Error())
			return
		}

		atomic.AddUint32(&p.created, 1)
	}

	points := make([]*whisper.TimeSeriesPoint, len(values.Data))
	for i, r := range values.Data {
		points[i] = &whisper.TimeSeriesPoint{Time: int(r.Timestamp), Value: r.Value}
	}

	atomic.AddUint32(&p.committedPoints, uint32(len(values.Data)))
	atomic.AddUint32(&p.updateOperations, 1)

	defer w.Close()

	defer func() {
		if r := recover(); r != nil {
			logrus.Errorf("[persister] UpdateMany %s recovered: %s", path, r)
		}
	}()

	// start = time.Now()
	w.UpdateMany(points)
	// atomic.AddUint64(&p.blockUpdateManyNs, uint64(time.Since(start).Nanoseconds()))
}

func (p *Whisper) worker(recv func(chan bool) *points.Points, exit chan bool) {
	storeFunc := store
	var doneCb func()
	if p.mockStore != nil {
		storeFunc, doneCb = p.mockStore()
	}

LOOP:
	for {
		// start := time.Now()
		select {
		case <-p.throttleTicker.C:
			// atomic.AddUint64(&p.blockThrottleNs, uint64(time.Since(start).Nanoseconds()))
			// pass
		case <-exit:
			return
		}

		// start = time.Now()
		points := recv(exit)
		// atomic.AddUint64(&p.blockQueueGetNs, uint64(time.Since(start).Nanoseconds()))
		if points == nil {
			// exit closed
			break LOOP
		}
		storeFunc(p, points)
		if doneCb != nil {
			doneCb()
		}
	}
}

// Stat callback
func (p *Whisper) Stat(send helper.StatCallback) {
	updateOperations := atomic.LoadUint32(&p.updateOperations)
	committedPoints := atomic.LoadUint32(&p.committedPoints)
	atomic.AddUint32(&p.updateOperations, -updateOperations)
	atomic.AddUint32(&p.committedPoints, -committedPoints)

	created := atomic.LoadUint32(&p.created)
	atomic.AddUint32(&p.created, -created)

	send("updateOperations", float64(updateOperations))
	send("committedPoints", float64(committedPoints))
	if updateOperations > 0 {
		send("pointsPerUpdate", float64(committedPoints)/float64(updateOperations))
	} else {
		send("pointsPerUpdate", 0.0)
	}

	send("created", float64(created))

	// helper.SendAndSubstractUint64("blockThrottleNs", &p.blockThrottleNs, send)
	// helper.SendAndSubstractUint64("blockQueueGetNs", &p.blockQueueGetNs, send)
	// helper.SendAndSubstractUint64("blockAvoidConcurrentNs", &p.blockAvoidConcurrentNs, send)
	// helper.SendAndSubstractUint64("blockUpdateManyNs", &p.blockUpdateManyNs, send)

}

// Start worker
func (p *Whisper) Start() error {
	return p.StartFunc(func() error {
		p.WithExit(func(exitChan chan bool) {
			p.throttleTicker = NewThrottleTicker(p.maxUpdatesPerSecond)

			for i := 0; i < p.workersCount; i++ {
				p.Go(func(exit chan bool) {
					p.worker(p.recv, exit)
				})
			}

		})

		return nil
	})
}

func (p *Whisper) Stop() {
	p.StopFunc(func() {
		p.throttleTicker.Stop()
	})
}
<|endoftext|>


<commit_before>package main

import "strings"

type ARWObject struct {
	requestName string
	dataList    []map[string]string
	evntParams  SpecialEventParam
}

func (arwObj *ARWObject) PutString(key string, value string) {
	newField := make(map[string]string)
	newField[key] = value
	arwObj.dataList = append(arwObj.dataList, newField)
}

func (arwObj *ARWObject) GetString(key string) (value string) {
	for ii := 0; ii < len(arwObj.dataList); ii++ {
		c := arwObj.dataList[ii]
		for k, v := range c {
			if k == key {
				value = v
				return
			}
		}
	}
	return
}

func (arwObj *ARWObject) Compress() []byte {
	var data string

	data += arwObj.requestName + "."
	for ii := 0; ii < len(arwObj.dataList); ii++ {
		p := arwObj.dataList[ii]
		for k, v := range p {
			data += k + "#" + v + "_"
		}
	}
	data = strings.TrimRight(data, "_")
	bytes := make([]byte, 1024)
	bytes = []byte(data)
	return bytes
}

func (arwObj *ARWObject) Extract(bytes []byte) {
	data := string(bytes)

	dataParts := strings.Split(data, ".")
	if len(dataParts) == 3 {
		reqName := dataParts[0]
		arwObj.requestName = reqName

		params := strings.Split(dataParts[1], "_")

		for ii := 0; ii < len(params); ii++ {
			paramParts := strings.Split(params[ii], "#")

			newMap := make(map[string]string)

			if len(paramParts) == 2 {
				newMap[paramParts[0]] = paramParts[1]
				arwObj.dataList = append(arwObj.dataList, newMap)
			}
		}

		arwObj.evntParams.Extract(dataParts[2])
	}
}
<commit_msg>Put And Get Func Updated<commit_after>package main

import (
	"strconv"
	"strings"
)

type ARWObject struct {
	requestName string
	dataList    []map[string]string
	eventParams SpecialEventParam
}

func (arwObj *ARWObject) PutString(key string, value string) {
	newField := make(map[string]string)
	newField[key] = value
	arwObj.dataList = append(arwObj.dataList, newField)
}

func (arwObj *ARWObject) PutFloat(key string, value float64) {
	newField := make(map[string]string)
	newField[key] = strconv.FormatFloat(value, 'f', -1, 64)
	arwObj.dataList = append(arwObj.dataList, newField)
}

func (arwObj *ARWObject) PutInt(key string, value int) {
	newField := make(map[string]string)
	newField[key] = strconv.Itoa(value)
	arwObj.dataList = append(arwObj.dataList, newField)
}

func (arwObj *ARWObject) GetString(key string) (value string) {
	for ii := 0; ii < len(arwObj.dataList); ii++ {
		c := arwObj.dataList[ii]
		for k, v := range c {
			if k == key {
				value = v
				return
			}
		}
	}
	return
}
func (arwObj *ARWObject) GetFloat(key string) (value float64) {
	for ii := 0; ii < len(arwObj.dataList); ii++ {
		c := arwObj.dataList[ii]
		for k, v := range c {
			if k == key {
				value, _ = strconv.ParseFloat(v, 64)
				return
			}
		}
	}
	return
}

func (arwObj *ARWObject) GetInt(key string) (value int) {
	for ii := 0; ii < len(arwObj.dataList); ii++ {
		c := arwObj.dataList[ii]
		for k, v := range c {
			if k == key {
				value, _ = strconv.Atoi(v)
				return
			}
		}
	}
	return
}

func (arwObj *ARWObject) Compress() []byte {
	var data string

	data += arwObj.requestName + "."
	for ii := 0; ii < len(arwObj.dataList); ii++ {
		p := arwObj.dataList[ii]
		for k, v := range p {
			data += k + "#" + v + "_"
		}
	}
	data = strings.TrimRight(data, "_")
	bytes := make([]byte, 1024)
	bytes = []byte(data)
	return bytes
}

func (arwObj *ARWObject) Extract(bytes []byte) {
	data := string(bytes)

	dataParts := strings.Split(data, ".")
	if len(dataParts) == 3 {
		reqName := dataParts[0]
		arwObj.requestName = reqName

		params := strings.Split(dataParts[1], "_")

		for ii := 0; ii < len(params); ii++ {
			paramParts := strings.Split(params[ii], "#")

			newMap := make(map[string]string)

			if len(paramParts) == 2 {
				newMap[paramParts[0]] = paramParts[1]
				arwObj.dataList = append(arwObj.dataList, newMap)
			}
		}

		arwObj.eventParams.Extract(dataParts[2])
	}
}
<|endoftext|>


<commit_before>package notaryapi

import (
	"github.com/FactomProject/gocoding"
	"github.com/FactomProject/gocoding/json"
	"reflect"
	"strings"
	"encoding/hex"
)

func EncodeBinary(bytes *[]byte) (string){
	return hex.EncodeToString(*bytes)
}

func DecodeBinary(bytes *string) ([]byte, error){
	return hex.DecodeString(*bytes)
}

func NewUnmarshaller(decoding gocoding.Decoding) gocoding.Unmarshaller {
	return gocoding.NewUnmarshaller(NewDecoding(decoding))
}

func NewJSONUnmarshaller() gocoding.Unmarshaller {
	return NewUnmarshaller(json.Decoding)
}

func UnmarshalJSON(reader gocoding.SliceableRuneReader, obj interface{}) error {
	return NewJSONUnmarshaller().Unmarshal(json.Scan(reader), obj)
}

var signatureType = reflect.TypeOf(new(Signature)).Elem()

func NewDecoding(decoding gocoding.Decoding) gocoding.Decoding {
	return func(unmarshaller gocoding.Unmarshaller, theType reflect.Type) gocoding.Decoder {
		if !theType.ConvertibleTo(signatureType) {
			return decoding(unmarshaller, theType)
		}

		return SignatureDecoding(unmarshaller, theType)
	}
}

func SignatureDecoding(unmarshaller gocoding.Unmarshaller, theType reflect.Type) gocoding.Decoder {
	decoders := []gocoding.Decoder{
		(&ECDSASignature{}).ElementDecoding(unmarshaller, theType),
	}

	return func(scratch [64]byte, scanner gocoding.Scanner, value reflect.Value) {
		if scanner.Peek() == gocoding.ScannedLiteralBegin {
			null := scanner.NextValue()
			if null.IsValid() && null.IsNil() {
				value.Set(reflect.Zero(theType))
				return
			}
		}

		if !gocoding.PeekCheck(scanner, gocoding.ScannedStructBegin, gocoding.ScannedMapBegin) { return }

		// get the next code, should have at least a type key
		code := scanner.Continue()
		if code != gocoding.ScannedKeyBegin {
			gocoding.PeekCheck(scanner, gocoding.ScannedKeyBegin)
			return
		}

		// get the key
		key := scanner.NextValue()
		if key.Kind() != reflect.String {
			scanner.Error(gocoding.ErrorPrint("Decoding", "Invalid key type %s", key.Type().String()))
			return
		}
		if str := strings.ToLower(key.String()); str != `type` {
			scanner.Error(gocoding.ErrorPrintf("Decoding", "Decoding signature: expected 'type', got '%s'", str))
			return
		}

		scanner.Continue()
		_type := scanner.NextValue()
		switch _type.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
			 reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:

		default:
			scanner.Error(gocoding.ErrorPrint("Decoding", "Invalid type code type %s", _type.Type().String()))
		}

		switch _type.Int() {
		case ECDSASignatureType:
			value.Set(reflect.ValueOf(&ECDSASignature{}))
			decoders[0](scratch, scanner, value)
		}
	}
}
<commit_msg>cleanup for notaryapi/coding.go<commit_after>package notaryapi

import (
	"encoding/hex"
	"reflect"
	"strings"

	"github.com/FactomProject/gocoding"
	"github.com/FactomProject/gocoding/json"
)

func EncodeBinary(bytes *[]byte) (string) {
	return hex.EncodeToString(*bytes)
}

func DecodeBinary(bytes *string) ([]byte, error) {
	return hex.DecodeString(*bytes)
}

func NewUnmarshaller(decoding gocoding.Decoding) gocoding.Unmarshaller {
	return gocoding.NewUnmarshaller(NewDecoding(decoding))
}

func NewJSONUnmarshaller() gocoding.Unmarshaller {
	return NewUnmarshaller(json.Decoding)
}

func UnmarshalJSON(reader gocoding.SliceableRuneReader, obj interface{}) error {
	return NewJSONUnmarshaller().Unmarshal(json.Scan(reader), obj)
}

var signatureType = reflect.TypeOf(new(Signature)).Elem()

func NewDecoding(decoding gocoding.Decoding) gocoding.Decoding {
	return func(m gocoding.Unmarshaller, t reflect.Type) gocoding.Decoder {
		if !t.ConvertibleTo(signatureType) {
			return decoding(m, t)
		}

		return SignatureDecoding(m, t)
	}
}

func SignatureDecoding(m gocoding.Unmarshaller, t reflect.Type) gocoding.Decoder {
	decoders := []gocoding.Decoder{
		(&ECDSASignature{}).ElementDecoding(m, t),
	}

	return func(scratch [64]byte, scanner gocoding.Scanner, value reflect.Value) {
		if scanner.Peek() == gocoding.ScannedLiteralBegin {
			null := scanner.NextValue()
			if null.IsValid() && null.IsNil() {
				value.Set(reflect.Zero(t))
				return
			}
		}

		if !gocoding.PeekCheck(scanner, gocoding.ScannedStructBegin, gocoding.ScannedMapBegin) {
			return
		}

		// get the next code, should have at least a type key
		code := scanner.Continue()
		if code != gocoding.ScannedKeyBegin {
			gocoding.PeekCheck(scanner, gocoding.ScannedKeyBegin)
			return
		}

		// get the key
		key := scanner.NextValue()
		if key.Kind() != reflect.String {
			scanner.Error(gocoding.ErrorPrint("Decoding", "Invalid key type %s", key.Type().String()))
			return
		}
		if str := strings.ToLower(key.String()); str != `type` {
			scanner.Error(gocoding.ErrorPrintf("Decoding", "Decoding signature: expected 'type', got '%s'", str))
			return
		}

		scanner.Continue()
		_type := scanner.NextValue()
		switch _type.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:

		default:
			scanner.Error(gocoding.ErrorPrint("Decoding", "Invalid type code type %s", _type.Type().String()))
		}

		switch _type.Int() {
		case ECDSASignatureType:
			value.Set(reflect.ValueOf(&ECDSASignature{}))
			decoders[0](scratch, scanner, value)
		}
	}
}
<|endoftext|>


<commit_before>// Package status is here just to say that the API is up and that it can
// access the CouchDB databases, for debugging and monitoring purposes.
package status

import (
	"net/http"

	"github.com/cozy/checkup"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/echo"
)

// Status responds with the status of the service
func Status(c echo.Context) error {
	checker := checkup.HTTPChecker{
		Name:     "CouchDB",
		Client:   config.GetConfig().CouchDB.Client,
		URL:      config.CouchURL().String(),
		Attempts: 3,
	}

	var message string
	couchdb, err := checker.Check()
	if err != nil || couchdb.Status() != checkup.Healthy {
		message = "KO"
	} else {
		message = "OK"
	}

	return c.JSON(http.StatusOK, echo.Map{
		"message": message,
		"couchdb": couchdb.Status(),
	})
}

// Routes sets the routing for the status service
func Routes(router *echo.Group) {
	router.GET("", Status)
	router.HEAD("", Status)
	router.GET("/", Status)
	router.HEAD("/", Status)
}
<commit_msg>Use /_up for checking CouchDB status (#1809)<commit_after>// Package status is here just to say that the API is up and that it can
// access the CouchDB databases, for debugging and monitoring purposes.
package status

import (
	"net/http"

	"github.com/cozy/checkup"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/echo"
)

// Status responds with the status of the service
func Status(c echo.Context) error {
	u := config.CouchURL()
	u.Path = "/_up"
	checker := checkup.HTTPChecker{
		Name:     "CouchDB",
		Client:   config.GetConfig().CouchDB.Client,
		URL:      u.String(),
		Attempts: 3,
	}

	var message string
	couchdb, err := checker.Check()
	if err != nil || couchdb.Status() != checkup.Healthy {
		message = "KO"
	} else {
		message = "OK"
	}

	return c.JSON(http.StatusOK, echo.Map{
		"message": message,
		"couchdb": couchdb.Status(),
	})
}

// Routes sets the routing for the status service
func Routes(router *echo.Group) {
	router.GET("", Status)
	router.HEAD("", Status)
	router.GET("/", Status)
	router.HEAD("/", Status)
}
<|endoftext|>


<commit_before>package web

var (
	templateFile = `

  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <link href="https://fonts.googleapis.com/css?family=Source+Code+Pro&display=swap" rel="stylesheet">

Graph

Nodes

Routes

Services

    {{.}}

`

	graphTemplate = `

  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <link href="https://fonts.googleapis.com/css?family=Source+Code+Pro&display=swap" rel="stylesheet">

Graph

Nodes

Routes

Services

    <canvas id="graph"></canvas>

`
)
<commit_msg>add network web heading<commit_after>package web

var (
	templateFile = `

  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <link href="https://fonts.googleapis.com/css?family=Source+Code+Pro&display=swap" rel="stylesheet">

Network

Graph

Nodes

Routes

Services

    {{.}}

`

	graphTemplate = `

  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <link href="https://fonts.googleapis.com/css?family=Source+Code+Pro&display=swap" rel="stylesheet">

Network

Graph

Nodes

Routes

Services

    <canvas id="graph"></canvas>

`
)
<|endoftext|>


<commit_before>// Copyright 2014 Dmitry Chestnykh. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bcrypt_pbkdf implements password-based key derivation function based
// on bcrypt compatible with bcrypt_pbkdf(3) from OpenBSD.
package bcrypt_pbkdf

import (
	"crypto/sha512"
	"errors"
)

// Key derives a key from the password, salt and rounds count, returning a
// []byte of length keyLen that can be used as cryptographic key.
//
// Remember to get a good random salt of at least 16 bytes.  Using a higher
// rounds count will increase the cost of an exhaustive search but will also
// make derivation proportionally slower.
func Key(password, salt []byte, rounds, keyLen int) ([]byte, error) {
	if rounds < 1 {
		return nil, errors.New("bcryptkdf: number of rounds is too small")
	}
	if len(password) == 0 {
		return nil, errors.New("bcryptkdf: empty password")
	}
	if len(salt) == 0 || len(salt) > 1<<20 {
		return nil, errors.New("bcryptkdf: bad salt length")
	}
	if keyLen > 1024 {
		return nil, errors.New("bcryptkdf: keyLen is too large")
	}
	var shapass, shasalt [sha512.Size]byte
	var out, tmp [32]byte
	var cnt [4]byte

	numBlocks := (keyLen + len(out) - 1) / len(out)
	key := make([]byte, numBlocks*len(out))

	h := sha512.New()
	h.Write(password)
	h.Sum(shapass[:0])

	for block := 1; block <= numBlocks; block++ {
		h.Reset()
		h.Write(salt)
		cnt[0] = byte(block >> 24)
		cnt[1] = byte(block >> 16)
		cnt[2] = byte(block >> 8)
		cnt[3] = byte(block)
		h.Write(cnt[:])
		bcryptHash(tmp[:], shapass[:], h.Sum(shasalt[:0]))
		copy(out[:], tmp[:])

		for i := 2; i <= rounds; i++ {
			h.Reset()
			h.Write(tmp[:])
			bcryptHash(tmp[:], shapass[:], h.Sum(shasalt[:0]))
			for j := 0; j < len(out); j++ {
				out[j] ^= tmp[j]
			}
		}

		for i, v := range out {
			key[i*numBlocks+(block-1)] = v
		}
	}
	return key[:keyLen], nil
}

var magic = []byte("OxychromaticBlowfishSwatDynamite")

func bcryptHash(out, shapass, shasalt []byte) {
	var c cipher
	initCipher(&c)
	expandKeyWithSalt(shapass, shasalt, &c)
	for i := 0; i < 64; i++ {
		expandKey(shasalt, &c)
		expandKey(shapass, &c)
	}
	var ct [8]uint32
	j := 0
	for i := range ct {
		ct[i] = getWord(magic, &j)
	}
	for i := 0; i < 8; i += 2 {
		l, r := ct[i], ct[i+1]
		for j := 0; j < 64; j++ {
			l, r = encryptBlock(l, r, &c)
		}
		ct[i], ct[i+1] = l, r
	}
	for i, v := range ct {
		out[4*i+3] = byte(v >> 24)
		out[4*i+2] = byte(v >> 16)
		out[4*i+1] = byte(v >> 8)
		out[4*i+0] = byte(v)
	}
}
<commit_msg>Fix package name in errors.<commit_after>// Copyright 2014 Dmitry Chestnykh. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bcrypt_pbkdf implements password-based key derivation function based
// on bcrypt compatible with bcrypt_pbkdf(3) from OpenBSD.
package bcrypt_pbkdf

import (
	"crypto/sha512"
	"errors"
)

// Key derives a key from the password, salt and rounds count, returning a
// []byte of length keyLen that can be used as cryptographic key.
//
// Remember to get a good random salt of at least 16 bytes.  Using a higher
// rounds count will increase the cost of an exhaustive search but will also
// make derivation proportionally slower.
func Key(password, salt []byte, rounds, keyLen int) ([]byte, error) {
	if rounds < 1 {
		return nil, errors.New("bcrypt_pbkdf: number of rounds is too small")
	}
	if len(password) == 0 {
		return nil, errors.New("bcrypt_pbkdf: empty password")
	}
	if len(salt) == 0 || len(salt) > 1<<20 {
		return nil, errors.New("bcrypt_pbkdf: bad salt length")
	}
	if keyLen > 1024 {
		return nil, errors.New("bcrypt_pbkdf: keyLen is too large")
	}
	var shapass, shasalt [sha512.Size]byte
	var out, tmp [32]byte
	var cnt [4]byte

	numBlocks := (keyLen + len(out) - 1) / len(out)
	key := make([]byte, numBlocks*len(out))

	h := sha512.New()
	h.Write(password)
	h.Sum(shapass[:0])

	for block := 1; block <= numBlocks; block++ {
		h.Reset()
		h.Write(salt)
		cnt[0] = byte(block >> 24)
		cnt[1] = byte(block >> 16)
		cnt[2] = byte(block >> 8)
		cnt[3] = byte(block)
		h.Write(cnt[:])
		bcryptHash(tmp[:], shapass[:], h.Sum(shasalt[:0]))
		copy(out[:], tmp[:])

		for i := 2; i <= rounds; i++ {
			h.Reset()
			h.Write(tmp[:])
			bcryptHash(tmp[:], shapass[:], h.Sum(shasalt[:0]))
			for j := 0; j < len(out); j++ {
				out[j] ^= tmp[j]
			}
		}

		for i, v := range out {
			key[i*numBlocks+(block-1)] = v
		}
	}
	return key[:keyLen], nil
}

var magic = []byte("OxychromaticBlowfishSwatDynamite")

func bcryptHash(out, shapass, shasalt []byte) {
	var c cipher
	initCipher(&c)
	expandKeyWithSalt(shapass, shasalt, &c)
	for i := 0; i < 64; i++ {
		expandKey(shasalt, &c)
		expandKey(shapass, &c)
	}
	var ct [8]uint32
	j := 0
	for i := range ct {
		ct[i] = getWord(magic, &j)
	}
	for i := 0; i < 8; i += 2 {
		l, r := ct[i], ct[i+1]
		for j := 0; j < 64; j++ {
			l, r = encryptBlock(l, r, &c)
		}
		ct[i], ct[i+1] = l, r
	}
	for i, v := range ct {
		out[4*i+3] = byte(v >> 24)
		out[4*i+2] = byte(v >> 16)
		out[4*i+1] = byte(v >> 8)
		out[4*i+0] = byte(v)
	}
}
<|endoftext|>


<commit_before>//
// Generate an RSS feed from a PostgreSQL database containing tweets.
//
// The tweet database is the one populated by my twitter-tcl twitter_poll
// program.
//
package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/horgh/config"
	"github.com/horgh/gorse/gorselib"
	_ "github.com/lib/pq"
)

// FeedURI is the URI set on the RSS feed's channel element's link element. It
// need not be a real URI but should be unique.
var FeedURI = "https://leviathan.summercat.com/tweets/"

// Tweet describe a tweet pulled from the database.
type Tweet struct {
	Nick    string
	Text    string
	Time    time.Time
	TweetID int64
}

// MyConfig holds configuration values.
type MyConfig struct {
	DBUser string
	DBPass string
	DBName string
	DBHost string
	// The number of recent tweets to put in the XML.
	NumTweets uint64
}

// connectToDB opens a new connection to the database.
func connectToDB(name string, user string, pass string, host string) (*sql.DB,
	error) {
	dsn := fmt.Sprintf("user=%s password=%s dbname=%s host=%s", user, pass, name,
		host)

	db, err := sql.Open("postgres", dsn)
	if err != nil {
		return nil, fmt.Errorf("failed to connect to the database: %s", err)
	}

	return db, nil
}

// getTweets retrieves tweets from a database.
func getTweets(config *MyConfig) ([]Tweet, error) {
	db, err := connectToDB(config.DBName, config.DBUser, config.DBPass,
		config.DBHost)
	if err != nil {
		return nil, err
	}

	defer func() {
		err := db.Close()
		if err != nil {
			log.Printf("Database close: %s", err)
		}
	}()

	// get most recent tweets.
	sql := `
SELECT nick, text, time, tweet_id
FROM tweet
ORDER BY time DESC
LIMIT $1
`
	rows, err := db.Query(sql, config.NumTweets)
	if err != nil {
		return nil, fmt.Errorf("query failure: %s", err)
	}

	var tweets []Tweet
	for rows.Next() {
		tweet := Tweet{}

		err = rows.Scan(&tweet.Nick, &tweet.Text, &tweet.Time, &tweet.TweetID)
		if err != nil {
			return nil, fmt.Errorf("failed to scan row: %s", err)
		}

		tweets = append(tweets, tweet)
	}

	return tweets, nil
}

// Create a URL to the status.
//
// Apparently this URL is not in the tweet status payload.
//
// Form: https://twitter.com/<screenname>/status/<tweetid>
func createStatusURL(screenName string, tweetID int64) string {
	return fmt.Sprintf("https://twitter.com/%s/status/%d", screenName, tweetID)
}

func main() {
	log.SetFlags(log.Ltime | log.Llongfile)

	outputFile := flag.String("output-file", "", "Output XML file to write.")
	configFile := flag.String("config-file", "", "Config file")

	flag.Parse()

	if len(*outputFile) == 0 || len(*configFile) == 0 {
		fmt.Println("You must provide a config file.")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var settings MyConfig
	err := config.GetConfig(*configFile, &settings)
	if err != nil {
		log.Fatalf("Failed to retrieve config: %s", err)
	}

	// TODO: We could run validation on each config item.

	gorselib.SetQuiet(true)

	tweets, err := getTweets(&settings)
	if err != nil {
		log.Fatalf("Failed to retrieve tweets: %s", err)
	}

	rss := gorselib.Feed{
		Title:       "Twitreader",
		Link:        FeedURI,
		Description: "Twitreader tweets",
		PubDate:     time.Now(),
	}

	for _, tweet := range tweets {
		rss.Items = append(rss.Items, gorselib.Item{
			Title:       fmt.Sprintf("%s", tweet.Nick),
			Link:        createStatusURL(tweet.Nick, tweet.TweetID),
			Description: tweet.Text,
			PubDate:     tweet.Time,
		})
	}

	err = gorselib.WriteFeedXML(rss, *outputFile)
	if err != nil {
		log.Fatalf("Failed to write XML: %s", err)
	}
}
<commit_msg>Update error message to mention both parameters<commit_after>//
// Generate an RSS feed from a PostgreSQL database containing tweets.
//
// The tweet database is the one populated by my twitter-tcl twitter_poll
// program.
//
package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/horgh/config"
	"github.com/horgh/gorse/gorselib"
	_ "github.com/lib/pq"
)

// FeedURI is the URI set on the RSS feed's channel element's link element. It
// need not be a real URI but should be unique.
var FeedURI = "https://leviathan.summercat.com/tweets/"

// Tweet describe a tweet pulled from the database.
type Tweet struct {
	Nick    string
	Text    string
	Time    time.Time
	TweetID int64
}

// MyConfig holds configuration values.
type MyConfig struct {
	DBUser string
	DBPass string
	DBName string
	DBHost string
	// The number of recent tweets to put in the XML.
	NumTweets uint64
}

// connectToDB opens a new connection to the database.
func connectToDB(name string, user string, pass string, host string) (*sql.DB,
	error) {
	dsn := fmt.Sprintf("user=%s password=%s dbname=%s host=%s", user, pass, name,
		host)

	db, err := sql.Open("postgres", dsn)
	if err != nil {
		return nil, fmt.Errorf("failed to connect to the database: %s", err)
	}

	return db, nil
}

// getTweets retrieves tweets from a database.
func getTweets(config *MyConfig) ([]Tweet, error) {
	db, err := connectToDB(config.DBName, config.DBUser, config.DBPass,
		config.DBHost)
	if err != nil {
		return nil, err
	}

	defer func() {
		err := db.Close()
		if err != nil {
			log.Printf("Database close: %s", err)
		}
	}()

	// get most recent tweets.
	sql := `
SELECT nick, text, time, tweet_id
FROM tweet
ORDER BY time DESC
LIMIT $1
`
	rows, err := db.Query(sql, config.NumTweets)
	if err != nil {
		return nil, fmt.Errorf("query failure: %s", err)
	}

	var tweets []Tweet
	for rows.Next() {
		tweet := Tweet{}

		err = rows.Scan(&tweet.Nick, &tweet.Text, &tweet.Time, &tweet.TweetID)
		if err != nil {
			return nil, fmt.Errorf("failed to scan row: %s", err)
		}

		tweets = append(tweets, tweet)
	}

	return tweets, nil
}

// Create a URL to the status.
//
// Apparently this URL is not in the tweet status payload.
//
// Form: https://twitter.com/<screenname>/status/<tweetid>
func createStatusURL(screenName string, tweetID int64) string {
	return fmt.Sprintf("https://twitter.com/%s/status/%d", screenName, tweetID)
}

func main() {
	log.SetFlags(log.Ltime | log.Llongfile)

	outputFile := flag.String("output-file", "", "Output XML file to write.")
	configFile := flag.String("config-file", "", "Config file")

	flag.Parse()

	if len(*outputFile) == 0 || len(*configFile) == 0 {
		fmt.Println("You must provide an output file and a config file.")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var settings MyConfig
	err := config.GetConfig(*configFile, &settings)
	if err != nil {
		log.Fatalf("Failed to retrieve config: %s", err)
	}

	// TODO: We could run validation on each config item.

	gorselib.SetQuiet(true)

	tweets, err := getTweets(&settings)
	if err != nil {
		log.Fatalf("Failed to retrieve tweets: %s", err)
	}

	rss := gorselib.Feed{
		Title:       "Twitreader",
		Link:        FeedURI,
		Description: "Twitreader tweets",
		PubDate:     time.Now(),
	}

	for _, tweet := range tweets {
		rss.Items = append(rss.Items, gorselib.Item{
			Title:       fmt.Sprintf("%s", tweet.Nick),
			Link:        createStatusURL(tweet.Nick, tweet.TweetID),
			Description: tweet.Text,
			PubDate:     tweet.Time,
		})
	}

	err = gorselib.WriteFeedXML(rss, *outputFile)
	if err != nil {
		log.Fatalf("Failed to write XML: %s", err)
	}
}
<|endoftext|>


<commit_before>package transactionpool

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// purge removes all transactions from the transaction pool.
func (tp *TransactionPool) purge() {
	tp.knownObjects = make(map[ObjectID]TransactionSetID)
	tp.transactionSets = make(map[TransactionSetID][]types.Transaction)
	tp.transactionSetDiffs = make(map[TransactionSetID]modules.ConsensusChange)
	tp.transactionListSize = 0
}

// ProcessConsensusChange gets called to inform the transaction pool of changes
// to the consensus set.
func (tp *TransactionPool) ProcessConsensusChange(cc modules.ConsensusChange) {
	lockID := tp.mu.Lock()

	// TODO: Right now, transactions that were reverted to not get saved and
	// retried, because some transactions such as storage proofs might be
	// illegal, and there's no good way to preserve dependencies when illegal
	// transactions are suddenly involved.
	//
	// One potential solution is to have modules manually do resubmission if
	// something goes wrong. Another is to have the transaction pool remember
	// recent transaction sets on the off chance that they become valid again
	// due to a reorg.
	//
	// Another option is to scan through the blocks transactions one at a time
	// check if they are valid. If so, lump them in a set with the next guy.
	// When they stop being valid, you've found a guy to throw away. It's n^2
	// in the number of transactions in the block.

	// Save all of the current unconfirmed transaction sets into a list.
	var unconfirmedSets [][]types.Transaction
	for _, tSet := range tp.transactionSets {
		unconfirmedSets = append(unconfirmedSets, tSet)
	}

	// Purge the transaction pool. Some of the transactions sets may be invalid
	// after the consensus change.
	tp.purge()

	// Add all of the unconfirmed transaction sets back to the transaction
	// pool. The ones that are invalid will throw an error and will not be
	// re-added.
	//
	// Accepting a transaction set requires locking the consensus set (to check
	// validity). But, ProcessConsensusChange is only called when the consensus
	// set is already locked, causing a deadlock problem. Therefore,
	// transactions are readded to the pool in a goroutine, so that this
	// function can finish and consensus can unlock. The tpool lock is held
	// however until the goroutine completes.
	//
	// Which means that no other modules can require a tpool lock when
	// processing consensus changes. Overall, the locking is pretty fragile and
	// more rules need to be put in place.
	for _, set := range unconfirmedSets {
		tp.acceptTransactionSet(set) // Error is not checked.
	}

	// Inform subscribers that an update has executed.
	tp.consensusChangeIndex++
	tp.updateSubscribersConsensus(cc)
	tp.updateSubscribersTransactions()
	tp.mu.Unlock(lockID)
}

// PurgeTransactionPool deletes all transactions from the transaction pool.
func (tp *TransactionPool) PurgeTransactionPool() {
	lockID := tp.mu.Lock()
	tp.purge()
	tp.mu.Unlock(lockID)
}
<commit_msg>correct demoting on transaciton pool<commit_after>package transactionpool

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// purge removes all transactions from the transaction pool.
func (tp *TransactionPool) purge() {
	tp.knownObjects = make(map[ObjectID]TransactionSetID)
	tp.transactionSets = make(map[TransactionSetID][]types.Transaction)
	tp.transactionSetDiffs = make(map[TransactionSetID]modules.ConsensusChange)
	tp.transactionListSize = 0
}

// ProcessConsensusChange gets called to inform the transaction pool of changes
// to the consensus set.
func (tp *TransactionPool) ProcessConsensusChange(cc modules.ConsensusChange) {
	lockID := tp.mu.Lock()

	// TODO: Right now, transactions that were reverted to not get saved and
	// retried, because some transactions such as storage proofs might be
	// illegal, and there's no good way to preserve dependencies when illegal
	// transactions are suddenly involved.
	//
	// One potential solution is to have modules manually do resubmission if
	// something goes wrong. Another is to have the transaction pool remember
	// recent transaction sets on the off chance that they become valid again
	// due to a reorg.
	//
	// Another option is to scan through the blocks transactions one at a time
	// check if they are valid. If so, lump them in a set with the next guy.
	// When they stop being valid, you've found a guy to throw away. It's n^2
	// in the number of transactions in the block.

	// Save all of the current unconfirmed transaction sets into a list.
	var unconfirmedSets [][]types.Transaction
	for _, tSet := range tp.transactionSets {
		unconfirmedSets = append(unconfirmedSets, tSet)
	}

	// Purge the transaction pool. Some of the transactions sets may be invalid
	// after the consensus change.
	tp.purge()

	// Add all of the unconfirmed transaction sets back to the transaction
	// pool. The ones that are invalid will throw an error and will not be
	// re-added.
	//
	// Accepting a transaction set requires locking the consensus set (to check
	// validity). But, ProcessConsensusChange is only called when the consensus
	// set is already locked, causing a deadlock problem. Therefore,
	// transactions are readded to the pool in a goroutine, so that this
	// function can finish and consensus can unlock. The tpool lock is held
	// however until the goroutine completes.
	//
	// Which means that no other modules can require a tpool lock when
	// processing consensus changes. Overall, the locking is pretty fragile and
	// more rules need to be put in place.
	for _, set := range unconfirmedSets {
		tp.acceptTransactionSet(set) // Error is not checked.
	}

	// Inform subscribers that an update has executed.
	tp.consensusChangeIndex++
	tp.mu.Demote()
	tp.updateSubscribersConsensus(cc)
	tp.updateSubscribersTransactions()
	tp.mu.RUnlock(lockID)
}

// PurgeTransactionPool deletes all transactions from the transaction pool.
func (tp *TransactionPool) PurgeTransactionPool() {
	lockID := tp.mu.Lock()
	tp.purge()
	tp.mu.Unlock(lockID)
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/metrics"
	"github.com/grafana/grafana/pkg/middleware"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/search"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
)

func isDasboardStarredByUser(c *middleware.Context, dashId int64) (bool, error) {
	if !c.IsSignedIn {
		return false, nil
	}

	query := m.IsStarredByUserQuery{UserId: c.UserId, DashboardId: dashId}
	if err := bus.Dispatch(&query); err != nil {
		return false, err
	}

	return query.Result, nil
}

func GetDashboard(c *middleware.Context) {
	metrics.M_Api_Dashboard_Get.Inc(1)

	slug := strings.ToLower(c.Params(":slug"))

	query := m.GetDashboardQuery{Slug: slug, OrgId: c.OrgId}
	err := bus.Dispatch(&query)
	if err != nil {
		c.JsonApiErr(404, "Dashboard not found", nil)
		return
	}

	isStarred, err := isDasboardStarredByUser(c, query.Result.Id)
	if err != nil {
		c.JsonApiErr(500, "Error while checking if dashboard was starred by user", err)
		return
	}

	dash := query.Result
	dto := dtos.DashboardFullWithMeta{
		Dashboard: dash.Data,
		Meta: dtos.DashboardMeta{
			IsStarred: isStarred,
			Slug:      slug,
			Type:      m.DashTypeDB,
			CanStar:   c.IsSignedIn,
			CanSave:   c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR,
			CanEdit:   canEditDashboard(c.OrgRole),
			Created:   dash.Created,
			Updated:   dash.Updated,
			UpdatedBy: strconv.FormatInt(dash.UpdatedBy, 10),
		},
	}

	c.JSON(200, dto)
}

func DeleteDashboard(c *middleware.Context) {
	slug := c.Params(":slug")

	query := m.GetDashboardQuery{Slug: slug, OrgId: c.OrgId}
	if err := bus.Dispatch(&query); err != nil {
		c.JsonApiErr(404, "Dashboard not found", nil)
		return
	}

	cmd := m.DeleteDashboardCommand{Slug: slug, OrgId: c.OrgId}
	if err := bus.Dispatch(&cmd); err != nil {
		c.JsonApiErr(500, "Failed to delete dashboard", err)
		return
	}

	var resp = map[string]interface{}{"title": query.Result.Title}

	c.JSON(200, resp)
}

func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) {
	cmd.OrgId = c.OrgId
	cmd.UpdatedBy = c.UserId

	dash := cmd.GetDashboardModel()
	if dash.Id == 0 {
		limitReached, err := middleware.QuotaReached(c, "dashboard")
		if err != nil {
			c.JsonApiErr(500, "failed to get quota", err)
			return
		}
		if limitReached {
			c.JsonApiErr(403, "Quota reached", nil)
			return
		}
	}

	err := bus.Dispatch(&cmd)
	if err != nil {
		if err == m.ErrDashboardWithSameNameExists {
			c.JSON(412, util.DynMap{"status": "name-exists", "message": err.Error()})
			return
		}
		if err == m.ErrDashboardVersionMismatch {
			c.JSON(412, util.DynMap{"status": "version-mismatch", "message": err.Error()})
			return
		}
		if err == m.ErrDashboardNotFound {
			c.JSON(404, util.DynMap{"status": "not-found", "message": err.Error()})
			return
		}
		c.JsonApiErr(500, "Failed to save dashboard", err)
		return
	}

	metrics.M_Api_Dashboard_Post.Inc(1)

	c.JSON(200, util.DynMap{"status": "success", "slug": cmd.Result.Slug, "version": cmd.Result.Version})
}

func canEditDashboard(role m.RoleType) bool {
	return role == m.ROLE_ADMIN || role == m.ROLE_EDITOR || role == m.ROLE_READ_ONLY_EDITOR
}

func GetHomeDashboard(c *middleware.Context) {
	filePath := path.Join(setting.StaticRootPath, "dashboards/home.json")
	file, err := os.Open(filePath)
	if err != nil {
		c.JsonApiErr(500, "Failed to load home dashboard", err)
		return
	}

	dash := dtos.DashboardFullWithMeta{}
	dash.Meta.IsHome = true
	dash.Meta.CanEdit = canEditDashboard(c.OrgRole)
	jsonParser := json.NewDecoder(file)
	if err := jsonParser.Decode(&dash.Dashboard); err != nil {
		c.JsonApiErr(500, "Failed to load home dashboard", err)
		return
	}

	c.JSON(200, &dash)
}

func GetDashboardFromJsonFile(c *middleware.Context) {
	file := c.Params(":file")

	dashboard := search.GetDashboardFromJsonIndex(file)
	if dashboard == nil {
		c.JsonApiErr(404, "Dashboard not found", nil)
		return
	}

	dash := dtos.DashboardFullWithMeta{Dashboard: dashboard.Data}
	dash.Meta.Type = m.DashTypeJson
	dash.Meta.CanEdit = canEditDashboard(c.OrgRole)

	c.JSON(200, &dash)
}

func GetDashboardTags(c *middleware.Context) {
	query := m.GetDashboardTagsQuery{OrgId: c.OrgId}
	err := bus.Dispatch(&query)
	if err != nil {
		c.JsonApiErr(500, "Failed to get tags from database", err)
		return
	}

	c.JSON(200, query.Result)
}
<commit_msg>Able to display login Id of the user in Last Updated By<commit_after>package api

import (
	"encoding/json"
	"os"
	"path"
	"strings"

	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/metrics"
	"github.com/grafana/grafana/pkg/middleware"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/search"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
)

func isDasboardStarredByUser(c *middleware.Context, dashId int64) (bool, error) {
	if !c.IsSignedIn {
		return false, nil
	}

	query := m.IsStarredByUserQuery{UserId: c.UserId, DashboardId: dashId}
	if err := bus.Dispatch(&query); err != nil {
		return false, err
	}

	return query.Result, nil
}

func GetDashboard(c *middleware.Context) {
	metrics.M_Api_Dashboard_Get.Inc(1)

	slug := strings.ToLower(c.Params(":slug"))

	query := m.GetDashboardQuery{Slug: slug, OrgId: c.OrgId}
	err := bus.Dispatch(&query)
	if err != nil {
		c.JsonApiErr(404, "Dashboard not found", nil)
		return
	}

	isStarred, err := isDasboardStarredByUser(c, query.Result.Id)
	if err != nil {
		c.JsonApiErr(500, "Error while checking if dashboard was starred by user", err)
		return
	}

	dash := query.Result

	// Finding the last updater of the dashboard
	updater := "Anonymous"
	if dash.UpdatedBy != 0 {
		userQuery := m.GetUserByIdQuery{Id: dash.UpdatedBy}
		userErr := bus.Dispatch(&userQuery)
		if userErr != nil {
			updater = "Unknown"
		} else {
			user := userQuery.Result
			updater = user.Login
		}
	}

	dto := dtos.DashboardFullWithMeta{
		Dashboard: dash.Data,
		Meta: dtos.DashboardMeta{
			IsStarred: isStarred,
			Slug:      slug,
			Type:      m.DashTypeDB,
			CanStar:   c.IsSignedIn,
			CanSave:   c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR,
			CanEdit:   canEditDashboard(c.OrgRole),
			Created:   dash.Created,
			Updated:   dash.Updated,
			UpdatedBy: updater,
		},
	}

	c.JSON(200, dto)
}

func DeleteDashboard(c *middleware.Context) {
	slug := c.Params(":slug")

	query := m.GetDashboardQuery{Slug: slug, OrgId: c.OrgId}
	if err := bus.Dispatch(&query); err != nil {
		c.JsonApiErr(404, "Dashboard not found", nil)
		return
	}

	cmd := m.DeleteDashboardCommand{Slug: slug, OrgId: c.OrgId}
	if err := bus.Dispatch(&cmd); err != nil {
		c.JsonApiErr(500, "Failed to delete dashboard", err)
		return
	}

	var resp = map[string]interface{}{"title": query.Result.Title}

	c.JSON(200, resp)
}

func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) {
	cmd.OrgId = c.OrgId

	if !c.IsSignedIn {
		cmd.UpdatedBy = 0
	} else {
		cmd.UpdatedBy = c.UserId
	}

	dash := cmd.GetDashboardModel()
	if dash.Id == 0 {
		limitReached, err := middleware.QuotaReached(c, "dashboard")
		if err != nil {
			c.JsonApiErr(500, "failed to get quota", err)
			return
		}
		if limitReached {
			c.JsonApiErr(403, "Quota reached", nil)
			return
		}
	}

	err := bus.Dispatch(&cmd)
	if err != nil {
		if err == m.ErrDashboardWithSameNameExists {
			c.JSON(412, util.DynMap{"status": "name-exists", "message": err.Error()})
			return
		}
		if err == m.ErrDashboardVersionMismatch {
			c.JSON(412, util.DynMap{"status": "version-mismatch", "message": err.Error()})
			return
		}
		if err == m.ErrDashboardNotFound {
			c.JSON(404, util.DynMap{"status": "not-found", "message": err.Error()})
			return
		}
		c.JsonApiErr(500, "Failed to save dashboard", err)
		return
	}

	metrics.M_Api_Dashboard_Post.Inc(1)

	c.JSON(200, util.DynMap{"status": "success", "slug": cmd.Result.Slug, "version": cmd.Result.Version})
}

func canEditDashboard(role m.RoleType) bool {
	return role == m.ROLE_ADMIN || role == m.ROLE_EDITOR || role == m.ROLE_READ_ONLY_EDITOR
}

func GetHomeDashboard(c *middleware.Context) {
	filePath := path.Join(setting.StaticRootPath, "dashboards/home.json")
	file, err := os.Open(filePath)
	if err != nil {
		c.JsonApiErr(500, "Failed to load home dashboard", err)
		return
	}

	dash := dtos.DashboardFullWithMeta{}
	dash.Meta.IsHome = true
	dash.Meta.CanEdit = canEditDashboard(c.OrgRole)
	jsonParser := json.NewDecoder(file)
	if err := jsonParser.Decode(&dash.Dashboard); err != nil {
		c.JsonApiErr(500, "Failed to load home dashboard", err)
		return
	}

	c.JSON(200, &dash)
}

func GetDashboardFromJsonFile(c *middleware.Context) {
	file := c.Params(":file")

	dashboard := search.GetDashboardFromJsonIndex(file)
	if dashboard == nil {
		c.JsonApiErr(404, "Dashboard not found", nil)
		return
	}

	dash := dtos.DashboardFullWithMeta{Dashboard: dashboard.Data}
	dash.Meta.Type = m.DashTypeJson
	dash.Meta.CanEdit = canEditDashboard(c.OrgRole)

	c.JSON(200, &dash)
}

func GetDashboardTags(c *middleware.Context) {
	query := m.GetDashboardTagsQuery{OrgId: c.OrgId}
	err := bus.Dispatch(&query)
	if err != nil {
		c.JsonApiErr(500, "Failed to get tags from database", err)
		return
	}

	c.JSON(200, query.Result)
}
<|endoftext|>


<commit_before>package loader

import (
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
	"github.com/hellofresh/janus/pkg/web"
)

// Params initialization options.
type Params struct {
	Router      router.Router
	Storage     store.Store
	APIRepo     api.Repository
	OAuthRepo   oauth.Repository
	ProxyParams proxy.Params
}

// Load loads all the basic components and definitions into a router
func Load(params Params) {
	pluginLoader := plugin.NewLoader()
	pluginLoader.Add(
		plugin.NewRateLimit(params.Storage, params.ProxyParams.StatsClient),
		plugin.NewCORS(),
		plugin.NewOAuth2(params.OAuthRepo, params.Storage),
		plugin.NewCompression(),
		plugin.NewRequestTransformer(),
	)

	// create proxy register
	register := proxy.NewRegister(params.Router, params.ProxyParams)

	apiLoader := NewAPILoader(register, pluginLoader)
	apiLoader.LoadDefinitions(params.APIRepo)

	oauthLoader := NewOAuthLoader(register, params.Storage)
	oauthLoader.LoadDefinitions(params.OAuthRepo)

	// some routers may panic when have empty routes list, so add one dummy 404 route to avoid this
	if params.Router.RoutesCount() < 1 {
		params.Router.Any("/", web.NotFound)
	}
}
<commit_msg>Updated how plugins are initialized<commit_after>package loader

import (
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
	"github.com/hellofresh/janus/pkg/web"
	stats "github.com/hellofresh/stats-go"

	// this is needed to call the init function on each plugin
	_ "github.com/hellofresh/janus/pkg/plugin/compression"
	_ "github.com/hellofresh/janus/pkg/plugin/cors"
	_ "github.com/hellofresh/janus/pkg/plugin/oauth2"
	_ "github.com/hellofresh/janus/pkg/plugin/rate"
	_ "github.com/hellofresh/janus/pkg/plugin/requesttransformer"
)

// Params initialization options.
type Params struct {
	Router      router.Router
	Storage     store.Store
	APIRepo     api.Repository
	OAuthRepo   oauth.Repository
	StatsClient stats.Client
	ProxyParams proxy.Params
}

// Load loads all the basic components and definitions into a router
func Load(params Params) {
	// create proxy register
	register := proxy.NewRegister(params.Router, params.ProxyParams)

	apiLoader := NewAPILoader(register, plugin.Params{
		Router:    params.Router,
		Storage:   params.Storage,
		APIRepo:   params.APIRepo,
		OAuthRepo: params.OAuthRepo,
	})
	apiLoader.LoadDefinitions(params.APIRepo)

	oauthLoader := NewOAuthLoader(register, params.Storage)
	oauthLoader.LoadDefinitions(params.OAuthRepo)

	// some routers may panic when have empty routes list, so add one dummy 404 route to avoid this
	if params.Router.RoutesCount() < 1 {
		params.Router.Any("/", web.NotFound)
	}
}
<|endoftext|>