<commit_before>// Copyright 2016 The Minimal Configuration Manager Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shlib_test

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/zombiezen/mcm/catalog"
	"github.com/zombiezen/mcm/internal/applytests"
	"github.com/zombiezen/mcm/internal/catpogs"
	"github.com/zombiezen/mcm/internal/system"
	"github.com/zombiezen/mcm/shellify/shlib"
)

var keepScripts = flag.Bool("keep_scripts", false, "do not remove generated scripts from temporary directory")

const tmpDirEnv = "TEST_TMPDIR"

func TestIntegration(t *testing.T) {
	bashPath, err := exec.LookPath("bash")
	if err != nil {
		t.Skipf("Can't find bash: %v", err)
	}
	t.Logf("using %s for bash", bashPath)
	applytests.Run(t, (&fixtureFactory{bashPath: bashPath}).newFixture)
}

func TestExecBash(t *testing.T) {
	bashPath, err := exec.LookPath("bash")
	if err != nil {
		t.Skipf("Can't find bash: %v", err)
	}
	t.Logf("using %s for bash", bashPath)

	ctx, cancel := context.WithCancel(context.Background())
	f, err := (&fixtureFactory{bashPath: bashPath}).newFixture(ctx, t, "execbash")
	if err != nil {
		cancel()
		t.Fatal("fixture:", err)
	}
	defer func() {
		cancel()
		if err := f.Close(); err != nil {
			t.Error("fixture close:", err)
		}
	}()

	info := f.SystemInfo()
	fpath := filepath.Join(info.Root, "canary")
	c, err := (&catpogs.Catalog{
		Resources: []*catpogs.Resource{
			{
				ID:      42,
				Comment: "exec",
				Which:   catalog.Resource_Which_exec,
				Exec: &catpogs.Exec{
					Command: &catpogs.Command{
						Which: catalog.Exec_Command_Which_bash,
						Bash:  info.TouchPath + " '" + fpath + "'\n",
					},
				},
			},
		},
	}).ToCapnp()
	if err != nil {
		t.Fatalf("build catalog: %v", err)
	}
	err = f.Apply(ctx, c)
	if err != nil {
		t.Errorf("run catalog: %v", err)
	}
	if _, err := os.Lstat(fpath); err != nil {
		t.Errorf("checking for %q: %v", fpath, err)
	}
}

type fixtureFactory struct {
	bashPath string
}

func (ff *fixtureFactory) newFixture(ctx context.Context, log applytests.Logger, name string) (applytests.Fixture, error) {
	f := &fixture{
		name:     name,
		log:      log,
		bashPath: ff.bashPath,
	}
	var err error
	f.root, err = ioutil.TempDir(os.Getenv(tmpDirEnv), "shlib_testdir")
	if err != nil {
		return nil, err
	}
	return f, nil
}

type fixture struct {
	name     string
	log      applytests.Logger
	bashPath string

	root string
}

func (f *fixture) System() system.System {
	return system.Local{}
}

func (f *fixture) SystemInfo() *applytests.SystemInfo {
	return &applytests.SystemInfo{
		Root:      f.root,
		TruePath:  "/bin/true",
		FalsePath: "/bin/false",
		TouchPath: "/usr/bin/touch",
	}
}

func (f *fixture) Apply(ctx context.Context, c catalog.Catalog) error {
	sc, err := ioutil.TempFile(os.Getenv(tmpDirEnv), "shlib_testscript_"+f.name)
	if err != nil {
		return err
	}
	scriptPath := sc.Name()
	if !*keepScripts {
		defer func() {
			if err := os.Remove(scriptPath); err != nil {
				f.log.Logf("removing temporary script file: %v", err)
			}
		}()
	}
	err = shlib.WriteScript(sc, c)
	cerr := sc.Close()
	if err != nil {
		return err
	}
	if cerr != nil {
		return cerr
	}
	f.log.Logf("%s -- %s", f.bashPath, scriptPath)
	cmd := exec.Command(f.bashPath, []string{"--", scriptPath}...)
	stderr := new(bytes.Buffer)
	cmd.Stderr = stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("bash failed: %v; stderr:\n%s", err, stderr.Bytes())
	}
	return nil
}

func (f *fixture) Close() error {
	if err := os.RemoveAll(f.root); err != nil {
		return fmt.Errorf("removing temporary directory: %v", err)
	}
	return nil
}
<commit_msg>shellify: search PATH for true/false/touch in integration tests<commit_after>// Copyright 2016 The Minimal Configuration Manager Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shlib_test

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/zombiezen/mcm/catalog"
	"github.com/zombiezen/mcm/internal/applytests"
	"github.com/zombiezen/mcm/internal/catpogs"
	"github.com/zombiezen/mcm/internal/system"
	"github.com/zombiezen/mcm/shellify/shlib"
)

var keepScripts = flag.Bool("keep_scripts", false, "do not remove generated scripts from temporary directory")

const tmpDirEnv = "TEST_TMPDIR"

func TestIntegration(t *testing.T) {
	bashPath, err := exec.LookPath("bash")
	if err != nil {
		t.Skipf("Can't find bash: %v", err)
	}
	t.Logf("using %s for bash", bashPath)
	u, err := findSysutils()
	if err != nil {
		t.Fatal(err)
	}
	ff := &fixtureFactory{bashPath: bashPath, sysutils: u}
	applytests.Run(t, ff.newFixture)
}

func TestExecBash(t *testing.T) {
	bashPath, err := exec.LookPath("bash")
	if err != nil {
		t.Skipf("Can't find bash: %v", err)
	}
	t.Logf("using %s for bash", bashPath)
	u, err := findSysutils()
	if err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	f, err := (&fixtureFactory{bashPath: bashPath, sysutils: u}).newFixture(ctx, t, "execbash")
	if err != nil {
		cancel()
		t.Fatal("fixture:", err)
	}
	defer func() {
		cancel()
		if err := f.Close(); err != nil {
			t.Error("fixture close:", err)
		}
	}()

	info := f.SystemInfo()
	fpath := filepath.Join(info.Root, "canary")
	c, err := (&catpogs.Catalog{
		Resources: []*catpogs.Resource{
			{
				ID:      42,
				Comment: "exec",
				Which:   catalog.Resource_Which_exec,
				Exec: &catpogs.Exec{
					Command: &catpogs.Command{
						Which: catalog.Exec_Command_Which_bash,
						Bash:  info.TouchPath + " '" + fpath + "'\n",
					},
				},
			},
		},
	}).ToCapnp()
	if err != nil {
		t.Fatalf("build catalog: %v", err)
	}
	err = f.Apply(ctx, c)
	if err != nil {
		t.Errorf("run catalog: %v", err)
	}
	if _, err := os.Lstat(fpath); err != nil {
		t.Errorf("checking for %q: %v", fpath, err)
	}
}

type fixtureFactory struct {
	bashPath string
	*sysutils
}

type sysutils struct {
	truePath  string
	falsePath string
	touchPath string
}

func findSysutils() (*sysutils, error) {
	u := new(sysutils)
	var err error
	u.truePath, err = exec.LookPath("true")
	if err != nil {
		return nil, fmt.Errorf("can't find true: %v", err)
	}
	u.falsePath, err = exec.LookPath("false")
	if err != nil {
		return nil, fmt.Errorf("can't find false: %v", err)
	}
	u.touchPath, err = exec.LookPath("touch")
	if err != nil {
		return nil, fmt.Errorf("can't find touch: %v", err)
	}
	return u, nil
}

func (ff *fixtureFactory) newFixture(ctx context.Context, log applytests.Logger, name string) (applytests.Fixture, error) {
	f := &fixture{
		name:     name,
		log:      log,
		bashPath: ff.bashPath,
		sysutils: ff.sysutils,
	}
	var err error
	f.root, err = ioutil.TempDir(os.Getenv(tmpDirEnv), "shlib_testdir")
	if err != nil {
		return nil, err
	}
	return f, nil
}

type fixture struct {
	name     string
	log      applytests.Logger
	bashPath string
	*sysutils

	root string
}

func (f *fixture) System() system.System {
	return system.Local{}
}

func (f *fixture) SystemInfo() *applytests.SystemInfo {
	return &applytests.SystemInfo{
		Root:      f.root,
		TruePath:  f.truePath,
		FalsePath: f.falsePath,
		TouchPath: f.touchPath,
	}
}

func (f *fixture) Apply(ctx context.Context, c catalog.Catalog) error {
	sc, err := ioutil.TempFile(os.Getenv(tmpDirEnv), "shlib_testscript_"+f.name)
	if err != nil {
		return err
	}
	scriptPath := sc.Name()
	if !*keepScripts {
		defer func() {
			if err := os.Remove(scriptPath); err != nil {
				f.log.Logf("removing temporary script file: %v", err)
			}
		}()
	}
	err = shlib.WriteScript(sc, c)
	cerr := sc.Close()
	if err != nil {
		return err
	}
	if cerr != nil {
		return cerr
	}
	f.log.Logf("%s -- %s", f.bashPath, scriptPath)
	cmd := exec.Command(f.bashPath, []string{"--", scriptPath}...)
	stderr := new(bytes.Buffer)
	cmd.Stderr = stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("bash failed: %v; stderr:\n%s", err, stderr.Bytes())
	}
	return nil
}

func (f *fixture) Close() error {
	if err := os.RemoveAll(f.root); err != nil {
		return fmt.Errorf("removing temporary directory: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package service

import (
	"net/http"
	"sync"
	"time"

	commonModel "github.com/Cepave/open-falcon-backend/common/model"
	commonQueue "github.com/Cepave/open-falcon-backend/common/queue"
	commonSling "github.com/Cepave/open-falcon-backend/common/sling"
	"github.com/Cepave/open-falcon-backend/modules/hbs/cache"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	"github.com/dghubble/sling"
)

type AgentHeartbeatService struct {
	sync.WaitGroup
	safeQ            *commonQueue.Queue
	qConfig          *commonQueue.Config
	started          bool
	slingInit        *sling.Sling
	rowsAffectedCnt  int64
	agentsDroppedCnt int64
}

func NewAgentHeartbeatService(config *commonQueue.Config) *AgentHeartbeatService {
	return &AgentHeartbeatService{
		qConfig:   config,
		slingInit: NewSlingBase().Post("api/v1/agent/heartbeat"),
	}
}

func (s *AgentHeartbeatService) Start() {
	if s.started {
		return
	}
	s.started = true
	s.safeQ = commonQueue.New()

	s.Add(1)
	go func() {
		defer s.Done()

		for {
			if !s.started {
				break
			}

			s.consumeHeartbeatQueue(100*time.Millisecond, false)

			if !s.started {
				break
			}

			time.Sleep(5 * time.Second)
		}

		s.consumeHeartbeatQueue(0, true)
	}()
}

func (s *AgentHeartbeatService) consumeHeartbeatQueue(waitForQueue time.Duration, logFlag bool) {
	for {
		/*
		 * ToDo
		 * Configuration
		 * ToReview
		 */
		c := commonQueue.Config{}
		var elementType *model.AgentHeartbeat
		absArray := s.safeQ.DrainNWithDurationByType(&c, elementType)
		agents := absArray.([]*model.AgentHeartbeat)
		agentsNum := len(agents)
		if agentsNum == 0 {
			break
		}

		s.heartbeat(agents)
		if logFlag {
			logger.Infof("Flushing [%d] agents", agentsNum)
		}
	}
}

func (s *AgentHeartbeatService) Stop() {
	if !s.started {
		return
	}

	s.started = false
	logger.Infof("Stopping AgentHeartbeatService. Size of queue: [%d]", s.CurrentSize())

	/**
	 * Waiting for queue to be processed
	 */
	s.Wait()
	s.safeQ = nil
}

func (s *AgentHeartbeatService) Put(req *commonModel.AgentReportRequest) {
	if !s.started {
		return
	}
	now := time.Now().Unix()
	cache.Agents.Put(req, now)
	agent := &model.AgentHeartbeat{
		Hostname:      req.Hostname,
		IP:            req.IP,
		AgentVersion:  req.AgentVersion,
		PluginVersion: req.PluginVersion,
		UpdateTime:    now,
	}
	s.safeQ.Enqueue(agent)
}

func (s *AgentHeartbeatService) CurrentSize() int {
	return s.safeQ.Len()
}

func (s *AgentHeartbeatService) CumulativeAgentsDropped() int64 {
	return s.agentsDroppedCnt
}

func (s *AgentHeartbeatService) CumulativeRowsAffected() int64 {
	return s.rowsAffectedCnt
}

func (s *AgentHeartbeatService) heartbeat(agents []*model.AgentHeartbeat) {
	param := struct {
		UpdateOnly bool `json:"update_only"`
	}{updateOnlyFlag}
	s.slingInit = s.slingInit.BodyJSON(agents).QueryStruct(&param)

	res := model.AgentHeartbeatResult{}
	err := commonSling.ToSlintExt(s.slingInit).DoReceive(http.StatusOK, &res)
	if err != nil {
		s.agentsDroppedCnt += int64(len(agents))
		logger.Errorln("Heartbeat:", err)
		return
	}

	s.rowsAffectedCnt += res.RowsAffected
}
<commit_msg>[OWL-1695] Assign correct config of queue.<commit_after>package service

import (
	"net/http"
	"sync"
	"time"

	commonModel "github.com/Cepave/open-falcon-backend/common/model"
	commonQueue "github.com/Cepave/open-falcon-backend/common/queue"
	commonSling "github.com/Cepave/open-falcon-backend/common/sling"
	"github.com/Cepave/open-falcon-backend/modules/hbs/cache"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	"github.com/dghubble/sling"
)

type AgentHeartbeatService struct {
	sync.WaitGroup
	safeQ            *commonQueue.Queue
	qConfig          *commonQueue.Config
	started          bool
	slingInit        *sling.Sling
	rowsAffectedCnt  int64
	agentsDroppedCnt int64
}

func NewAgentHeartbeatService(config *commonQueue.Config) *AgentHeartbeatService {
	return &AgentHeartbeatService{
		qConfig:   config,
		slingInit: NewSlingBase().Post("api/v1/agent/heartbeat"),
	}
}

func (s *AgentHeartbeatService) Start() {
	if s.started {
		return
	}
	s.started = true
	s.safeQ = commonQueue.New()

	s.Add(1)
	go func() {
		defer s.Done()

		for {
			if !s.started {
				break
			}

			s.consumeHeartbeatQueue(100*time.Millisecond, false)

			if !s.started {
				break
			}

			time.Sleep(5 * time.Second)
		}

		s.consumeHeartbeatQueue(0, true)
	}()
}

func (s *AgentHeartbeatService) consumeHeartbeatQueue(waitForQueue time.Duration, logFlag bool) {
	for {
		var elementType *model.AgentHeartbeat
		absArray := s.safeQ.DrainNWithDurationByType(s.qConfig, elementType)
		agents := absArray.([]*model.AgentHeartbeat)
		agentsNum := len(agents)
		if agentsNum == 0 {
			break
		}

		s.heartbeat(agents)
		if logFlag {
			logger.Infof("Flushing [%d] agents", agentsNum)
		}
	}
}

func (s *AgentHeartbeatService) Stop() {
	if !s.started {
		return
	}

	s.started = false
	logger.Infof("Stopping AgentHeartbeatService. Size of queue: [%d]", s.CurrentSize())

	/**
	 * Waiting for queue to be processed
	 */
	s.Wait()
	s.safeQ = nil
}

func (s *AgentHeartbeatService) Put(req *commonModel.AgentReportRequest) {
	if !s.started {
		return
	}
	now := time.Now().Unix()
	cache.Agents.Put(req, now)
	agent := &model.AgentHeartbeat{
		Hostname:      req.Hostname,
		IP:            req.IP,
		AgentVersion:  req.AgentVersion,
		PluginVersion: req.PluginVersion,
		UpdateTime:    now,
	}
	s.safeQ.Enqueue(agent)
}

func (s *AgentHeartbeatService) CurrentSize() int {
	return s.safeQ.Len()
}

func (s *AgentHeartbeatService) CumulativeAgentsDropped() int64 {
	return s.agentsDroppedCnt
}

func (s *AgentHeartbeatService) CumulativeRowsAffected() int64 {
	return s.rowsAffectedCnt
}

func (s *AgentHeartbeatService) heartbeat(agents []*model.AgentHeartbeat) {
	param := struct {
		UpdateOnly bool `json:"update_only"`
	}{updateOnlyFlag}
	s.slingInit = s.slingInit.BodyJSON(agents).QueryStruct(&param)

	res := model.AgentHeartbeatResult{}
	err := commonSling.ToSlintExt(s.slingInit).DoReceive(http.StatusOK, &res)
	if err != nil {
		s.agentsDroppedCnt += int64(len(agents))
		logger.Errorln("Heartbeat:", err)
		return
	}

	s.rowsAffectedCnt += res.RowsAffected
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package envs

import (
	"encoding/json"
	"net/http"
	"strings"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Update : responds to PUT /projects/:project:/envs/:env/ by updating an
// existing environment
func Update(au models.User, name string, body []byte) (int, []byte) {
	var err error
	var resp []byte
	var e models.Env
	var input models.Env
	var p models.Project
	var r models.Role
	var roles []models.Role
	var pRoles []models.Role

	computedRoles := make(map[string]models.Role, 0)

	if input.Map(body) != nil {
		return 400, models.NewJSONError("Input is not valid")
	}

	if input.Name != name {
		return 400, models.NewJSONError("Environment name does not match payload name")
	}

	if err = json.Unmarshal(body, &input); err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, []byte(err.Error())
	}

	// Get existing environment
	if err = e.FindByName(name); err != nil {
		return 404, models.NewJSONError(err.Error())
	}

	if err = p.FindByID(e.ProjectID); err != nil {
		if strings.Contains(err.Error(), "not found") {
			return 404, models.NewJSONError("Specified environment name does not exist")
		}
		h.L.Error(err.Error())
		return 500, models.NewJSONError("Internal error")
	}

	if err = r.FindAllByResource(e.Project, p.GetType(), &pRoles); err == nil {
		for _, v := range pRoles {
			computedRoles[v.UserID] = v
		}
	}
	if err = r.FindAllByResource(e.GetID(), e.GetType(), &roles); err == nil {
		for _, v := range roles {
			computedRoles[v.UserID] = v
		}
	}

	for _, v := range computedRoles {
		e.Members = append(e.Members, v)
	}

	if st, res := h.IsAuthorizedToResource(&au, h.UpdateEnv, input.GetType(), name); st != 200 {
		return st, res
	}

	e.Options = input.Options
	e.Schedules = input.Schedules
	e.Credentials = input.Credentials

	if err = e.Save(); err != nil {
		return 500, models.NewJSONError(err.Error())
	}

	if input.Members == nil {
		resp, err = json.Marshal(e)
		if err != nil {
			h.L.Error(err.Error())
			return http.StatusBadRequest, models.NewJSONError(err.Error())
		}

		return http.StatusOK, resp
	}

	for _, ir := range input.Members {
		// create role
		if ir.ID == 0 {
			if !strings.Contains(ir.ResourceID, "/") || ir.ResourceType != "environment" {
				return http.StatusBadRequest, models.NewJSONError("project memberships must be modified on the project")
			}

			if !au.IsAdmin() {
				if ok := au.IsOwner(ir.ResourceType, ir.ResourceID); !ok {
					return 403, models.NewJSONError("You're not authorized to perform this action")
				}
			}

			err = ir.Save()
			if err != nil {
				h.L.Error(err.Error())
				return http.StatusBadRequest, models.NewJSONError(err.Error())
			}

			continue
		}

		for _, er := range e.Members {
			// update role
			if ir.ID == er.ID && ir.Role != er.Role {
				if !strings.Contains(er.ResourceID, "/") || ir.ResourceType != "environment" {
					return http.StatusBadRequest, models.NewJSONError("project memberships must be modified on the project")
				}

				if !au.IsAdmin() {
					if ok := au.IsOwner(ir.ResourceType, ir.ResourceID); !ok {
						return 403, models.NewJSONError("You're not authorized to perform this action")
					}
				}

				err = ir.Save()
				if err != nil {
					h.L.Error(err.Error())
					return http.StatusBadRequest, models.NewJSONError(err.Error())
				}
			}
		}
	}

	for _, er := range e.Members {
		var exists bool

		for _, ir := range input.Members {
			if ir.ID == er.ID {
				exists = true
			}
		}

		// delete roles
		if !exists {
			if !strings.Contains(er.ResourceID, "/") || er.ResourceType != "environment" {
				return http.StatusBadRequest, models.NewJSONError("project memberships must be removed on the project")
			}

			if !au.IsAdmin() {
				if ok := au.IsOwner(er.ResourceType, er.ResourceID); !ok {
					return 403, models.NewJSONError("You're not authorized to perform this action")
				}
			}

			err = er.Delete()
			if err != nil {
				h.L.Error(err.Error())
				return http.StatusBadRequest, models.NewJSONError(err.Error())
			}
		}
	}

	e.Members = input.Members

	resp, err = json.Marshal(e)
	if err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, models.NewJSONError(err.Error())
	}

	return http.StatusOK, resp
}
<commit_msg>additional validation<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package envs

import (
	"encoding/json"
	"net/http"
	"strings"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Update : responds to PUT /projects/:project:/envs/:env/ by updating an
// existing environment
func Update(au models.User, name string, body []byte) (int, []byte) {
	var err error
	var resp []byte
	var e models.Env
	var input models.Env
	var p models.Project
	var r models.Role
	var roles []models.Role
	var pRoles []models.Role

	computedRoles := make(map[string]models.Role, 0)

	if input.Map(body) != nil {
		return 400, models.NewJSONError("Input is not valid")
	}

	err = e.Validate()
	if err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, models.NewJSONError(err.Error())
	}

	if input.Name != name {
		return 400, models.NewJSONError("Environment name does not match payload name")
	}

	// Get existing environment
	if err = e.FindByName(name); err != nil {
		return 404, models.NewJSONError(err.Error())
	}

	if err = p.FindByID(e.ProjectID); err != nil {
		if strings.Contains(err.Error(), "not found") {
			return 404, models.NewJSONError("Specified environment name does not exist")
		}
		h.L.Error(err.Error())
		return 500, models.NewJSONError("Internal error")
	}

	if err = r.FindAllByResource(e.Project, p.GetType(), &pRoles); err == nil {
		for _, v := range pRoles {
			computedRoles[v.UserID] = v
		}
	}
	if err = r.FindAllByResource(e.GetID(), e.GetType(), &roles); err == nil {
		for _, v := range roles {
			computedRoles[v.UserID] = v
		}
	}

	for _, v := range computedRoles {
		e.Members = append(e.Members, v)
	}

	if st, res := h.IsAuthorizedToResource(&au, h.UpdateEnv, input.GetType(), name); st != 200 {
		return st, res
	}

	e.Options = input.Options
	e.Schedules = input.Schedules
	e.Credentials = input.Credentials

	if err = e.Save(); err != nil {
		return 500, models.NewJSONError(err.Error())
	}

	if input.Members == nil {
		resp, err = json.Marshal(e)
		if err != nil {
			h.L.Error(err.Error())
			return http.StatusBadRequest, models.NewJSONError(err.Error())
		}

		return http.StatusOK, resp
	}

	for _, ir := range input.Members {
		// create role
		if ir.ID == 0 {
			if !strings.Contains(ir.ResourceID, "/") || ir.ResourceType != "environment" {
				return http.StatusBadRequest, models.NewJSONError("project memberships must be modified on the project")
			}

			if !au.IsAdmin() {
				if ok := au.IsOwner(ir.ResourceType, ir.ResourceID); !ok {
					return 403, models.NewJSONError("You're not authorized to perform this action")
				}
			}

			err = ir.Save()
			if err != nil {
				h.L.Error(err.Error())
				return http.StatusBadRequest, models.NewJSONError(err.Error())
			}

			continue
		}

		for _, er := range e.Members {
			// update role
			if ir.ID == er.ID && ir.Role != er.Role {
				if !strings.Contains(er.ResourceID, "/") || ir.ResourceType != "environment" {
					return http.StatusBadRequest, models.NewJSONError("project memberships must be modified on the project")
				}

				if !au.IsAdmin() {
					if ok := au.IsOwner(ir.ResourceType, ir.ResourceID); !ok {
						return 403, models.NewJSONError("You're not authorized to perform this action")
					}
				}

				err = ir.Save()
				if err != nil {
					h.L.Error(err.Error())
					return http.StatusBadRequest, models.NewJSONError(err.Error())
				}
			}
		}
	}

	for _, er := range e.Members {
		var exists bool

		for _, ir := range input.Members {
			if ir.ID == er.ID {
				exists = true
			}
		}

		// delete roles
		if !exists {
			if !strings.Contains(er.ResourceID, "/") || er.ResourceType != "environment" {
				return http.StatusBadRequest, models.NewJSONError("project memberships must be removed on the project")
			}

			if !au.IsAdmin() {
				if ok := au.IsOwner(er.ResourceType, er.ResourceID); !ok {
					return 403, models.NewJSONError("You're not authorized to perform this action")
				}
			}

			err = er.Delete()
			if err != nil {
				h.L.Error(err.Error())
				return http.StatusBadRequest, models.NewJSONError(err.Error())
			}
		}
	}

	e.Members = input.Members

	resp, err = json.Marshal(e)
	if err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, models.NewJSONError(err.Error())
	}

	return http.StatusOK, resp
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The suffixarray package implements substring search in logarithmic time
// using an in-memory suffix array.
//
// Example use:
//
//	// create index for some data
//	index := suffixarray.New(data)
//
//	// lookup byte slice s
//	offsets1 := index.Lookup(s, -1) // the list of all indices where s occurs in data
//	offsets2 := index.Lookup(s, 3)  // the list of at most 3 indices where s occurs in data
//
package suffixarray

import (
	"bytes"
	"container/vector"
	"sort"
)

// BUG(gri): For larger data (10MB) which contains very long (say 100000)
// contiguous sequences of identical bytes, index creation time will be extremely slow.

// TODO(gri): Use a more sophisticated algorithm to create the suffix array.

// Index implements a suffix array for fast substring search.
type Index struct {
	data []byte
	sa   []int // suffix array for data
}

// New creates a new Index for data.
// Index creation time is approximately O(N*log(N)) for N = len(data).
//
func New(data []byte) *Index {
	sa := make([]int, len(data))
	for i, _ := range sa {
		sa[i] = i
	}
	x := &Index{data, sa}
	sort.Sort((*index)(x))
	return x
}

func (x *Index) at(i int) []byte {
	return x.data[x.sa[i]:]
}

// Binary search according to "A Method of Programming", E.W. Dijkstra.
func (x *Index) search(s []byte) int {
	i, j := 0, len(x.sa)
	// i < j for non-empty x
	for i+1 < j {
		// 0 <= i < j <= len(x.sa) && (x.at(i) <= s < x.at(j) || (s is not in x))
		h := i + (j-i)/2 // i < h < j
		if bytes.Compare(x.at(h), s) <= 0 {
			i = h
		} else { // s < x.at(h)
			j = h
		}
	}
	// i+1 == j for non-empty x
	return i
}

// Lookup returns an unsorted list of at most n indices where the byte string s
// occurs in the indexed data. If n < 0, all occurrences are returned.
// The result is nil if s is empty, s is not found, or n == 0.
// Lookup time is O((log(N) + len(result))*len(s)) where N is the
// size of the indexed data.
//
func (x *Index) Lookup(s []byte, n int) []int {
	var res vector.IntVector

	if len(s) > 0 && n != 0 {
		// find matching suffix index i
		i := x.search(s)
		// x.at(i) <= s < x.at(i+1)

		// ignore the first suffix if it is < s
		if i < len(x.sa) && bytes.Compare(x.at(i), s) < 0 {
			i++
		}

		// collect the following suffixes with matching prefixes
		for (n < 0 || len(res) < n) && i < len(x.sa) && bytes.HasPrefix(x.at(i), s) {
			res.Push(x.sa[i])
			i++
		}
	}

	return res
}

// index is used to hide the sort.Interface
type index Index

func (x *index) Len() int           { return len(x.sa) }
func (x *index) Less(i, j int) bool { return bytes.Compare(x.at(i), x.at(j)) < 0 }
func (x *index) Swap(i, j int)      { x.sa[i], x.sa[j] = x.sa[j], x.sa[i] }
func (a *index) at(i int) []byte    { return a.data[a.sa[i]:] }
<commit_msg>index/suffixarray: use sort.Search<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The suffixarray package implements substring search in logarithmic time
// using an in-memory suffix array.
//
// Example use:
//
//	// create index for some data
//	index := suffixarray.New(data)
//
//	// lookup byte slice s
//	offsets1 := index.Lookup(s, -1) // the list of all indices where s occurs in data
//	offsets2 := index.Lookup(s, 3)  // the list of at most 3 indices where s occurs in data
//
package suffixarray

import (
	"bytes"
	"container/vector"
	"sort"
)

// BUG(gri): For larger data (10MB) which contains very long (say 100000)
// contiguous sequences of identical bytes, index creation time will be extremely slow.

// TODO(gri): Use a more sophisticated algorithm to create the suffix array.

// Index implements a suffix array for fast substring search.
type Index struct {
	data []byte
	sa   []int // suffix array for data
}

// New creates a new Index for data.
// Index creation time is approximately O(N*log(N)) for N = len(data).
//
func New(data []byte) *Index {
	sa := make([]int, len(data))
	for i, _ := range sa {
		sa[i] = i
	}
	x := &Index{data, sa}
	sort.Sort((*index)(x))
	return x
}

func (x *Index) at(i int) []byte {
	return x.data[x.sa[i]:]
}

func (x *Index) search(s []byte) int {
	return sort.Search(len(x.sa), func(i int) bool { return bytes.Compare(x.at(i), s) >= 0 })
}

// Lookup returns an unsorted list of at most n indices where the byte string s
// occurs in the indexed data. If n < 0, all occurrences are returned.
// The result is nil if s is empty, s is not found, or n == 0.
// Lookup time is O((log(N) + len(result))*len(s)) where N is the
// size of the indexed data.
//
func (x *Index) Lookup(s []byte, n int) []int {
	var res vector.IntVector

	if len(s) > 0 && n != 0 {
		// find matching suffix index i
		i := x.search(s)
		// x.at(i-1) < s <= x.at(i)

		// collect the following suffixes with matching prefixes
		for (n < 0 || len(res) < n) && i < len(x.sa) && bytes.HasPrefix(x.at(i), s) {
			res.Push(x.sa[i])
			i++
		}
	}

	return res
}

// index is used to hide the sort.Interface
type index Index

func (x *index) Len() int           { return len(x.sa) }
func (x *index) Less(i, j int) bool { return bytes.Compare(x.at(i), x.at(j)) < 0 }
func (x *index) Swap(i, j int)      { x.sa[i], x.sa[j] = x.sa[j], x.sa[i] }
func (a *index) at(i int) []byte    { return a.data[a.sa[i]:] }
<|endoftext|>


<commit_before>package config

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func init() {
	configDirectoryPath = "./tmp"
}

func TestDefaultDatabasePathIsSetWhenConfigIsEmpty(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	assert.NotEmpty(t, config.DatabasePath)
}

func TestDefaultIndexPathIsSetWhenConfigIsEmpty(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	assert.NotEmpty(t, config.IndexPath)
}

func TestCanSetDatabasePath(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	config.DatabasePath = "database-path-foo"
	err = config.WriteConfig()
	if err != nil {
		t.Fatal(err)
	}
	contents, err := ioutil.ReadFile(configFilePath())
	if err != nil {
		t.Fatal(err)
	}
	rmdirConfig()
	assert.True(t, strings.IndexAny(string(contents), "database-path-foo") >= 0)
}

func TestCanSetIndexPath(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	config.IndexPath = "index-path-foo"
	err = config.WriteConfig()
	if err != nil {
		t.Fatal(err)
	}
	contents, err := ioutil.ReadFile(configFilePath())
	if err != nil {
		t.Fatal(err)
	}
	rmdirConfig()
	assert.True(t, strings.IndexAny(string(contents), "index-path-foo") >= 0)
}

func TestGetServiceReturnsEmptyWhenServiceDoesNotExist(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}

	svcCfg := config.GetService("foo")
	assert.Equal(t, "", svcCfg.Token)
}

func TestReadConfigFileReadsFileWhenExists(t *testing.T) {
	rmdirConfig()

	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}

	config.DatabasePath = "foo"
	err = config.WriteConfig()
	if err != nil {
		t.Fatal(err)
	}

	cfg2, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	assert.Equal(t, "foo", cfg2.DatabasePath)

	rmdirConfig()
}

func mkdirConfig() {
	if err := os.MkdirAll(configDirectoryPath, 0700); err != nil {
		panic(err)
	}
}

func rmdirConfig() {
	if err := os.RemoveAll(configDirectoryPath); err != nil {
		panic(err)
	}
}
<commit_msg>Add test for malformed yaml file<commit_after>package config

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"fmt"
)

func init() {
	configDirectoryPath = "./tmp"
}

func TestDefaultDatabasePathIsSetWhenConfigIsEmpty(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	assert.NotEmpty(t, config.DatabasePath)
}

func TestDefaultIndexPathIsSetWhenConfigIsEmpty(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	assert.NotEmpty(t, config.IndexPath)
}

func TestCanSetDatabasePath(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	config.DatabasePath = "database-path-foo"
	err = config.WriteConfig()
	if err != nil {
		t.Fatal(err)
	}
	contents, err := ioutil.ReadFile(configFilePath())
	if err != nil {
		t.Fatal(err)
	}
	rmdirConfig()
	assert.True(t, strings.IndexAny(string(contents), "database-path-foo") >= 0)
}

func TestCanSetIndexPath(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	config.IndexPath = "index-path-foo"
	err = config.WriteConfig()
	if err != nil {
		t.Fatal(err)
	}
	contents, err := ioutil.ReadFile(configFilePath())
	if err != nil {
		t.Fatal(err)
	}
	rmdirConfig()
	assert.True(t, strings.IndexAny(string(contents), "index-path-foo") >= 0)
}

func TestGetServiceReturnsEmptyWhenServiceDoesNotExist(t *testing.T) {
	rmdirConfig()
	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}

	svcCfg := config.GetService("foo")
	assert.Equal(t, "", svcCfg.Token)
}

func TestReadConfigFileReadsFileWhenExists(t *testing.T) {
	rmdirConfig()

	config, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}

	config.DatabasePath = "foo"
	err = config.WriteConfig()
	if err != nil {
		t.Fatal(err)
	}

	cfg2, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}
	assert.Equal(t, "foo", cfg2.DatabasePath)

	rmdirConfig()
}

func TestReadConfigDoesNotPanicForMalformedConfigurationFile(t *testing.T) {
	rmdirConfig()
	mkdirConfig()

	contents := "{this is not a yaml file}"
	err := ioutil.WriteFile(fmt.Sprintf("%s/limo.yaml", configDirectoryPath), []byte(contents), 0700)
	if err != nil {
		t.Fatal(err)
	}

	cfg, err := ReadConfig()
	if err != nil {
		t.Fatal(err)
	}

	assert.False(t, cfg.DatabasePath == "")

	rmdirConfig()
}

func mkdirConfig() {
	if err := os.MkdirAll(configDirectoryPath, 0700); err != nil {
		panic(err)
	}
}

func rmdirConfig() {
	if err := os.RemoveAll(configDirectoryPath); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package packstream

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
)

// Decoder decodes
type Decoder struct {
	r io.Reader
}

// NewDecoder returns a new Decoder
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// Decode returns
func (decoder *Decoder) Decode() (interface{}, error) {
	r := bufio.NewReader(decoder.r)
	marker, err := r.ReadByte()
	if err != nil {
		return nil, err
	}
	// markerHighNibble := marker & 0xF0
	// markerLowNibble := marker & 0x0F

	switch marker {
	case Null:
		return nil, nil
	case False:
		return false, nil
	case True:
		return true, nil
	case Float64:
		return decodeFloat(r)

		// Ints
	case Int8:
		return decodeInt(r, 1)
	case Int16:
		return decodeInt(r, 2)
	case Int32:
		return decodeInt(r, 4)
	case Int64:
		return decodeInt(r, 8)

		// Bytes
	case Bytes8:
		return decodeBytes(r, 1)
	case Bytes16:
		return decodeBytes(r, 2)
	case Bytes32:
		return decodeBytes(r, 2)

		// Strings
	case String8:
		return decodeString(r, 1)
	case String16:
		return decodeString(r, 2)
	case String32:
		return decodeString(r, 4)

	case List8:
		return nil, nil
	case List16:
		return nil, nil
	case List32:
		return nil, nil
	case ListStream:
		return nil, nil
	case Map8:
		return nil, nil
	case Map16:
		return nil, nil
	case Map32:
		return nil, nil
	case MapStream:
		return nil, nil
	case Struct8:
		return nil, nil
	case Struct16:
		return nil, nil
	case EndOfStream:
		return nil, nil
	}
	return nil, fmt.Errorf("decoding error: unsupported type: %x", marker)
}

func decodeBytes(r io.Reader, size int) ([]byte, error) {
	b := make([]byte, size)
	n, err := r.Read(b[:])
	if err != nil {
		return b, err
	}
	if n != cap(b) {
		return b, fmt.Errorf("failed to read all bytes")
	}

	return b, nil
}

func decodeString(r io.Reader, size int) (string, error) {
	s := ""
	b, err := decodeBytes(r, size)
	if err != nil {
		return s, err
	}
	return string(b), nil
}

func decodeInt(r io.Reader, size int) (int, error) {
	i := 0
	b, err := decodeBytes(r, size)
	if err != nil {
		return i, err
	}

	buf := bytes.NewReader(b)
	err = binary.Read(buf, binary.BigEndian, &i)
	if err != nil {
		return i, err
	}
	return i, nil
}

func decodeFloat(r io.Reader) (float64, error) {
	f := 0.0
	b, err := decodeBytes(r, 8)
	if err != nil {
		return f, err
	}

	buf := bytes.NewReader(b)
	err = binary.Read(buf, binary.BigEndian, &f)
	if err != nil {
		return f, err
	}
	return f, nil
}

func (decoder *Decoder) peekNextType() Type {
	reader := bufio.NewReader(decoder.r)
	markerbytes, err := reader.Peek(1)
	if err != nil {
		return PSNull
	}
	marker := markerbytes[0]
	markerHighNibble := marker & 0xF0

	switch markerHighNibble {
	case TinyString:
		return PSString
	case TinyList:
		return PSList
	case TinyMap:
		return PSMap
	case TinyStruct:
		return PSStruct
	}

	switch marker {
	case Null:
		return PSNull
	case True:
	case False:
		return PSBool
	case Float64:
		return PSFloat
	case Bytes8:
	case Bytes16:
	case Bytes32:
		return PSBytes
	case String8:
	case String16:
	case String32:
		return PSString
	case List8:
	case List16:
	case List32:
		return PSList
	case Struct8:
	case Struct16:
		return PSStruct
	default:
		return PSInt
	}
	return PSNull
}
<commit_msg>correct camel case<commit_after>package packstream

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
)

// Decoder decodes
type Decoder struct {
	r io.Reader
}

// NewDecoder returns a new Decoder
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// Decode returns
func (decoder *Decoder) Decode() (interface{}, error) {
	r := bufio.NewReader(decoder.r)
	marker, err := r.ReadByte()
	if err != nil {
		return nil, err
	}
	// markerHighNibble := marker & 0xF0
	// markerLowNibble := marker & 0x0F

	switch marker {
	case Null:
		return nil, nil
	case False:
		return false, nil
	case True:
		return true, nil
	case Float64:
		return decodeFloat(r)

		// Ints
	case Int8:
		return decodeInt(r, 1)
	case Int16:
		return decodeInt(r, 2)
	case Int32:
		return decodeInt(r, 4)
	case Int64:
		return decodeInt(r, 8)

		// Bytes
	case Bytes8:
		return decodeBytes(r, 1)
	case Bytes16:
		return decodeBytes(r, 2)
	case Bytes32:
		return decodeBytes(r, 2)

		// Strings
	case String8:
		return decodeString(r, 1)
	case String16:
		return decodeString(r, 2)
	case String32:
		return decodeString(r, 4)

	case List8:
		return nil, nil
	case List16:
		return nil, nil
	case List32:
		return nil, nil
	case ListStream:
		return nil, nil
	case Map8:
		return nil, nil
	case Map16:
		return nil, nil
	case Map32:
		return nil, nil
	case MapStream:
		return nil, nil
	case Struct8:
		return nil, nil
	case Struct16:
		return nil, nil
	case EndOfStream:
		return nil, nil
	}
	return nil, fmt.Errorf("decoding error: unsupported type: %x", marker)
}

func decodeBytes(r io.Reader, size int) ([]byte, error) {
	b := make([]byte, size)
	n, err := r.Read(b[:])
	if err != nil {
		return b, err
	}
	if n != cap(b) {
		return b, fmt.Errorf("failed to read all bytes")
	}

	return b, nil
}

func decodeString(r io.Reader, size int) (string, error) {
	s := ""
	b, err := decodeBytes(r, size)
	if err != nil {
		return s, err
	}
	return string(b), nil
}

func decodeInt(r io.Reader, size int) (int, error) {
	i := 0
	b, err := decodeBytes(r, size)
	if err != nil {
		return i, err
	}

	buf := bytes.NewReader(b)
	err = binary.Read(buf, binary.BigEndian, &i)
	if err != nil {
		return i, err
	}
	return i, nil
}

func decodeFloat(r io.Reader) (float64, error) {
	f := 0.0
	b, err := decodeBytes(r, 8)
	if err != nil {
		return f, err
	}

	buf := bytes.NewReader(b)
	err = binary.Read(buf, binary.BigEndian, &f)
	if err != nil {
		return f, err
	}
	return f, nil
}

func (decoder *Decoder) peekNextType() Type {
	reader := bufio.NewReader(decoder.r)
	markerBytes, err := reader.Peek(1)
	if err != nil {
		return PSNull
	}
	marker := markerBytes[0]
	markerHighNibble := marker & 0xF0

	switch markerHighNibble {
	case TinyString:
		return PSString
	case TinyList:
		return PSList
	case TinyMap:
		return PSMap
	case TinyStruct:
		return PSStruct
	}

	switch marker {
	case Null:
		return PSNull
	case True:
	case False:
		return PSBool
	case Float64:
		return PSFloat
	case Bytes8:
	case Bytes16:
	case Bytes32:
		return PSBytes
	case String8:
	case String16:
	case String32:
		return PSString
	case List8:
	case List16:
	case List32:
		return PSList
	case Struct8:
	case Struct16:
		return PSStruct
	default:
		return PSInt
	}
	return PSNull
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flag

import (
	"crypto/tls"
	"fmt"

	"k8s.io/apimachinery/pkg/util/sets"
)

// ciphers maps strings into tls package cipher constants in
// https://golang.org/pkg/crypto/tls/#pkg-constants
// to be replaced by tls.CipherSuites() when the project migrates to go1.14.
var ciphers = map[string]uint16{
	"TLS_RSA_WITH_3DES_EDE_CBC_SHA":           tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
	"TLS_RSA_WITH_AES_128_CBC_SHA":            tls.TLS_RSA_WITH_AES_128_CBC_SHA,
	"TLS_RSA_WITH_AES_256_CBC_SHA":            tls.TLS_RSA_WITH_AES_256_CBC_SHA,
	"TLS_RSA_WITH_AES_128_GCM_SHA256":         tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
	"TLS_RSA_WITH_AES_256_GCM_SHA384":         tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
	"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA":     tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
	"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
	"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384":   tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
	"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
	"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305":    tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
	"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305":  tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
	"TLS_AES_128_GCM_SHA256":                  tls.TLS_AES_128_GCM_SHA256,
	"TLS_CHACHA20_POLY1305_SHA256":            tls.TLS_CHACHA20_POLY1305_SHA256,
	"TLS_AES_256_GCM_SHA384":                  tls.TLS_AES_256_GCM_SHA384,
}

// to be replaced by tls.InsecureCipherSuites() when the project migrates to go1.14.
var insecureCiphers = map[string]uint16{
	"TLS_RSA_WITH_RC4_128_SHA":                tls.TLS_RSA_WITH_RC4_128_SHA,
	"TLS_RSA_WITH_AES_128_CBC_SHA256":         tls.TLS_RSA_WITH_AES_128_CBC_SHA256,
	"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":        tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,
	"TLS_ECDHE_RSA_WITH_RC4_128_SHA":          tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
}

// InsecureTLSCiphers returns the cipher suites implemented by crypto/tls which have
// security issues.
func InsecureTLSCiphers() map[string]uint16 {
	cipherKeys := make(map[string]uint16, len(insecureCiphers))
	for k, v := range insecureCiphers {
		cipherKeys[k] = v
	}
	return cipherKeys
}

// InsecureTLSCipherNames returns a list of cipher suite names implemented by crypto/tls
// which have security issues.
func InsecureTLSCipherNames() []string {
	cipherKeys := sets.NewString()
	for key := range insecureCiphers {
		cipherKeys.Insert(key)
	}
	return cipherKeys.List()
}

// PreferredTLSCipherNames returns a list of cipher suite names implemented by crypto/tls.
func PreferredTLSCipherNames() []string {
	cipherKeys := sets.NewString()
	for key := range ciphers {
		cipherKeys.Insert(key)
	}
	return cipherKeys.List()
}

func allCiphers() map[string]uint16 {
	acceptedCiphers := make(map[string]uint16, len(ciphers)+len(insecureCiphers))
	for k, v := range ciphers {
		acceptedCiphers[k] = v
	}
	for k, v := range insecureCiphers {
		acceptedCiphers[k] = v
	}
	return acceptedCiphers
}

// TLSCipherPossibleValues returns all acceptable cipher suite names.
// This is a combination of both InsecureTLSCipherNames() and PreferredTLSCipherNames().
func TLSCipherPossibleValues() []string {
	cipherKeys := sets.NewString()
	acceptedCiphers := allCiphers()
	for key := range acceptedCiphers {
		cipherKeys.Insert(key)
	}
	return cipherKeys.List()
}

// TLSCipherSuites returns a list of cipher suite IDs from the cipher suite names passed.
func TLSCipherSuites(cipherNames []string) ([]uint16, error) {
	if len(cipherNames) == 0 {
		return nil, nil
	}
	ciphersIntSlice := make([]uint16, 0)
	possibleCiphers := allCiphers()
	for _, cipher := range cipherNames {
		intValue, ok := possibleCiphers[cipher]
		if !ok {
			return nil, fmt.Errorf("Cipher suite %s not supported or doesn't exist", cipher)
		}
		ciphersIntSlice = append(ciphersIntSlice, intValue)
	}
	return ciphersIntSlice, nil
}

var versions = map[string]uint16{
	"VersionTLS10": tls.VersionTLS10,
	"VersionTLS11": tls.VersionTLS11,
	"VersionTLS12": tls.VersionTLS12,
	"VersionTLS13": tls.VersionTLS13,
}

// TLSPossibleVersions returns all acceptable values for TLS Version.
func TLSPossibleVersions() []string {
	versionsKeys := sets.NewString()
	for key := range versions {
		versionsKeys.Insert(key)
	}
	return versionsKeys.List()
}

// TLSVersion returns the TLS Version ID for the version name passed.
func TLSVersion(versionName string) (uint16, error) {
	if len(versionName) == 0 {
		return DefaultTLSVersion(), nil
	}
	if version, ok := versions[versionName]; ok {
		return version, nil
	}
	return 0, fmt.Errorf("unknown tls version %q", versionName)
}

// DefaultTLSVersion defines the default TLS Version.
func DefaultTLSVersion() uint16 {
	// Can't use SSLv3 because of POODLE and BEAST
	// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
	// Can't use TLSv1.1 because of RC4 cipher usage
	return tls.VersionTLS12
}
<commit_msg>go1.14: add new ciphers<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flag

import (
	"crypto/tls"
	"fmt"

	"k8s.io/apimachinery/pkg/util/sets"
)

// ciphers maps strings into tls package cipher constants in
// https://golang.org/pkg/crypto/tls/#pkg-constants
// to be replaced by tls.CipherSuites() when the project migrates to go1.14.
var ciphers = map[string]uint16{
	"TLS_RSA_WITH_3DES_EDE_CBC_SHA":           tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
	"TLS_RSA_WITH_AES_128_CBC_SHA":            tls.TLS_RSA_WITH_AES_128_CBC_SHA,
	"TLS_RSA_WITH_AES_256_CBC_SHA":            tls.TLS_RSA_WITH_AES_256_CBC_SHA,
	"TLS_RSA_WITH_AES_128_GCM_SHA256":         tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
	"TLS_RSA_WITH_AES_256_GCM_SHA384":         tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
	"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA":     tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
	"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
	"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
	"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384":   tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
	"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
	"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305":    tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
	"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305":  tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
	"TLS_AES_128_GCM_SHA256":                  tls.TLS_AES_128_GCM_SHA256,
	"TLS_CHACHA20_POLY1305_SHA256":            tls.TLS_CHACHA20_POLY1305_SHA256,
	"TLS_AES_256_GCM_SHA384":                  tls.TLS_AES_256_GCM_SHA384,

	// support official IANA names as well
	"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
	"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256":   tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
}

// to be replaced by tls.InsecureCipherSuites() when the project migrates to go1.14.
var insecureCiphers = map[string]uint16{
	"TLS_RSA_WITH_RC4_128_SHA":                tls.TLS_RSA_WITH_RC4_128_SHA,
	"TLS_RSA_WITH_AES_128_CBC_SHA256":         tls.TLS_RSA_WITH_AES_128_CBC_SHA256,
	"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":        tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,
	"TLS_ECDHE_RSA_WITH_RC4_128_SHA":          tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
}

// InsecureTLSCiphers returns the cipher suites implemented by crypto/tls which have
// security issues.
func InsecureTLSCiphers() map[string]uint16 {
	cipherKeys := make(map[string]uint16, len(insecureCiphers))
	for k, v := range insecureCiphers {
		cipherKeys[k] = v
	}
	return cipherKeys
}

// InsecureTLSCipherNames returns a list of cipher suite names implemented by crypto/tls
// which have security issues.
func InsecureTLSCipherNames() []string {
	cipherKeys := sets.NewString()
	for key := range insecureCiphers {
		cipherKeys.Insert(key)
	}
	return cipherKeys.List()
}

// PreferredTLSCipherNames returns a list of cipher suite names implemented by crypto/tls.
func PreferredTLSCipherNames() []string {
	cipherKeys := sets.NewString()
	for key := range ciphers {
		cipherKeys.Insert(key)
	}
	return cipherKeys.List()
}

func allCiphers() map[string]uint16 {
	acceptedCiphers := make(map[string]uint16, len(ciphers)+len(insecureCiphers))
	for k, v := range ciphers {
		acceptedCiphers[k] = v
	}
	for k, v := range insecureCiphers {
		acceptedCiphers[k] = v
	}
	return acceptedCiphers
}

// TLSCipherPossibleValues returns all acceptable cipher suite names.
// This is a combination of both InsecureTLSCipherNames() and PreferredTLSCipherNames().
func TLSCipherPossibleValues() []string {
	cipherKeys := sets.NewString()
	acceptedCiphers := allCiphers()
	for key := range acceptedCiphers {
		cipherKeys.Insert(key)
	}
	return cipherKeys.List()
}

// TLSCipherSuites returns a list of cipher suite IDs from the cipher suite names passed.
func TLSCipherSuites(cipherNames []string) ([]uint16, error) {
	if len(cipherNames) == 0 {
		return nil, nil
	}
	ciphersIntSlice := make([]uint16, 0)
	possibleCiphers := allCiphers()
	for _, cipher := range cipherNames {
		intValue, ok := possibleCiphers[cipher]
		if !ok {
			return nil, fmt.Errorf("Cipher suite %s not supported or doesn't exist", cipher)
		}
		ciphersIntSlice = append(ciphersIntSlice, intValue)
	}
	return ciphersIntSlice, nil
}

var versions = map[string]uint16{
	"VersionTLS10": tls.VersionTLS10,
	"VersionTLS11": tls.VersionTLS11,
	"VersionTLS12": tls.VersionTLS12,
	"VersionTLS13": tls.VersionTLS13,
}

// TLSPossibleVersions returns all acceptable values for TLS Version.
func TLSPossibleVersions() []string {
	versionsKeys := sets.NewString()
	for key := range versions {
		versionsKeys.Insert(key)
	}
	return versionsKeys.List()
}

// TLSVersion returns the TLS Version ID for the version name passed.
func TLSVersion(versionName string) (uint16, error) {
	if len(versionName) == 0 {
		return DefaultTLSVersion(), nil
	}
	if version, ok := versions[versionName]; ok {
		return version, nil
	}
	return 0, fmt.Errorf("unknown tls version %q", versionName)
}

// DefaultTLSVersion defines the default TLS Version.
func DefaultTLSVersion() uint16 {
	// Can't use SSLv3 because of POODLE and BEAST
	// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
	// Can't use TLSv1.1 because of RC4 cipher usage
	return tls.VersionTLS12
}
<|endoftext|>


<commit_before>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package block_device_utils

import (
	"bufio"
	"github.com/IBM/ubiquity/utils/logs"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"fmt"
)

const multipathCmd = "multipath"

func (b *blockDeviceUtils) ReloadMultipath() error {
	defer b.logger.Trace(logs.DEBUG)()
	if err := b.exec.IsExecutable(multipathCmd); err != nil {
		return b.logger.ErrorRet(&commandNotFoundError{multipathCmd, err}, "failed")
	}
	args := []string{"-r"}
	if _, err := b.exec.Execute(multipathCmd, args); err != nil {
		return b.logger.ErrorRet(&commandExecuteError{multipathCmd, err}, "failed")
	}
	return nil
}

func (b *blockDeviceUtils) Discover(volumeWwn string) (string, error) {
	defer b.logger.Trace(logs.DEBUG)()
	if err := b.exec.IsExecutable(multipathCmd); err != nil {
		return "", b.logger.ErrorRet(&commandNotFoundError{multipathCmd, err}, "failed")
	}
	args := []string{"-ll"}
	outputBytes, err := b.exec.Execute(multipathCmd, args)
	if err != nil {
		return "", b.logger.ErrorRet(&commandExecuteError{multipathCmd, err}, "failed")
	}
	scanner := bufio.NewScanner(strings.NewReader(string(outputBytes[:])))
	pattern := "(?i)" + volumeWwn
	regex, err := regexp.Compile(pattern)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	dev := ""
	for scanner.Scan() {
		if regex.MatchString(scanner.Text()) {
			dev = strings.Split(scanner.Text(), " ")[0]
			break
		}
	}

	if dev == "" {
		dev, err = b.DiscoverBySgInq(string(outputBytes[:]), volumeWwn)
		if err != nil {
			return "", b.logger.ErrorRet(&volumeNotFoundError{volumeWwn}, "failed")
		} else {
			b.logger.Debug(fmt.Sprintf("WWN %s was found using sg_inq, the device is %s.", volumeWwn, dev))
		}
	}
	mpath := b.mpathDevFullPath(dev)
	if _, err = b.exec.Stat(mpath); err != nil {
		return "", b.logger.ErrorRet(err, "Stat failed")
	}
	b.logger.Info("discovered", logs.Args{{"volumeWwn", volumeWwn}, {"mpath", mpath}})
	return mpath, nil
}

func (b *blockDeviceUtils) mpathDevFullPath(dev string) (string) {
	return path.Join(string(filepath.Separator), "dev", "mapper", dev)
}

func (b *blockDeviceUtils) DiscoverBySgInq(mpathOutput string, volumeWwn string) (string, error) {
	defer b.logger.Trace(logs.DEBUG)()

	scanner := bufio.NewScanner(strings.NewReader(mpathOutput))
	pattern := "(?i)" + "^mpath"
	regex, err := regexp.Compile(pattern)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	dev := ""
	for scanner.Scan() {
		line := scanner.Text()
		b.logger.Debug(fmt.Sprintf("%s", line))
		if regex.MatchString(line) {
			dev = strings.Split(line, " ")[0]
			mpathFullPath := b.mpathDevFullPath(dev)
			wwn, err := b.GetWwnByScsiInq(mpathFullPath)
			if err != nil {
				return "", b.logger.ErrorRet(&volumeNotFoundError{volumeWwn}, "failed")
			}
			if strings.ToLower(wwn) == strings.ToLower(volumeWwn){
				return dev, nil
			}
		}
	}
	return "", b.logger.ErrorRet(&volumeNotFoundError{volumeWwn}, "failed")
}

func (b *blockDeviceUtils) GetWwnByScsiInq(dev string) (string, error) {
	defer b.logger.Trace(logs.DEBUG)()
	/* scsi inq example
		sg_inq -p 0x83 /dev/mapper/mpathhe
		VPD INQUIRY: Device Identification page
		  Designation descriptor number 1, descriptor length: 20
			designator_type: NAA,  code_set: Binary
			associated with the addressed logical unit
			  NAA 6, IEEE Company_id: 0x1738
			  Vendor Specific Identifier: 0xcfc9035eb
			  Vendor Specific Identifier Extension: 0xcea5f6
			  [0x6001738cfc9035eb0000000000ceaaaa]
		  Designation descriptor number 2, descriptor length: 52
			designator_type: T10 vendor identification,  code_set: ASCII
			associated with the addressed logical unit
			  vendor id: IBM
			  vendor specific: 2810XIV          60035EB0000000000CEAAAA
		  Designation descriptor number 3, descriptor length: 43
			designator_type: vendor specific [0x0],  code_set: ASCII
			associated with the addressed logical unit
			  vendor specific: vol=u_k8s_longevity_ibm-ubiquity-db
		  Designation descriptor number 4, descriptor length: 37
			designator_type: vendor specific [0x0],  code_set: ASCII
			associated with the addressed logical unit
			  vendor specific: host=k8s-acceptance-v18-node1
		  Designation descriptor number 5, descriptor length: 8
			designator_type: Target port group,  code_set: Binary
			associated with the target port
			  Target port group: 0x0
		  Designation descriptor number 6, descriptor length: 8
			designator_type: Relative target port,  code_set: Binary
			associated with the target port
			  Relative target port: 0xd22
	*/
	sgInqCmd := "sg_inq"

	args := []string{"-p",  "0x83", dev}
	outputBytes, err := b.exec.Execute(sgInqCmd, args)
	if err != nil {
		return "", b.logger.ErrorRet(&commandExecuteError{sgInqCmd, err}, "failed")
	}
	wwnRegex := `\[0x(.*?)\]`
	wwnRegexCompiled, err := regexp.Compile(wwnRegex)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	pattern := "(?i)" + "Vendor Specific Identifier Extension:"
	scanner := bufio.NewScanner(strings.NewReader(string(outputBytes[:])))
	regex, err := regexp.Compile(pattern)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	wwn := ""
	found := false
	for scanner.Scan() {
		line := scanner.Text()
		b.logger.Debug(fmt.Sprintf("%s", line))
		if found {
			matches := wwnRegexCompiled.FindStringSubmatch(line)
			if len(matches) != 2 {
				return "", b.logger.ErrorRet(&noRegexWwnMatchInScsiInqError{ dev, line }, "failed")
			}
			b.logger.Debug(fmt.Sprintf("%#v", matches))
			wwn = matches[1]
			return wwn, nil
		}
		if regex.MatchString(line) {
			found = true
			// next line is the line we need
			continue
		}

	}
	return "", b.logger.ErrorRet(&volumeNotFoundError{wwn}, "failed")
}

func (b *blockDeviceUtils) Cleanup(mpath string) error {
	defer b.logger.Trace(logs.DEBUG)()
	dev := path.Base(mpath)
	dmsetupCmd := "dmsetup"
	if err := b.exec.IsExecutable(dmsetupCmd); err != nil {
		return b.logger.ErrorRet(&commandNotFoundError{dmsetupCmd, err}, "failed")
	}
	args := []string{"message", dev, "0", "fail_if_no_path"}
	if _, err := b.exec.Execute(dmsetupCmd, args); err != nil {
		return b.logger.ErrorRet(&commandExecuteError{dmsetupCmd, err}, "failed")
	}
	if err := b.exec.IsExecutable(multipathCmd); err != nil {
		return b.logger.ErrorRet(&commandNotFoundError{multipathCmd, err}, "failed")
	}
	args = []string{"-f", dev}
	if _, err := b.exec.Execute(multipathCmd, args); err != nil {
		return b.logger.ErrorRet(&commandExecuteError{multipathCmd, err}, "failed")
	}
	b.logger.Info("flushed", logs.Args{{"mpath", mpath}})
	return nil
}
<commit_msg>Added dmsetup sleep<commit_after>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package block_device_utils

import (
	"bufio"
	"github.com/IBM/ubiquity/utils/logs"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"fmt"
	"time"
)

const multipathCmd = "multipath"

func (b *blockDeviceUtils) ReloadMultipath() error {
	defer b.logger.Trace(logs.DEBUG)()
	if err := b.exec.IsExecutable(multipathCmd); err != nil {
		return b.logger.ErrorRet(&commandNotFoundError{multipathCmd, err}, "failed")
	}
	args := []string{"-r"}
	if _, err := b.exec.Execute(multipathCmd, args); err != nil {
		return b.logger.ErrorRet(&commandExecuteError{multipathCmd, err}, "failed")
	}
	return nil
}

func (b *blockDeviceUtils) Discover(volumeWwn string) (string, error) {
	defer b.logger.Trace(logs.DEBUG)()
	if err := b.exec.IsExecutable(multipathCmd); err != nil {
		return "", b.logger.ErrorRet(&commandNotFoundError{multipathCmd, err}, "failed")
	}
	args := []string{"-ll"}
	outputBytes, err := b.exec.Execute(multipathCmd, args)
	if err != nil {
		return "", b.logger.ErrorRet(&commandExecuteError{multipathCmd, err}, "failed")
	}
	scanner := bufio.NewScanner(strings.NewReader(string(outputBytes[:])))
	pattern := "(?i)" + volumeWwn
	regex, err := regexp.Compile(pattern)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	dev := ""
	for scanner.Scan() {
		if regex.MatchString(scanner.Text()) {
			dev = strings.Split(scanner.Text(), " ")[0]
			break
		}
	}

	if dev == "" {
		dev, err = b.DiscoverBySgInq(string(outputBytes[:]), volumeWwn)
		if err != nil {
			return "", b.logger.ErrorRet(&volumeNotFoundError{volumeWwn}, "failed")
		} else {
			b.logger.Debug(fmt.Sprintf("WWN %s was found using sg_inq, the device is %s.", volumeWwn, dev))
		}
	}
	mpath := b.mpathDevFullPath(dev)
	if _, err = b.exec.Stat(mpath); err != nil {
		return "", b.logger.ErrorRet(err, "Stat failed")
	}
	b.logger.Info("discovered", logs.Args{{"volumeWwn", volumeWwn}, {"mpath", mpath}})
	return mpath, nil
}

func (b *blockDeviceUtils) mpathDevFullPath(dev string) (string) {
	return path.Join(string(filepath.Separator), "dev", "mapper", dev)
}

func (b *blockDeviceUtils) DiscoverBySgInq(mpathOutput string, volumeWwn string) (string, error) {
	defer b.logger.Trace(logs.DEBUG)()

	scanner := bufio.NewScanner(strings.NewReader(mpathOutput))
	pattern := "(?i)" + "^mpath"
	regex, err := regexp.Compile(pattern)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	dev := ""
	for scanner.Scan() {
		line := scanner.Text()
		b.logger.Debug(fmt.Sprintf("%s", line))
		if regex.MatchString(line) {
			dev = strings.Split(line, " ")[0]
			mpathFullPath := b.mpathDevFullPath(dev)
			wwn, err := b.GetWwnByScsiInq(mpathFullPath)
			if err != nil {
				return "", b.logger.ErrorRet(&volumeNotFoundError{volumeWwn}, "failed")
			}
			if strings.ToLower(wwn) == strings.ToLower(volumeWwn){
				return dev, nil
			}
		}
	}
	return "", b.logger.ErrorRet(&volumeNotFoundError{volumeWwn}, "failed")
}

func (b *blockDeviceUtils) GetWwnByScsiInq(dev string) (string, error) {
	defer b.logger.Trace(logs.DEBUG)()
	/* scsi inq example
		sg_inq -p 0x83 /dev/mapper/mpathhe
		VPD INQUIRY: Device Identification page
		  Designation descriptor number 1, descriptor length: 20
			designator_type: NAA,  code_set: Binary
			associated with the addressed logical unit
			  NAA 6, IEEE Company_id: 0x1738
			  Vendor Specific Identifier: 0xcfc9035eb
			  Vendor Specific Identifier Extension: 0xcea5f6
			  [0x6001738cfc9035eb0000000000ceaaaa]
		  Designation descriptor number 2, descriptor length: 52
			designator_type: T10 vendor identification,  code_set: ASCII
			associated with the addressed logical unit
			  vendor id: IBM
			  vendor specific: 2810XIV          60035EB0000000000CEAAAA
		  Designation descriptor number 3, descriptor length: 43
			designator_type: vendor specific [0x0],  code_set: ASCII
			associated with the addressed logical unit
			  vendor specific: vol=u_k8s_longevity_ibm-ubiquity-db
		  Designation descriptor number 4, descriptor length: 37
			designator_type: vendor specific [0x0],  code_set: ASCII
			associated with the addressed logical unit
			  vendor specific: host=k8s-acceptance-v18-node1
		  Designation descriptor number 5, descriptor length: 8
			designator_type: Target port group,  code_set: Binary
			associated with the target port
			  Target port group: 0x0
		  Designation descriptor number 6, descriptor length: 8
			designator_type: Relative target port,  code_set: Binary
			associated with the target port
			  Relative target port: 0xd22
	*/
	sgInqCmd := "sg_inq"

	args := []string{"-p",  "0x83", dev}
	outputBytes, err := b.exec.Execute(sgInqCmd, args)
	if err != nil {
		return "", b.logger.ErrorRet(&commandExecuteError{sgInqCmd, err}, "failed")
	}
	wwnRegex := `\[0x(.*?)\]`
	wwnRegexCompiled, err := regexp.Compile(wwnRegex)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	pattern := "(?i)" + "Vendor Specific Identifier Extension:"
	scanner := bufio.NewScanner(strings.NewReader(string(outputBytes[:])))
	regex, err := regexp.Compile(pattern)
	if err != nil {
		return "", b.logger.ErrorRet(err, "failed")
	}
	wwn := ""
	found := false
	for scanner.Scan() {
		line := scanner.Text()
		b.logger.Debug(fmt.Sprintf("%s", line))
		if found {
			matches := wwnRegexCompiled.FindStringSubmatch(line)
			if len(matches) != 2 {
				return "", b.logger.ErrorRet(&noRegexWwnMatchInScsiInqError{ dev, line }, "failed")
			}
			b.logger.Debug(fmt.Sprintf("%#v", matches))
			wwn = matches[1]
			return wwn, nil
		}
		if regex.MatchString(line) {
			found = true
			// next line is the line we need
			continue
		}

	}
	return "", b.logger.ErrorRet(&volumeNotFoundError{wwn}, "failed")
}

func (b *blockDeviceUtils) Cleanup(mpath string) error {
	defer b.logger.Trace(logs.DEBUG)()
	dev := path.Base(mpath)
	dmsetupCmd := "dmsetup"
	if err := b.exec.IsExecutable(dmsetupCmd); err != nil {
		return b.logger.ErrorRet(&commandNotFoundError{dmsetupCmd, err}, "failed")
	}
	args := []string{"message", dev, "0", "fail_if_no_path"}
	if _, err := b.exec.Execute(dmsetupCmd, args); err != nil {
		return b.logger.ErrorRet(&commandExecuteError{dmsetupCmd, err}, "failed")
	}
	time.Sleep(3000 * time.Millisecond)
	if err := b.exec.IsExecutable(multipathCmd); err != nil {
		return b.logger.ErrorRet(&commandNotFoundError{multipathCmd, err}, "failed")
	}
	args = []string{"-f", dev}
	if _, err := b.exec.Execute(multipathCmd, args); err != nil {
		return b.logger.ErrorRet(&commandExecuteError{multipathCmd, err}, "failed")
	}
	b.logger.Info("flushed", logs.Args{{"mpath", mpath}})
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Program yang crawls all modules at certain commit of `https://github.com/openconfig/public`.
// Usage: yang [--path DIR] [--url URL]
//
// DIR is a comma separated list of paths that are appended as the search directory.
// If DIR appears as DIR/... then DIR and all direct and indirect subdirectories are checked.
//
// URL is github URL prefix of git commit that program is crawling.
// THIS PROGRAM IS STILL JUST A DEVELOPMENT TOOL.
package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/openconfig/catalog-server/pkg/db"
	oc "github.com/openconfig/catalog-server/pkg/ygotgen"
	"github.com/openconfig/goyang/pkg/yang"
	"github.com/openconfig/ygot/ygot"
	"github.com/pborman/getopt"
)

// exitIfError writes errs to standard error and exits with an exit status of 1.
// If errs is empty then exitIfError does nothing and simply returns.
func exitIfError(errs []error) {
	if len(errs) > 0 {
		for _, err := range errs {
			fmt.Fprintln(os.Stderr, err)
		}
		stop(1)
	}
}

const (
	modelDir     = `release/models`   // modelDir is directory in openconfig/public github repo that contains modules in `models` directory.
	modelKeyword = `models`           // We check whether path of found module contains `models` to check whether it's under `models` directory.
	ietfDir      = `third_party/ietf` // ietfDir is directory in openconfig/public github repo that contains modules in `ietf` directory.
	ietfKeyword  = `ietf`             // We check whether path of found module contains `ietf` to check whether it's under `ietf` directory.
	orgName      = `openconfig`       // default orgName that is used when inserting modules into database.
)

// urlMap is map from model's name to its github URL.
var urlMap = map[string]string{}

// traverseDir traverses given directory *dir* to find all modules in this directory including its sub-directories.
// *url* is the url prefix of github repo at certain commit.
// It returns a slice of names of modules found.
func traverseDir(dir string, url string) ([]string, error) {
	dirfiles, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("traverseDir: read files from directory %s failed: %v\n", dir, err)
	}
	var names []string
	var dirs []string
	for _, f := range dirfiles {
		if f.Mode().IsDir() {
			// Append subdirectories into *dirs*, and traverse them later.
			dirs = append(dirs, f.Name())
		} else if f.Mode().IsRegular() && strings.HasSuffix(f.Name(), ".yang") {
			// Only search for files with suffix of `.yang`
			fullpath := dir + "/" + f.Name()
			currDir := path.Dir(fullpath)
			// currDir is the name of current directory containing this module file.
			currDir = currDir[strings.LastIndex(currDir, "/")+1:]
			file, err := os.Open(fullpath)
			if err != nil {
				return nil, fmt.Errorf("traverseDir: failed to open file %s: %v", fullpath, err)
			}
			// *name* is name of yang module/submodule, we get it by removing `.yang` from original file name.
			name := f.Name()[:len(f.Name())-5]

			// Check whether found module is under `models` directory or `ietf` dirctory.
			if strings.Contains(fullpath, modelKeyword) {
				// Modules/submodules under `models` dirctory are under subdirectory (*currDir*) of `models` directory.
				// Store url of found module/submodule in urlMap.
				urlMap[name] = url + modelDir + "/" + currDir + "/" + f.Name()
			} else {
				// Found modules must be either under `models` directory or `ietf` directory.
				if !(strings.Contains(fullpath, ietfKeyword)) {
					return nil, fmt.Errorf("traverseDir: model %s not in either models dir or ietd dir", f.Name())
				}
				// `ietf` directory does not have subdirectory.
				// Store url of found module/submodule in urlMap.
				urlMap[name] = url + ietfDir + "/" + f.Name()
			}
			scanner := bufio.NewScanner(file)
			scanner.Split(bufio.ScanLines)
			scanner.Scan()

			// Only append `module` and ignore `submodules`.
			if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "module ") {
				names = append(names, name)
			} else {
				log.Println(fullpath + " does not have modules")
			}
			file.Close()
		}
	}

	// Traverse found subdirectories.
	for _, dirName := range dirs {
		newnames, err := traverseDir(dir+"/"+dirName, url)
		if err != nil {
			return nil, err
		}
		names = append(names, newnames...)
	}
	return names, nil
}

var stop = os.Exit

func main() {
	var help bool
	var paths []string
	var url string
	getopt.ListVarLong(&paths, "path", 'p', "comma separated list of directories to add to search path", "DIR[,DIR...]")
	getopt.BoolVarLong(&help, "help", 'h', "display help")
	// *url* is the url prefix of github repo at certain commit that we are crawling modules from.
	getopt.StringVarLong(&url, "url", 'u', "url prefix of git commit that we are crawling")
	getopt.SetParameters("")

	if err := getopt.Getopt(func(o getopt.Option) bool {
		return true
	}); err != nil {
		fmt.Fprintln(os.Stderr, err)
		getopt.PrintUsage(os.Stderr)
		os.Exit(1)
	}

	if help {
		getopt.CommandLine.PrintUsage(os.Stderr)
		os.Exit(0)
	}

	for _, path := range paths {
		expanded, err := yang.PathsWithModules(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		yang.AddPath(expanded...)
	}

	// files stores names of all modules to search for.
	files := getopt.Args()

	ms := yang.NewModules()

	// If names of modules to search for is not given, we traverse all given path to find
	// all modules in these directories and crawl them later.
	if len(files) == 0 {
		for _, path := range paths {
			names, err := traverseDir(path, url)
			if err != nil {
				log.Fatalf("traverse directory %s failed: %v", path, err)
			}
			// Append all found modules into *files*.
			files = append(files, names...)
		}
	}

	for _, name := range files {
		if err := ms.Read(name); err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
	}

	// Process the read files, exiting if any errors were found.
	exitIfError(ms.Process())

	// Keep track of the top level modules we read in.
	// Those are the only modules we want to print below.
	mods := map[string]*yang.Module{}
	var names []string

	for _, m := range ms.Modules {
		if mods[m.Name] == nil {
			mods[m.Name] = m
			names = append(names, m.Name)
		}
	}
	sort.Strings(names)

	// Connect to DB
	if err := db.ConnectDB(); err != nil {
		log.Fatalf("Connect to db failed: %v", err)
		stop(1)
	}
	defer db.Close()

	// Convert all found modules into ygot go structure of Module and insert them into database.
	for _, n := range names {
		module := &oc.OpenconfigModuleCatalog_Organizations_Organization_Modules_Module{
			Name:      &mods[n].Name,
			Namespace: &mods[n].Namespace.Name,
			Prefix:    &mods[n].Prefix.Name,
			Summary:   &mods[n].Description.Name,
		}

		version, err := yang.MatchingExtensions(mods[n], "openconfig-extensions", "openconfig-version")
		if err != nil || len(version) == 0 {
			log.Printf("%s do not have version\n", mods[n].Name)
			continue
		}

		module.Version = &version[0].Argument

		// If there are multiple revisions, we directly use the lastest one.
		if len(mods[n].Revision) > 0 {
			module.Revision = &mods[n].Revision[0].Name
		}
		for i := 0; i < len(mods[n].Import); i++ {
			module.GetOrCreateDependencies().RequiredModule = append(module.GetOrCreateDependencies().RequiredModule, mods[n].Import[i].Name)
		}
		for i := 0; i < len(mods[n].Include); i++ {
			module.GetOrCreateSubmodules().GetOrCreateSubmodule(mods[n].Include[i].Name)
			submoduleURL := urlMap[mods[n].Include[i].Name]
			if submoduleURL == "" {
				log.Fatalf("cannot find url of submodule: %s", mods[n].Include[i].Name)
				continue
			}
			module.GetOrCreateSubmodules().GetOrCreateSubmodule(mods[n].Include[i].Name).GetOrCreateAccess().Uri = &submoduleURL

		}
		moduleURL := urlMap[n]
		module.GetOrCreateAccess().Uri = &moduleURL

		// Serialize module struct into json for insertion.
		json, err := ygot.EmitJSON(module, &ygot.EmitJSONConfig{
			Format: ygot.RFC7951,
			Indent: "  ",
			RFC7951Config: &ygot.RFC7951JSONConfig{
				AppendModuleName: true,
			},
		})
		if err != nil {
			log.Fatalf("Marshalling into json string failed\n")
		}

		if err := db.InsertModule(orgName, module.GetName(), module.GetVersion(), json); err != nil {
			log.Printf("Insert module, Name: %s, Version: %s failed: %v\n", module.GetName(), module.GetVersion(), err)
			continue
		}
		log.Printf("Inserting module succeeds, Name: %s, Version: %s\n", module.GetName(), module.GetVersion())
	}

}
<commit_msg>Crawler checks whether a module already exists in database before insertion. If the module already exists, we do not insert it into database again.<commit_after>// Copyright 2015 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Program yang crawls all modules at certain commit of `https://github.com/openconfig/public`.
// Usage: yang [--path DIR] [--url URL]
//
// DIR is a comma separated list of paths that are appended as the search directory.
// If DIR appears as DIR/... then DIR and all direct and indirect subdirectories are checked.
//
// URL is github URL prefix of git commit that program is crawling.
// THIS PROGRAM IS STILL JUST A DEVELOPMENT TOOL.
package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/openconfig/catalog-server/pkg/db"
	oc "github.com/openconfig/catalog-server/pkg/ygotgen"
	"github.com/openconfig/goyang/pkg/yang"
	"github.com/openconfig/ygot/ygot"
	"github.com/pborman/getopt"
)

// exitIfError writes errs to standard error and exits with an exit status of 1.
// If errs is empty then exitIfError does nothing and simply returns.
func exitIfError(errs []error) {
	if len(errs) > 0 {
		for _, err := range errs {
			fmt.Fprintln(os.Stderr, err)
		}
		stop(1)
	}
}

const (
	modelDir     = `release/models`   // modelDir is directory in openconfig/public github repo that contains modules in `models` directory.
	modelKeyword = `models`           // We check whether path of found module contains `models` to check whether it's under `models` directory.
	ietfDir      = `third_party/ietf` // ietfDir is directory in openconfig/public github repo that contains modules in `ietf` directory.
	ietfKeyword  = `ietf`             // We check whether path of found module contains `ietf` to check whether it's under `ietf` directory.
	orgName      = `openconfig`       // default orgName that is used when inserting modules into database.
)

// urlMap is map from model's name to its github URL.
var urlMap = map[string]string{}

// traverseDir traverses given directory *dir* to find all modules in this directory including its sub-directories.
// *url* is the url prefix of github repo at certain commit.
// It returns a slice of names of modules found.
func traverseDir(dir string, url string) ([]string, error) {
	dirfiles, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("traverseDir: read files from directory %s failed: %v\n", dir, err)
	}
	var names []string
	var dirs []string
	for _, f := range dirfiles {
		if f.Mode().IsDir() {
			// Append subdirectories into *dirs*, and traverse them later.
			dirs = append(dirs, f.Name())
		} else if f.Mode().IsRegular() && strings.HasSuffix(f.Name(), ".yang") {
			// Only search for files with suffix of `.yang`
			fullpath := dir + "/" + f.Name()
			currDir := path.Dir(fullpath)
			// currDir is the name of current directory containing this module file.
			currDir = currDir[strings.LastIndex(currDir, "/")+1:]
			file, err := os.Open(fullpath)
			if err != nil {
				return nil, fmt.Errorf("traverseDir: failed to open file %s: %v", fullpath, err)
			}
			// *name* is name of yang module/submodule, we get it by removing `.yang` from original file name.
			name := f.Name()[:len(f.Name())-5]

			// Check whether found module is under `models` directory or `ietf` dirctory.
			if strings.Contains(fullpath, modelKeyword) {
				// Modules/submodules under `models` dirctory are under subdirectory (*currDir*) of `models` directory.
				// Store url of found module/submodule in urlMap.
				urlMap[name] = url + modelDir + "/" + currDir + "/" + f.Name()
			} else {
				// Found modules must be either under `models` directory or `ietf` directory.
				if !(strings.Contains(fullpath, ietfKeyword)) {
					return nil, fmt.Errorf("traverseDir: model %s not in either models dir or ietd dir", f.Name())
				}
				// `ietf` directory does not have subdirectory.
				// Store url of found module/submodule in urlMap.
				urlMap[name] = url + ietfDir + "/" + f.Name()
			}
			scanner := bufio.NewScanner(file)
			scanner.Split(bufio.ScanLines)
			scanner.Scan()

			// Only append `module` and ignore `submodules`.
			if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "module ") {
				names = append(names, name)
			} else {
				log.Println(fullpath + " does not have modules")
			}
			file.Close()
		}
	}

	// Traverse found subdirectories.
	for _, dirName := range dirs {
		newnames, err := traverseDir(dir+"/"+dirName, url)
		if err != nil {
			return nil, err
		}
		names = append(names, newnames...)
	}
	return names, nil
}

var stop = os.Exit

func main() {
	var help bool
	var paths []string
	var url string
	getopt.ListVarLong(&paths, "path", 'p', "comma separated list of directories to add to search path", "DIR[,DIR...]")
	getopt.BoolVarLong(&help, "help", 'h', "display help")
	// *url* is the url prefix of github repo at certain commit that we are crawling modules from.
	getopt.StringVarLong(&url, "url", 'u', "url prefix of git commit that we are crawling")
	getopt.SetParameters("")

	if err := getopt.Getopt(func(o getopt.Option) bool {
		return true
	}); err != nil {
		fmt.Fprintln(os.Stderr, err)
		getopt.PrintUsage(os.Stderr)
		os.Exit(1)
	}

	if help {
		getopt.CommandLine.PrintUsage(os.Stderr)
		os.Exit(0)
	}

	for _, path := range paths {
		expanded, err := yang.PathsWithModules(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		yang.AddPath(expanded...)
	}

	// files stores names of all modules to search for.
	files := getopt.Args()

	ms := yang.NewModules()

	// If names of modules to search for is not given, we traverse all given path to find
	// all modules in these directories and crawl them later.
	if len(files) == 0 {
		for _, path := range paths {
			names, err := traverseDir(path, url)
			if err != nil {
				log.Fatalf("traverse directory %s failed: %v", path, err)
			}
			// Append all found modules into *files*.
			files = append(files, names...)
		}
	}

	for _, name := range files {
		if err := ms.Read(name); err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
	}

	// Process the read files, exiting if any errors were found.
	exitIfError(ms.Process())

	// Keep track of the top level modules we read in.
	// Those are the only modules we want to print below.
	mods := map[string]*yang.Module{}
	var names []string

	for _, m := range ms.Modules {
		if mods[m.Name] == nil {
			mods[m.Name] = m
			names = append(names, m.Name)
		}
	}
	sort.Strings(names)

	// Connect to DB
	if err := db.ConnectDB(); err != nil {
		log.Fatalf("Connect to db failed: %v", err)
		stop(1)
	}
	defer db.Close()

	// Convert all found modules into ygot go structure of Module and insert them into database.
	for _, n := range names {
		module := &oc.OpenconfigModuleCatalog_Organizations_Organization_Modules_Module{
			Name:      &mods[n].Name,
			Namespace: &mods[n].Namespace.Name,
			Prefix:    &mods[n].Prefix.Name,
			Summary:   &mods[n].Description.Name,
		}

		version, err := yang.MatchingExtensions(mods[n], "openconfig-extensions", "openconfig-version")
		if err != nil || len(version) == 0 {
			log.Printf("%s do not have version\n", mods[n].Name)
			continue
		}

		module.Version = &version[0].Argument

		// If there are multiple revisions, we directly use the lastest one.
		if len(mods[n].Revision) > 0 {
			module.Revision = &mods[n].Revision[0].Name
		}
		for i := 0; i < len(mods[n].Import); i++ {
			module.GetOrCreateDependencies().RequiredModule = append(module.GetOrCreateDependencies().RequiredModule, mods[n].Import[i].Name)
		}
		for i := 0; i < len(mods[n].Include); i++ {
			module.GetOrCreateSubmodules().GetOrCreateSubmodule(mods[n].Include[i].Name)
			submoduleURL := urlMap[mods[n].Include[i].Name]
			if submoduleURL == "" {
				log.Fatalf("cannot find url of submodule: %s", mods[n].Include[i].Name)
				continue
			}
			module.GetOrCreateSubmodules().GetOrCreateSubmodule(mods[n].Include[i].Name).GetOrCreateAccess().Uri = &submoduleURL

		}
		moduleURL := urlMap[n]
		module.GetOrCreateAccess().Uri = &moduleURL

		// Serialize module struct into json for insertion.
		json, err := ygot.EmitJSON(module, &ygot.EmitJSONConfig{
			Format: ygot.RFC7951,
			Indent: "  ",
			RFC7951Config: &ygot.RFC7951JSONConfig{
				AppendModuleName: true,
			},
		})
		if err != nil {
			log.Fatalf("Marshalling into json string failed\n")
		}

		// Query to check whether the key already exists before insertion.
		// As we crawl from the lastest version to the oldest one, we want to only insert the lastest data into database.
		queryRes, err := db.QueryModulesByKey(module.Name, module.Version)
		if err != nil {
			log.Printf("Query module, Name: %s, Version: %s failed: %v\n", module.GetName(), module.GetVersion(), err)
			continue
		}

		// If the key already matches an existing module, we would not do an insertion.
		if len(queryRes) > 0 {
			log.Printf("module, Name: %s, Version: %s already exists in database, do not update it\n", module.GetName(), module.GetVersion())
			continue
		}

		if err := db.InsertModule(orgName, module.GetName(), module.GetVersion(), json); err != nil {
			log.Printf("Insert module, Name: %s, Version: %s failed: %v\n", module.GetName(), module.GetVersion(), err)
			continue
		}
		log.Printf("Inserting module succeeds, Name: %s, Version: %s\n", module.GetName(), module.GetVersion())
	}

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/uber-common/cpustat/lib"
	"strconv"
	"strings"
	"time"
)

func main() {
	var interval = flag.Int("i", 200, "Interval (ms) between measurements")
	var pidList = flag.String("p", "", "Comma separated PID list to profile")
	var sampleCount = flag.Uint("n", 0, "Maximum number of samples to capture")

	flag.Parse()

	targetSleep := time.Duration(*interval) * time.Millisecond

	pidStrings := strings.Split(*pidList, ",")
	pids := make([]int, len(pidStrings))

	for i, pidString := range pidStrings {
		pid, err := strconv.Atoi(pidString)

		if err != nil {
			panic(err)
		}

		pids[i] = pid
	}

	procStats := cpustat.ProcStats{}
	procStatsReaderCount := len(pids)
	procStatsReaders := make([]*cpustat.ProcStatsSeekReader, procStatsReaderCount)

	samplesRemaining := int64(*sampleCount)
	if samplesRemaining == 0 {
		samplesRemaining = -1
	}

	for i, pid := range pids {
		procStatsReader := cpustat.ProcStatsSeekReader{
			PID: pid,
		}
		procStatsReaders[i] = &procStatsReader

		procStatsInitError := procStatsReader.Initialize()
		if procStatsInitError != nil {
			procStatsReaders[i] = nil
			procStatsReaderCount--
		}
	}

	fmt.Printf("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n",
		"pid",
		"time",
		"proc.utime",
		"proc.stime",
		"proc.cutime",
		"proc.cstime",
		"proc.numthreads",
		"proc.rss",
		"proc.guesttime",
		"proc.cguesttime",
	)

	t1 := time.Now()

	for procStatsReaderCount > 0 && samplesRemaining != 0 {
		for i, procStatsReader := range procStatsReaders {
			if procStatsReader == nil {
				continue
			}

			procStatsError := procStatsReader.ReadStats(&procStats)
			if procStatsError != nil {
				procStatsReaders[i] = nil
				procStatsReaderCount--
				continue
			}

			fmt.Printf(
				"%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\n",
				procStatsReader.PID,
				procStats.CaptureTime.UnixNano()/1e6,
				procStats.Utime,
				procStats.Stime,
				procStats.Cutime,
				procStats.Cstime,
				procStats.Numthreads,
				procStats.Rss,
				procStats.Guesttime,
				procStats.Cguesttime,
			)
		}

		if samplesRemaining > 0 {
			samplesRemaining--
		}

		if procStatsReaderCount > 0 && samplesRemaining != 0 {
			t2 := time.Now()
			adjustedSleep := targetSleep - t2.Sub(t1)
			time.Sleep(adjustedSleep)
			t1 = time.Now()
		}
	}
}
<commit_msg>cpustat-raw: fix import grouping<commit_after>package main

import (
	"flag"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/uber-common/cpustat/lib"
)

func main() {
	var interval = flag.Int("i", 200, "Interval (ms) between measurements")
	var pidList = flag.String("p", "", "Comma separated PID list to profile")
	var sampleCount = flag.Uint("n", 0, "Maximum number of samples to capture")

	flag.Parse()

	targetSleep := time.Duration(*interval) * time.Millisecond

	pidStrings := strings.Split(*pidList, ",")
	pids := make([]int, len(pidStrings))

	for i, pidString := range pidStrings {
		pid, err := strconv.Atoi(pidString)

		if err != nil {
			panic(err)
		}

		pids[i] = pid
	}

	procStats := cpustat.ProcStats{}
	procStatsReaderCount := len(pids)
	procStatsReaders := make([]*cpustat.ProcStatsSeekReader, procStatsReaderCount)

	samplesRemaining := int64(*sampleCount)
	if samplesRemaining == 0 {
		samplesRemaining = -1
	}

	for i, pid := range pids {
		procStatsReader := cpustat.ProcStatsSeekReader{
			PID: pid,
		}
		procStatsReaders[i] = &procStatsReader

		procStatsInitError := procStatsReader.Initialize()
		if procStatsInitError != nil {
			procStatsReaders[i] = nil
			procStatsReaderCount--
		}
	}

	fmt.Printf("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n",
		"pid",
		"time",
		"proc.utime",
		"proc.stime",
		"proc.cutime",
		"proc.cstime",
		"proc.numthreads",
		"proc.rss",
		"proc.guesttime",
		"proc.cguesttime",
	)

	t1 := time.Now()

	for procStatsReaderCount > 0 && samplesRemaining != 0 {
		for i, procStatsReader := range procStatsReaders {
			if procStatsReader == nil {
				continue
			}

			procStatsError := procStatsReader.ReadStats(&procStats)
			if procStatsError != nil {
				procStatsReaders[i] = nil
				procStatsReaderCount--
				continue
			}

			fmt.Printf(
				"%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\n",
				procStatsReader.PID,
				procStats.CaptureTime.UnixNano()/1e6,
				procStats.Utime,
				procStats.Stime,
				procStats.Cutime,
				procStats.Cstime,
				procStats.Numthreads,
				procStats.Rss,
				procStats.Guesttime,
				procStats.Cguesttime,
			)
		}

		if samplesRemaining > 0 {
			samplesRemaining--
		}

		if procStatsReaderCount > 0 && samplesRemaining != 0 {
			t2 := time.Now()
			adjustedSleep := targetSleep - t2.Sub(t1)
			time.Sleep(adjustedSleep)
			t1 = time.Now()
		}
	}
}
<|endoftext|>