<commit_before>package buildpack

import (
	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/flag_helpers"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/codegangsta/cli"
)

type UpdateBuildpack struct {
	ui                terminal.UI
	buildpackRepo     api.BuildpackRepository
	buildpackBitsRepo api.BuildpackBitsRepository
	buildpackReq      requirements.BuildpackRequirement
}

func NewUpdateBuildpack(ui terminal.UI, repo api.BuildpackRepository, bitsRepo api.BuildpackBitsRepository) (cmd *UpdateBuildpack) {
	cmd = new(UpdateBuildpack)
	cmd.ui = ui
	cmd.buildpackRepo = repo
	cmd.buildpackBitsRepo = bitsRepo
	return
}

func (cmd *UpdateBuildpack) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "update-buildpack",
		Description: T("Update a buildpack"),
		Usage:       T("CF_NAME update-buildpack BUILDPACK [-p PATH] [-i POSITION] [--enable|--disable] [--lock|--unlock]"),
		Flags: []cli.Flag{
			flag_helpers.NewIntFlag("i", T("Buildpack position among other buildpacks")),
			flag_helpers.NewStringFlag("p", T("Path to directory or zip file")),
			cli.BoolFlag{Name: "enable", Usage: T("Enable the buildpack")},
			cli.BoolFlag{Name: "disable", Usage: T("Disable the buildpack")},
			cli.BoolFlag{Name: "lock", Usage: T("Lock the buildpack")},
			cli.BoolFlag{Name: "unlock", Usage: T("Unlock the buildpack")},
		},
	}
}

func (cmd *UpdateBuildpack) GetRequirements(requirementsFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {
	if len(c.Args()) != 1 {
		cmd.ui.FailWithUsage(c)
	}

	loginReq := requirementsFactory.NewLoginRequirement()
	cmd.buildpackReq = requirementsFactory.NewBuildpackRequirement(c.Args()[0])

	reqs = []requirements.Requirement{
		loginReq,
		cmd.buildpackReq,
	}

	return
}

func (cmd *UpdateBuildpack) Run(c *cli.Context) {
	buildpack := cmd.buildpackReq.GetBuildpack()

	cmd.ui.Say(T("Updating buildpack {{.BuildpackName}}...", map[string]interface{}{"BuildpackName": terminal.EntityNameColor(buildpack.Name)}))

	updateBuildpack := false

	if c.IsSet("i") {
		position := c.Int("i")

		buildpack.Position = &position
		updateBuildpack = true
	}

	enabled := c.Bool("enable")
	disabled := c.Bool("disable")
	if enabled && disabled {
		cmd.ui.Failed(T("Cannot specify both enabled and disabled."))
		return
	}

	if enabled {
		buildpack.Enabled = &enabled
		updateBuildpack = true
	}
	if disabled {
		disabled = false
		buildpack.Enabled = &disabled
		updateBuildpack = true
	}

	lock := c.Bool("lock")
	unlock := c.Bool("unlock")
	if lock && unlock {
		cmd.ui.Failed(T("Cannot specify both lock and unlock options."))
		return
	}

	dir := c.String("p")
	if dir != "" && (lock || unlock) {
		cmd.ui.Failed(T("Cannot specify buildpack bits and lock/unlock."))
	}

	if lock {
		buildpack.Locked = &lock
		updateBuildpack = true
	}
	if unlock {
		unlock = false
		buildpack.Locked = &unlock
		updateBuildpack = true
	}

	if updateBuildpack {
		newBuildpack, apiErr := cmd.buildpackRepo.Update(buildpack)
		if apiErr != nil {
			cmd.ui.Failed(T("Error updating buildpack {{.Name}}\n{{.Error}}", map[string]interface{}{
				"Name":  terminal.EntityNameColor(buildpack.Name),
				"Error": apiErr.Error(),
			}))
		}
		buildpack = newBuildpack
	}

	if dir != "" {
		apiErr := cmd.buildpackBitsRepo.UploadBuildpack(buildpack, dir)
		if apiErr != nil {
			cmd.ui.Failed(T("Error uploading buildpack {{.Name}}\n{{.Error}}", map[string]interface{}{
				"Name":  terminal.EntityNameColor(buildpack.Name),
				"Error": apiErr.Error(),
			}))
		}
	}
	cmd.ui.Ok()
}
<commit_msg>Fixup buildpack translations<commit_after>package buildpack

import (
	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/flag_helpers"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/codegangsta/cli"
)

type UpdateBuildpack struct {
	ui                terminal.UI
	buildpackRepo     api.BuildpackRepository
	buildpackBitsRepo api.BuildpackBitsRepository
	buildpackReq      requirements.BuildpackRequirement
}

func NewUpdateBuildpack(ui terminal.UI, repo api.BuildpackRepository, bitsRepo api.BuildpackBitsRepository) (cmd *UpdateBuildpack) {
	cmd = new(UpdateBuildpack)
	cmd.ui = ui
	cmd.buildpackRepo = repo
	cmd.buildpackBitsRepo = bitsRepo
	return
}

func (cmd *UpdateBuildpack) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "update-buildpack",
		Description: T("Update a buildpack"),
		Usage:       T("CF_NAME update-buildpack BUILDPACK [-p PATH] [-i POSITION] [--enable|--disable] [--lock|--unlock]"),
		Flags: []cli.Flag{
			flag_helpers.NewIntFlag("i", T("Buildpack position among other buildpacks")),
			flag_helpers.NewStringFlag("p", T("Path to directory or zip file")),
			cli.BoolFlag{Name: "enable", Usage: T("Enable the buildpack")},
			cli.BoolFlag{Name: "disable", Usage: T("Disable the buildpack")},
			cli.BoolFlag{Name: "lock", Usage: T("Lock the buildpack")},
			cli.BoolFlag{Name: "unlock", Usage: T("Unlock the buildpack")},
		},
	}
}

func (cmd *UpdateBuildpack) GetRequirements(requirementsFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {
	if len(c.Args()) != 1 {
		cmd.ui.FailWithUsage(c)
	}

	loginReq := requirementsFactory.NewLoginRequirement()
	cmd.buildpackReq = requirementsFactory.NewBuildpackRequirement(c.Args()[0])

	reqs = []requirements.Requirement{
		loginReq,
		cmd.buildpackReq,
	}

	return
}

func (cmd *UpdateBuildpack) Run(c *cli.Context) {
	buildpack := cmd.buildpackReq.GetBuildpack()

	cmd.ui.Say(T("Updating buildpack {{.BuildpackName}}...", map[string]interface{}{"BuildpackName": terminal.EntityNameColor(buildpack.Name)}))

	updateBuildpack := false

	if c.IsSet("i") {
		position := c.Int("i")

		buildpack.Position = &position
		updateBuildpack = true
	}

	enabled := c.Bool("enable")
	disabled := c.Bool("disable")
	if enabled && disabled {
		cmd.ui.Failed(T("Cannot specify both {{.Enabled}} and {{.Disabled}}.", map[string]interface{}{
			"Enabled":  "enabled",
			"Disabled": "disabled",
		}))
	}

	if enabled {
		buildpack.Enabled = &enabled
		updateBuildpack = true
	}
	if disabled {
		disabled = false
		buildpack.Enabled = &disabled
		updateBuildpack = true
	}

	lock := c.Bool("lock")
	unlock := c.Bool("unlock")
	if lock && unlock {
		cmd.ui.Failed(T("Cannot specify both lock and unlock options."))
		return
	}

	dir := c.String("p")
	if dir != "" && (lock || unlock) {
		cmd.ui.Failed(T("Cannot specify buildpack bits and lock/unlock."))
	}

	if lock {
		buildpack.Locked = &lock
		updateBuildpack = true
	}
	if unlock {
		unlock = false
		buildpack.Locked = &unlock
		updateBuildpack = true
	}

	if updateBuildpack {
		newBuildpack, apiErr := cmd.buildpackRepo.Update(buildpack)
		if apiErr != nil {
			cmd.ui.Failed(T("Error updating buildpack {{.Name}}\n{{.Error}}", map[string]interface{}{
				"Name":  terminal.EntityNameColor(buildpack.Name),
				"Error": apiErr.Error(),
			}))
		}
		buildpack = newBuildpack
	}

	if dir != "" {
		apiErr := cmd.buildpackBitsRepo.UploadBuildpack(buildpack, dir)
		if apiErr != nil {
			cmd.ui.Failed(T("Error uploading buildpack {{.Name}}\n{{.Error}}", map[string]interface{}{
				"Name":  terminal.EntityNameColor(buildpack.Name),
				"Error": apiErr.Error(),
			}))
		}
	}
	cmd.ui.Ok()
}
<|endoftext|>


<commit_before>package hystrix

import (
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestReturn(t *testing.T) {
	defer Flush()

	Convey("when returning a ticket to the pool", t, func() {
		pool := newExecutorPool("pool")
		ticket := <-pool.Tickets
		pool.Return(ticket)
		Convey("total executed requests should increment", func() {
			So(pool.Metrics.Executed.Sum(time.Now()), ShouldEqual, 1)
		})
	})
}

func TestActiveCount(t *testing.T) {
	defer Flush()

	Convey("when 3 tickets are pulled", t, func() {
		pool := newExecutorPool("pool")
		<-pool.Tickets
		<-pool.Tickets
		ticket := <-pool.Tickets

		Convey("ActiveCount() should be 3", func() {
			So(pool.ActiveCount(), ShouldEqual, 3)
		})

		Convey("and one is returned", func() {
			pool.Return(ticket)

			Convey("max active requests should be 3", func() {
				time.Sleep(1 * time.Millisecond) // allow poolMetrics to process channel
				So(pool.Metrics.MaxActiveRequests.Max(time.Now()), ShouldEqual, 3)
			})
		})
	})
}
<commit_msg>have test account for buffered updates channel<commit_after>package hystrix

import (
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestReturn(t *testing.T) {
	defer Flush()

	Convey("when returning a ticket to the pool", t, func() {
		pool := newExecutorPool("pool")
		ticket := <-pool.Tickets
		pool.Return(ticket)
		time.Sleep(1 * time.Millisecond)
		Convey("total executed requests should increment", func() {
			So(pool.Metrics.Executed.Sum(time.Now()), ShouldEqual, 1)
		})
	})
}

func TestActiveCount(t *testing.T) {
	defer Flush()

	Convey("when 3 tickets are pulled", t, func() {
		pool := newExecutorPool("pool")
		<-pool.Tickets
		<-pool.Tickets
		ticket := <-pool.Tickets

		Convey("ActiveCount() should be 3", func() {
			So(pool.ActiveCount(), ShouldEqual, 3)
		})

		Convey("and one is returned", func() {
			pool.Return(ticket)

			Convey("max active requests should be 3", func() {
				time.Sleep(1 * time.Millisecond) // allow poolMetrics to process channel
				So(pool.Metrics.MaxActiveRequests.Max(time.Now()), ShouldEqual, 3)
			})
		})
	})
}
<|endoftext|>


<commit_before>package views

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"
	"unicode"

	"github.com/antihax/goesi"
	"golang.org/x/oauth2"

	"github.com/antihax/evedata/services/conservator"
	"github.com/antihax/evedata/services/vanguard"
	"github.com/antihax/evedata/services/vanguard/models"
)

func init() {
	vanguard.AddRoute("GET", "/account",
		func(w http.ResponseWriter, r *http.Request) {
			p := newPage(r, "Account Information")
			p["ScopeGroups"] = models.GetCharacterScopeGroups()
			renderTemplate(w, "account.html", time.Hour*24*31, p)
		})

	vanguard.AddAuthRoute("GET", "/U/accountInfo", accountInfo)
	vanguard.AddAuthRoute("POST", "/U/cursorChar", cursorChar)

	vanguard.AddAuthRoute("GET", "/U/crestTokens", apiGetCRESTTokens)
	vanguard.AddAuthRoute("DELETE", "/U/crestTokens", apiDeleteCRESTToken)

	vanguard.AddAuthRoute("GET", "/U/integrationTokens", apiGetIntegrationTokens)
	vanguard.AddAuthRoute("DELETE", "/U/integrationTokens", apiDeleteIntegrationToken)

	vanguard.AddAuthRoute("POST", "/U/toggleAuth", apiToggleAuth)

	vanguard.AddAuthRoute("GET", "/U/accessableIntegrations", apiAccessableIntegrations)
	vanguard.AddAuthRoute("POST", "/U/joinIntegration", apiJoinIntegration)

	vanguard.AddAuthRoute("POST", "/U/setMailPassword", apiSetMailPassword)

}

func apiToggleAuth(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	if s == nil {
		httpErrCode(w, errors.New("Cannot find session"), http.StatusInternalServerError)
		return
	}
	g := vanguard.GlobalsFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for toggle auth"), http.StatusUnauthorized)
		return
	}

	// Parse the characterID
	tokenCharacterID, err := strconv.ParseInt(r.FormValue("tokenCharacterID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusForbidden)
		return
	}

	_, err = g.Db.Exec("UPDATE evedata.crestTokens SET authCharacter = ! authCharacter WHERE characterID = ? and tokenCharacterID = ?", characterID, tokenCharacterID)
	if err != nil {
		httpErrCode(w, err, http.StatusInternalServerError)
		return
	}
}

func accountInfo(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		// Silently fail.
		return
	}

	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify response"), http.StatusForbidden)
		log.Printf("%+v\n", s.Values["character"])
		return
	}

	accountInfo, ok := s.Values["accountInfo"].([]byte)
	if !ok {
		if err := updateAccountInfo(s, int32(characterID), char.CharacterOwnerHash, char.CharacterName); err != nil {
			httpErr(w, err)
			return
		}

		if err := s.Save(r, w); err != nil {
			httpErr(w, err)
			return
		}
	}

	w.Write(accountInfo)
}

func cursorChar(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for cursor"), http.StatusUnauthorized)
		return
	}

	// Parse the cursorCharacterID
	cursorCharacterID, err := strconv.ParseInt(r.FormValue("cursorCharacterID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusForbidden)
		return
	}

	// Set our new cursor
	err = models.SetCursorCharacter(characterID, int32(cursorCharacterID))
	if err != nil {
		httpErr(w, err)
		return
	}

	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify for cursor"), http.StatusForbidden)
		return
	}

	// Update the account information in redis
	if err = updateAccountInfo(s, characterID, char.CharacterOwnerHash, char.CharacterName); err != nil {
		httpErr(w, err)
		return
	}

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiGetCRESTTokens(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find character response"), http.StatusForbidden)
		log.Printf("%+v\n", s.Values["character"])
		return
	}

	v, err := models.GetCRESTTokens(char.CharacterID, char.CharacterOwnerHash)
	if err != nil {
		httpErr(w, err)
		return
	}

	// Change scopes to groups
	for i := range v {
		v[i].Scopes = models.GetCharacterGroupsByScopesString(v[i].Scopes)
	}

	renderJSON(w, v, 0)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiDeleteCRESTToken(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	g := vanguard.GlobalsFromContext(r.Context())

	// Get the sessions main characterID
	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify response to delete"), http.StatusForbidden)
		return
	}

	cid, err := strconv.ParseInt(r.FormValue("tokenCharacterID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusNotFound)
		return
	}

	// Revoke the token before we delete. Do not error out if this fails.
	if tok, err := models.GetCRESTToken(char.CharacterID, char.CharacterOwnerHash, int32(cid)); err != nil {
		log.Println(err)
	} else {
		err = g.TokenAuthenticator.TokenRevoke(tok.RefreshToken)
		if err != nil {
			log.Println(err)
		}
	}

	if err := models.DeleteCRESTToken(char.CharacterID, int32(cid)); err != nil {
		httpErrCode(w, err, http.StatusConflict)
		return
	}

	if err = updateAccountInfo(s, char.CharacterID, char.CharacterOwnerHash, char.CharacterName); err != nil {
		httpErr(w, err)
		return
	}

	key := fmt.Sprintf("EVEDATA_TOKENSTORE_%d_%d", char.CharacterID, cid)
	red := g.Cache.Get()
	defer red.Close()
	red.Do("DEL", key)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiAccessableIntegrations(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token"), http.StatusUnauthorized)
		return
	}

	v, err := models.GetAvailableIntegrations(characterID)
	if err != nil {
		httpErr(w, err)
		return
	}

	renderJSON(w, v, 0)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiJoinIntegration(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	g := vanguard.GlobalsFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token"), http.StatusUnauthorized)
		return
	}

	integrationID, err := strconv.ParseInt(r.FormValue("integrationID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusNotFound)
		return
	}

	i, err := models.GetIntegrationsForCharacter(characterID, int32(integrationID))
	if err != nil {
		httpErr(w, err)
		return
	}

	token := &oauth2.Token{
		Expiry:       i.Expiry,
		AccessToken:  i.AccessToken,
		RefreshToken: i.RefreshToken,
		TokenType:    "Bearer",
	}

	// refresh the token if it expired
	if token.Expiry.After(time.Now()) {
		src := g.DiscordAuthenticator.TokenSource(token)

		token, err = src.Token()
		if err != nil {
			httpErr(w, err)
			return
		}
	}

	if err := g.RPCall("Conservator.JoinUser", conservator.JoinUser{
		IntegrationID: i.IntegrationID,
		AccessToken:   token.AccessToken,
		UserID:        i.IntegrationUserID,
		CharacterName: i.CharacterName,
		CharacterID:   i.TokenCharacterID,
	}, &ok); err != nil || !ok {
		httpErr(w, err)
		return
	}

}

func apiGetIntegrationTokens(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token"), http.StatusUnauthorized)
		return
	}

	v, err := models.GetIntegrationTokens(characterID)
	if err != nil {
		httpErr(w, err)
		return
	}

	renderJSON(w, v, 0)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiDeleteIntegrationToken(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token to delete"), http.StatusUnauthorized)
		return
	}

	if err := models.DeleteIntegrationToken(r.FormValue("type"), characterID, r.FormValue("userID")); err != nil {
		httpErrCode(w, err, http.StatusConflict)
		return
	}
}

func verifyPassword(s string) bool {
	var upper, lower, number bool
	for _, s := range s {
		switch {
		case unicode.IsNumber(s):
			number = true
		case unicode.IsUpper(s):
			upper = true
		case unicode.IsLower(s):
			lower = true
		}
	}
	if upper && lower && number && len(s) >= 12 {
		return true
	}
	return false
}

func apiSetMailPassword(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	if s == nil {
		httpErrCode(w, errors.New("could not find session"), http.StatusUnauthorized)
		return
	}

	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify response to change mail password"), http.StatusForbidden)
		return
	}

	tokenCharacterID, err := strconv.ParseInt(r.FormValue("tokenCharacterID"), 10, 32)
	if err != nil {
		httpErrCode(w, errors.New("invalid tokenCharacterID"), http.StatusBadRequest)
		return
	}

	if !verifyPassword(r.FormValue("password")) {
		httpErrCode(w, errors.New("Password must be at least 12 characters with one uppercase, one lowercase, and one number"), http.StatusBadRequest)
		return
	}

	if err := models.SetMailPassword(char.CharacterID, int32(tokenCharacterID), char.CharacterOwnerHash, r.FormValue("password")); err != nil {
		httpErr(w, err)
		return
	}
}
<commit_msg>remove unneeded message<commit_after>package views

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"
	"unicode"

	"github.com/antihax/goesi"
	"golang.org/x/oauth2"

	"github.com/antihax/evedata/services/conservator"
	"github.com/antihax/evedata/services/vanguard"
	"github.com/antihax/evedata/services/vanguard/models"
)

func init() {
	vanguard.AddRoute("GET", "/account",
		func(w http.ResponseWriter, r *http.Request) {
			p := newPage(r, "Account Information")
			p["ScopeGroups"] = models.GetCharacterScopeGroups()
			renderTemplate(w, "account.html", time.Hour*24*31, p)
		})

	vanguard.AddAuthRoute("GET", "/U/accountInfo", accountInfo)
	vanguard.AddAuthRoute("POST", "/U/cursorChar", cursorChar)

	vanguard.AddAuthRoute("GET", "/U/crestTokens", apiGetCRESTTokens)
	vanguard.AddAuthRoute("DELETE", "/U/crestTokens", apiDeleteCRESTToken)

	vanguard.AddAuthRoute("GET", "/U/integrationTokens", apiGetIntegrationTokens)
	vanguard.AddAuthRoute("DELETE", "/U/integrationTokens", apiDeleteIntegrationToken)

	vanguard.AddAuthRoute("POST", "/U/toggleAuth", apiToggleAuth)

	vanguard.AddAuthRoute("GET", "/U/accessableIntegrations", apiAccessableIntegrations)
	vanguard.AddAuthRoute("POST", "/U/joinIntegration", apiJoinIntegration)

	vanguard.AddAuthRoute("POST", "/U/setMailPassword", apiSetMailPassword)

}

func apiToggleAuth(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	if s == nil {
		httpErrCode(w, errors.New("Cannot find session"), http.StatusInternalServerError)
		return
	}
	g := vanguard.GlobalsFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for toggle auth"), http.StatusUnauthorized)
		return
	}

	// Parse the characterID
	tokenCharacterID, err := strconv.ParseInt(r.FormValue("tokenCharacterID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusForbidden)
		return
	}

	_, err = g.Db.Exec("UPDATE evedata.crestTokens SET authCharacter = ! authCharacter WHERE characterID = ? and tokenCharacterID = ?", characterID, tokenCharacterID)
	if err != nil {
		httpErrCode(w, err, http.StatusInternalServerError)
		return
	}
}

func accountInfo(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		// Silently fail.
		return
	}

	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify response"), http.StatusForbidden)
		log.Printf("%+v\n", s.Values["character"])
		return
	}

	accountInfo, ok := s.Values["accountInfo"].([]byte)
	if !ok {
		if err := updateAccountInfo(s, int32(characterID), char.CharacterOwnerHash, char.CharacterName); err != nil {
			httpErr(w, err)
			return
		}

		if err := s.Save(r, w); err != nil {
			httpErr(w, err)
			return
		}
	}

	w.Write(accountInfo)
}

func cursorChar(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for cursor"), http.StatusUnauthorized)
		return
	}

	// Parse the cursorCharacterID
	cursorCharacterID, err := strconv.ParseInt(r.FormValue("cursorCharacterID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusForbidden)
		return
	}

	// Set our new cursor
	err = models.SetCursorCharacter(characterID, int32(cursorCharacterID))
	if err != nil {
		httpErr(w, err)
		return
	}

	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify for cursor"), http.StatusForbidden)
		return
	}

	// Update the account information in redis
	if err = updateAccountInfo(s, characterID, char.CharacterOwnerHash, char.CharacterName); err != nil {
		httpErr(w, err)
		return
	}

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiGetCRESTTokens(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find character response"), http.StatusForbidden)
		return
	}

	v, err := models.GetCRESTTokens(char.CharacterID, char.CharacterOwnerHash)
	if err != nil {
		httpErr(w, err)
		return
	}

	// Change scopes to groups
	for i := range v {
		v[i].Scopes = models.GetCharacterGroupsByScopesString(v[i].Scopes)
	}

	renderJSON(w, v, 0)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiDeleteCRESTToken(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	g := vanguard.GlobalsFromContext(r.Context())

	// Get the sessions main characterID
	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify response to delete"), http.StatusForbidden)
		return
	}

	cid, err := strconv.ParseInt(r.FormValue("tokenCharacterID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusNotFound)
		return
	}

	// Revoke the token before we delete. Do not error out if this fails.
	if tok, err := models.GetCRESTToken(char.CharacterID, char.CharacterOwnerHash, int32(cid)); err != nil {
		log.Println(err)
	} else {
		err = g.TokenAuthenticator.TokenRevoke(tok.RefreshToken)
		if err != nil {
			log.Println(err)
		}
	}

	if err := models.DeleteCRESTToken(char.CharacterID, int32(cid)); err != nil {
		httpErrCode(w, err, http.StatusConflict)
		return
	}

	if err = updateAccountInfo(s, char.CharacterID, char.CharacterOwnerHash, char.CharacterName); err != nil {
		httpErr(w, err)
		return
	}

	key := fmt.Sprintf("EVEDATA_TOKENSTORE_%d_%d", char.CharacterID, cid)
	red := g.Cache.Get()
	defer red.Close()
	red.Do("DEL", key)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiAccessableIntegrations(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token"), http.StatusUnauthorized)
		return
	}

	v, err := models.GetAvailableIntegrations(characterID)
	if err != nil {
		httpErr(w, err)
		return
	}

	renderJSON(w, v, 0)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiJoinIntegration(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	g := vanguard.GlobalsFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token"), http.StatusUnauthorized)
		return
	}

	integrationID, err := strconv.ParseInt(r.FormValue("integrationID"), 10, 64)
	if err != nil {
		httpErrCode(w, err, http.StatusNotFound)
		return
	}

	i, err := models.GetIntegrationsForCharacter(characterID, int32(integrationID))
	if err != nil {
		httpErr(w, err)
		return
	}

	token := &oauth2.Token{
		Expiry:       i.Expiry,
		AccessToken:  i.AccessToken,
		RefreshToken: i.RefreshToken,
		TokenType:    "Bearer",
	}

	// refresh the token if it expired
	if token.Expiry.After(time.Now()) {
		src := g.DiscordAuthenticator.TokenSource(token)

		token, err = src.Token()
		if err != nil {
			httpErr(w, err)
			return
		}
	}

	if err := g.RPCall("Conservator.JoinUser", conservator.JoinUser{
		IntegrationID: i.IntegrationID,
		AccessToken:   token.AccessToken,
		UserID:        i.IntegrationUserID,
		CharacterName: i.CharacterName,
		CharacterID:   i.TokenCharacterID,
	}, &ok); err != nil || !ok {
		httpErr(w, err)
		return
	}

}

func apiGetIntegrationTokens(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token"), http.StatusUnauthorized)
		return
	}

	v, err := models.GetIntegrationTokens(characterID)
	if err != nil {
		httpErr(w, err)
		return
	}

	renderJSON(w, v, 0)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}

func apiDeleteIntegrationToken(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, errors.New("could not find character ID for integration token to delete"), http.StatusUnauthorized)
		return
	}

	if err := models.DeleteIntegrationToken(r.FormValue("type"), characterID, r.FormValue("userID")); err != nil {
		httpErrCode(w, err, http.StatusConflict)
		return
	}
}

func verifyPassword(s string) bool {
	var upper, lower, number bool
	for _, s := range s {
		switch {
		case unicode.IsNumber(s):
			number = true
		case unicode.IsUpper(s):
			upper = true
		case unicode.IsLower(s):
			lower = true
		}
	}
	if upper && lower && number && len(s) >= 12 {
		return true
	}
	return false
}

func apiSetMailPassword(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())
	if s == nil {
		httpErrCode(w, errors.New("could not find session"), http.StatusUnauthorized)
		return
	}

	char, ok := s.Values["character"].(goesi.VerifyResponse)
	if !ok {
		httpErrCode(w, errors.New("could not find verify response to change mail password"), http.StatusForbidden)
		return
	}

	tokenCharacterID, err := strconv.ParseInt(r.FormValue("tokenCharacterID"), 10, 32)
	if err != nil {
		httpErrCode(w, errors.New("invalid tokenCharacterID"), http.StatusBadRequest)
		return
	}

	if !verifyPassword(r.FormValue("password")) {
		httpErrCode(w, errors.New("Password must be at least 12 characters with one uppercase, one lowercase, and one number"), http.StatusBadRequest)
		return
	}

	if err := models.SetMailPassword(char.CharacterID, int32(tokenCharacterID), char.CharacterOwnerHash, r.FormValue("password")); err != nil {
		httpErr(w, err)
		return
	}
}
<|endoftext|>


<commit_before>package cgutil

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/opencontainers/runc/libcontainer/cgroups"

	"github.com/hashicorp/go-hclog"

	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"

	"github.com/hashicorp/nomad/lib/cpuset"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	"github.com/opencontainers/runc/libcontainer/configs"

	"github.com/hashicorp/nomad/nomad/structs"
)

func NewCpusetManager(cgroupParent string, logger hclog.Logger) CpusetManager {
	if cgroupParent == "" {
		cgroupParent = DefaultCgroupParent
	}
	return &cpusetManager{
		cgroupParent: cgroupParent,
		cgroupInfo:   map[string]allocTaskCgroupInfo{},
		logger:       logger,
	}
}

var (
	cpusetReconcileInterval = 30 * time.Second
)

type cpusetManager struct {
	// cgroupParent relative to the cgroup root. ex. '/nomad'
	cgroupParent string
	// cgroupParentPath is the absolute path to the cgroup parent.
	cgroupParentPath string

	parentCpuset cpuset.CPUSet

	// all exported functions are synchronized
	mu sync.Mutex

	cgroupInfo map[string]allocTaskCgroupInfo

	doneCh   chan struct{}
	signalCh chan struct{}
	logger   hclog.Logger
}

func (c *cpusetManager) AddAlloc(alloc *structs.Allocation) {
	if alloc == nil || alloc.AllocatedResources == nil {
		return
	}
	allocInfo := allocTaskCgroupInfo{}
	for task, resources := range alloc.AllocatedResources.Tasks {
		taskCpuset := cpuset.New(resources.Cpu.ReservedCores...)
		cgroupPath := filepath.Join(c.cgroupParentPath, SharedCpusetCgroupName)
		relativeCgroupPath := filepath.Join(c.cgroupParent, SharedCpusetCgroupName)
		if taskCpuset.Size() > 0 {
			cgroupPath, relativeCgroupPath = c.getCgroupPathsForTask(alloc.ID, task)
		}
		allocInfo[task] = &TaskCgroupInfo{
			CgroupPath:         cgroupPath,
			RelativeCgroupPath: relativeCgroupPath,
			Cpuset:             taskCpuset,
		}
	}
	c.mu.Lock()
	c.cgroupInfo[alloc.ID] = allocInfo
	c.mu.Unlock()
	go c.signalReconcile()
}

func (c *cpusetManager) RemoveAlloc(allocID string) {
	c.mu.Lock()
	delete(c.cgroupInfo, allocID)
	c.mu.Unlock()
	go c.signalReconcile()
}

func (c *cpusetManager) CgroupPathFor(allocID, task string) CgroupPathGetter {
	return func(ctx context.Context) (string, error) {
		c.mu.Lock()
		allocInfo, ok := c.cgroupInfo[allocID]
		if !ok {
			c.mu.Unlock()
			return "", fmt.Errorf("alloc not found for id %q", allocID)
		}

		taskInfo, ok := allocInfo[task]
		c.mu.Unlock()
		if !ok {
			return "", fmt.Errorf("task %q not found", task)
		}

		for {
			if taskInfo.Error != nil {
				break
			}
			if _, err := os.Stat(taskInfo.CgroupPath); os.IsNotExist(err) {
				select {
				case <-ctx.Done():
					return taskInfo.CgroupPath, ctx.Err()
				case <-time.After(100 * time.Millisecond):
					continue
				}
			}
			break
		}

		return taskInfo.CgroupPath, taskInfo.Error
	}

}

type allocTaskCgroupInfo map[string]*TaskCgroupInfo

// Init checks that the cgroup parent and expected child cgroups have been created
// If the cgroup parent is set to /nomad then this will ensure that the /nomad/shared
// cgroup is initialized.
func (c *cpusetManager) Init() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	cgroupParentPath, err := getCgroupPathHelper("cpuset", c.cgroupParent)
	if err != nil {
		return err
	}
	c.cgroupParentPath = cgroupParentPath

	// ensures that shared cpuset exists and that the cpuset values are copied from the parent if created
	if err := cpusetEnsureParent(filepath.Join(cgroupParentPath, SharedCpusetCgroupName)); err != nil {
		return err
	}

	parentCpus, parentMems, err := getCpusetSubsystemSettings(cgroupParentPath)
	if err != nil {
		return fmt.Errorf("failed to detect parent cpuset settings: %v", err)
	}
	c.parentCpuset, err = cpuset.Parse(parentCpus)
	if err != nil {
		return fmt.Errorf("failed to parse parent cpuset.cpus setting: %v", err)
	}

	// ensure the reserved cpuset exists, but only copy the mems from the parent if creating the cgroup
	if err := os.Mkdir(filepath.Join(cgroupParentPath, ReservedCpusetCgroupName), 0755); err == nil {
		// cgroup created, leave cpuset.cpus empty but copy cpuset.mems from parent
		if err != nil {
			return err
		}
		if err := fscommon.WriteFile(filepath.Join(cgroupParentPath, ReservedCpusetCgroupName), "cpuset.mems", parentMems); err != nil {
			return err
		}
	} else if !os.IsExist(err) {
		return err
	}

	c.doneCh = make(chan struct{})
	c.signalCh = make(chan struct{})

	c.logger.Info("initialized cpuset cgroup manager", "parent", c.cgroupParent, "cpuset", c.parentCpuset.String())

	go c.reconcileLoop()
	return nil
}

func (c *cpusetManager) reconcileLoop() {
	timer := time.NewTimer(0)
	if !timer.Stop() {
		<-timer.C
	}
	defer timer.Stop()

	for {
		select {
		case <-c.doneCh:
			c.logger.Debug("shutting down reconcile loop")
			return
		case <-c.signalCh:
			timer.Reset(500 * time.Millisecond)
		case <-timer.C:
			c.reconcileCpusets()
			timer.Reset(cpusetReconcileInterval)
		}
	}
}

func (c *cpusetManager) reconcileCpusets() {
	c.mu.Lock()
	defer c.mu.Unlock()
	sharedCpuset := cpuset.New(c.parentCpuset.ToSlice()...)
	reservedCpuset := cpuset.New()
	taskCpusets := map[string]*TaskCgroupInfo{}
	for _, alloc := range c.cgroupInfo {
		for _, task := range alloc {
			if task.Cpuset.Size() == 0 {
				continue
			}
			sharedCpuset = sharedCpuset.Difference(task.Cpuset)
			reservedCpuset = reservedCpuset.Union(task.Cpuset)
			taskCpusets[task.CgroupPath] = task
		}
	}

	// look for reserved cpusets which we don't know about and remove
	files, err := ioutil.ReadDir(c.reservedCpusetPath())
	if err != nil {
		c.logger.Error("failed to list files in reserved cgroup path during reconciliation", "path", c.reservedCpusetPath(), "error", err)
	}
	for _, f := range files {
		if !f.IsDir() {
			continue
		}
		path := filepath.Join(c.reservedCpusetPath(), f.Name())
		if _, ok := taskCpusets[path]; ok {
			continue
		}
		c.logger.Debug("removing reserved cpuset cgroup", "path", path)
		err := cgroups.RemovePaths(map[string]string{"cpuset": path})
		if err != nil {
			c.logger.Error("removal of existing cpuset cgroup failed", "path", path, "error", err)
		}
	}

	if err := c.setCgroupCpusetCPUs(c.sharedCpusetPath(), sharedCpuset.String()); err != nil {
		c.logger.Error("could not write shared cpuset.cpus", "path", c.sharedCpusetPath(), "cpuset.cpus", sharedCpuset.String(), "error", err)
	}
	if err := c.setCgroupCpusetCPUs(c.reservedCpusetPath(), reservedCpuset.String()); err != nil {
		c.logger.Error("could not write reserved cpuset.cpus", "path", c.reservedCpusetPath(), "cpuset.cpus", reservedCpuset.String(), "error", err)
	}
	for _, info := range taskCpusets {
		if err := os.Mkdir(info.CgroupPath, 0755); err != nil && !os.IsExist(err) {
			c.logger.Error("failed to create new cgroup path for task", "path", info.CgroupPath, "error", err)
			info.Error = err
			continue
		}

		// copy cpuset.mems from parent
		_, parentMems, err := getCpusetSubsystemSettings(filepath.Dir(info.CgroupPath))
		if err != nil {
			c.logger.Error("failed to read parent cgroup settings for task", "path", info.CgroupPath, "error", err)
			info.Error = err
			continue
		}
		if err := fscommon.WriteFile(info.CgroupPath, "cpuset.mems", parentMems); err != nil {
			c.logger.Error("failed to write cgroup cpuset.mems setting for task", "path", info.CgroupPath, "mems", parentMems, "error", err)
			info.Error = err
			continue
		}
		if err := c.setCgroupCpusetCPUs(info.CgroupPath, info.Cpuset.String()); err != nil {
			c.logger.Error("failed to write cgroup cpuset.cpus settings for task", "path", info.CgroupPath, "cpus", info.Cpuset.String(), "error", err)
			info.Error = err
			continue
		}
	}
}

// setCgroupCpusetCPUs will compare an existing cpuset.cpus value with an expected value, overwriting the existing if different
// must hold a lock on cpusetManager.mu before calling
func (_ *cpusetManager) setCgroupCpusetCPUs(path, cpus string) error {
	currentCpusRaw, err := fscommon.ReadFile(path, "cpuset.cpus")
	if err != nil {
		return err
	}

	if cpus != strings.TrimSpace(currentCpusRaw) {
		if err := fscommon.WriteFile(path, "cpuset.cpus", cpus); err != nil {
			return err
		}
	}
	return nil
}

func (c *cpusetManager) signalReconcile() {
	select {
	case c.signalCh <- struct{}{}:
	case <-c.doneCh:
	}
}

func (c *cpusetManager) getCpuset(group string) (cpuset.CPUSet, error) {
	man := cgroupFs.NewManager(
		&configs.Cgroup{
			Path: filepath.Join(c.cgroupParent, group),
		},
		map[string]string{"cpuset": filepath.Join(c.cgroupParentPath, group)},
		false,
	)
	stats, err := man.GetStats()
	if err != nil {
		return cpuset.CPUSet{}, err
	}
	return cpuset.New(stats.CPUSetStats.CPUs...), nil
}

func (c *cpusetManager) getCgroupPathsForTask(allocID, task string) (absolute, relative string) {
	return filepath.Join(c.reservedCpusetPath(), fmt.Sprintf("%s-%s", allocID, task)),
		filepath.Join(c.cgroupParent, ReservedCpusetCgroupName, fmt.Sprintf("%s-%s", allocID, task))
}

func (c *cpusetManager) sharedCpusetPath() string {
	return filepath.Join(c.cgroupParentPath, SharedCpusetCgroupName)
}

func (c *cpusetManager) reservedCpusetPath() string {
	return filepath.Join(c.cgroupParentPath, ReservedCpusetCgroupName)
}
<commit_msg>cgutil: set reserved mems on init even if already exist<commit_after>package cgutil

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/opencontainers/runc/libcontainer/cgroups"

	"github.com/hashicorp/go-hclog"

	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"

	"github.com/hashicorp/nomad/lib/cpuset"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	"github.com/opencontainers/runc/libcontainer/configs"

	"github.com/hashicorp/nomad/nomad/structs"
)

func NewCpusetManager(cgroupParent string, logger hclog.Logger) CpusetManager {
	if cgroupParent == "" {
		cgroupParent = DefaultCgroupParent
	}
	return &cpusetManager{
		cgroupParent: cgroupParent,
		cgroupInfo:   map[string]allocTaskCgroupInfo{},
		logger:       logger,
	}
}

var (
	cpusetReconcileInterval = 30 * time.Second
)

type cpusetManager struct {
	// cgroupParent relative to the cgroup root. ex. '/nomad'
	cgroupParent string
	// cgroupParentPath is the absolute path to the cgroup parent.
	cgroupParentPath string

	parentCpuset cpuset.CPUSet

	// all exported functions are synchronized
	mu sync.Mutex

	cgroupInfo map[string]allocTaskCgroupInfo

	doneCh   chan struct{}
	signalCh chan struct{}
	logger   hclog.Logger
}

func (c *cpusetManager) AddAlloc(alloc *structs.Allocation) {
	if alloc == nil || alloc.AllocatedResources == nil {
		return
	}
	allocInfo := allocTaskCgroupInfo{}
	for task, resources := range alloc.AllocatedResources.Tasks {
		taskCpuset := cpuset.New(resources.Cpu.ReservedCores...)
		cgroupPath := filepath.Join(c.cgroupParentPath, SharedCpusetCgroupName)
		relativeCgroupPath := filepath.Join(c.cgroupParent, SharedCpusetCgroupName)
		if taskCpuset.Size() > 0 {
			cgroupPath, relativeCgroupPath = c.getCgroupPathsForTask(alloc.ID, task)
		}
		allocInfo[task] = &TaskCgroupInfo{
			CgroupPath:         cgroupPath,
			RelativeCgroupPath: relativeCgroupPath,
			Cpuset:             taskCpuset,
		}
	}
	c.mu.Lock()
	c.cgroupInfo[alloc.ID] = allocInfo
	c.mu.Unlock()
	go c.signalReconcile()
}

func (c *cpusetManager) RemoveAlloc(allocID string) {
	c.mu.Lock()
	delete(c.cgroupInfo, allocID)
	c.mu.Unlock()
	go c.signalReconcile()
}

func (c *cpusetManager) CgroupPathFor(allocID, task string) CgroupPathGetter {
	return func(ctx context.Context) (string, error) {
		c.mu.Lock()
		allocInfo, ok := c.cgroupInfo[allocID]
		if !ok {
			c.mu.Unlock()
			return "", fmt.Errorf("alloc not found for id %q", allocID)
		}

		taskInfo, ok := allocInfo[task]
		c.mu.Unlock()
		if !ok {
			return "", fmt.Errorf("task %q not found", task)
		}

		for {
			if taskInfo.Error != nil {
				break
			}
			if _, err := os.Stat(taskInfo.CgroupPath); os.IsNotExist(err) {
				select {
				case <-ctx.Done():
					return taskInfo.CgroupPath, ctx.Err()
				case <-time.After(100 * time.Millisecond):
					continue
				}
			}
			break
		}

		return taskInfo.CgroupPath, taskInfo.Error
	}

}

type allocTaskCgroupInfo map[string]*TaskCgroupInfo

// Init checks that the cgroup parent and expected child cgroups have been created
// If the cgroup parent is set to /nomad then this will ensure that the /nomad/shared
// cgroup is initialized.
func (c *cpusetManager) Init() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	cgroupParentPath, err := getCgroupPathHelper("cpuset", c.cgroupParent)
	if err != nil {
		return err
	}
	c.cgroupParentPath = cgroupParentPath

	// ensures that shared cpuset exists and that the cpuset values are copied from the parent if created
	if err := cpusetEnsureParent(filepath.Join(cgroupParentPath, SharedCpusetCgroupName)); err != nil {
		return err
	}

	parentCpus, parentMems, err := getCpusetSubsystemSettings(cgroupParentPath)
	if err != nil {
		return fmt.Errorf("failed to detect parent cpuset settings: %v", err)
	}
	c.parentCpuset, err = cpuset.Parse(parentCpus)
	if err != nil {
		return fmt.Errorf("failed to parse parent cpuset.cpus setting: %v", err)
	}

	// ensure the reserved cpuset exists, but only copy the mems from the parent if creating the cgroup
	if err := os.Mkdir(filepath.Join(cgroupParentPath, ReservedCpusetCgroupName), 0755); err == nil {
		// cgroup created, leave cpuset.cpus empty but copy cpuset.mems from parent
		if err != nil {
			return err
		}
	} else if !os.IsExist(err) {
		return err
	}

	if err := fscommon.WriteFile(filepath.Join(cgroupParentPath, ReservedCpusetCgroupName), "cpuset.mems", parentMems); err != nil {
		return err
	}

	c.doneCh = make(chan struct{})
	c.signalCh = make(chan struct{})

	c.logger.Info("initialized cpuset cgroup manager", "parent", c.cgroupParent, "cpuset", c.parentCpuset.String())

	go c.reconcileLoop()
	return nil
}

func (c *cpusetManager) reconcileLoop() {
	timer := time.NewTimer(0)
	if !timer.Stop() {
		<-timer.C
	}
	defer timer.Stop()

	for {
		select {
		case <-c.doneCh:
			c.logger.Debug("shutting down reconcile loop")
			return
		case <-c.signalCh:
			timer.Reset(500 * time.Millisecond)
		case <-timer.C:
			c.reconcileCpusets()
			timer.Reset(cpusetReconcileInterval)
		}
	}
}

func (c *cpusetManager) reconcileCpusets() {
	c.mu.Lock()
	defer c.mu.Unlock()
	sharedCpuset := cpuset.New(c.parentCpuset.ToSlice()...)
	reservedCpuset := cpuset.New()
	taskCpusets := map[string]*TaskCgroupInfo{}
	for _, alloc := range c.cgroupInfo {
		for _, task := range alloc {
			if task.Cpuset.Size() == 0 {
				continue
			}
			sharedCpuset = sharedCpuset.Difference(task.Cpuset)
			reservedCpuset = reservedCpuset.Union(task.Cpuset)
			taskCpusets[task.CgroupPath] = task
		}
	}

	// look for reserved cpusets which we don't know about and remove
	files, err := ioutil.ReadDir(c.reservedCpusetPath())
	if err != nil {
		c.logger.Error("failed to list files in reserved cgroup path during reconciliation", "path", c.reservedCpusetPath(), "error", err)
	}
	for _, f := range files {
		if !f.IsDir() {
			continue
		}
		path := filepath.Join(c.reservedCpusetPath(), f.Name())
		if _, ok := taskCpusets[path]; ok {
			continue
		}
		c.logger.Debug("removing reserved cpuset cgroup", "path", path)
		err := cgroups.RemovePaths(map[string]string{"cpuset": path})
		if err != nil {
			c.logger.Error("removal of existing cpuset cgroup failed", "path", path, "error", err)
		}
	}

	if err := c.setCgroupCpusetCPUs(c.sharedCpusetPath(), sharedCpuset.String()); err != nil {
		c.logger.Error("could not write shared cpuset.cpus", "path", c.sharedCpusetPath(), "cpuset.cpus", sharedCpuset.String(), "error", err)
	}
	if err := c.setCgroupCpusetCPUs(c.reservedCpusetPath(), reservedCpuset.String()); err != nil {
		c.logger.Error("could not write reserved cpuset.cpus", "path", c.reservedCpusetPath(), "cpuset.cpus", reservedCpuset.String(), "error", err)
	}
	for _, info := range taskCpusets {
		if err := os.Mkdir(info.CgroupPath, 0755); err != nil && !os.IsExist(err) {
			c.logger.Error("failed to create new cgroup path for task", "path", info.CgroupPath, "error", err)
			info.Error = err
			continue
		}

		// copy cpuset.mems from parent
		_, parentMems, err := getCpusetSubsystemSettings(filepath.Dir(info.CgroupPath))
		if err != nil {
			c.logger.Error("failed to read parent cgroup settings for task", "path", info.CgroupPath, "error", err)
			info.Error = err
			continue
		}
		if err := fscommon.WriteFile(info.CgroupPath, "cpuset.mems", parentMems); err != nil {
			c.logger.Error("failed to write cgroup cpuset.mems setting for task", "path", info.CgroupPath, "mems", parentMems, "error", err)
			info.Error = err
			continue
		}
		if err := c.setCgroupCpusetCPUs(info.CgroupPath, info.Cpuset.String()); err != nil {
			c.logger.Error("failed to write cgroup cpuset.cpus settings for task", "path", info.CgroupPath, "cpus", info.Cpuset.String(), "error", err)
			info.Error = err
			continue
		}
	}
}

// setCgroupCpusetCPUs will compare an existing cpuset.cpus value with an expected value, overwriting the existing if different
// must hold a lock on cpusetManager.mu before calling
func (_ *cpusetManager) setCgroupCpusetCPUs(path, cpus string) error {
	currentCpusRaw, err := fscommon.ReadFile(path, "cpuset.cpus")
	if err != nil {
		return err
	}

	if cpus != strings.TrimSpace(currentCpusRaw) {
		if err := fscommon.WriteFile(path, "cpuset.cpus", cpus); err != nil {
			return err
		}
	}
	return nil
}

func (c *cpusetManager) signalReconcile() {
	select {
	case c.signalCh <- struct{}{}:
	case <-c.doneCh:
	}
}

func (c *cpusetManager) getCpuset(group string) (cpuset.CPUSet, error) {
	man := cgroupFs.NewManager(
		&configs.Cgroup{
			Path: filepath.Join(c.cgroupParent, group),
		},
		map[string]string{"cpuset": filepath.Join(c.cgroupParentPath, group)},
		false,
	)
	stats, err := man.GetStats()
	if err != nil {
		return cpuset.CPUSet{}, err
	}
	return cpuset.New(stats.CPUSetStats.CPUs...), nil
}

func (c *cpusetManager) getCgroupPathsForTask(allocID, task string) (absolute, relative string) {
	return filepath.Join(c.reservedCpusetPath(), fmt.Sprintf("%s-%s", allocID, task)),
		filepath.Join(c.cgroupParent, ReservedCpusetCgroupName, fmt.Sprintf("%s-%s", allocID, task))
}

func (c *cpusetManager) sharedCpusetPath() string {
	return filepath.Join(c.cgroupParentPath, SharedCpusetCgroupName)
}

func (c *cpusetManager) reservedCpusetPath() string {
	return filepath.Join(c.cgroupParentPath, ReservedCpusetCgroupName)
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	apitypes "github.com/emccode/libstorage/api/types"
)

func (c *CLI) initVolumeCmdsAndFlags() {
	c.initVolumeCmds()
	c.initVolumeFlags()
}

func (c *CLI) initVolumeCmds() {

	c.volumeCmd = &cobra.Command{
		Use:   "volume",
		Short: "The volume manager",
		Run: func(cmd *cobra.Command, args []string) {
			if isHelpFlags(cmd) {
				cmd.Usage()
			} else {
				c.volumeGetCmd.Run(c.volumeGetCmd, args)
			}
		},
	}
	c.c.AddCommand(c.volumeCmd)

	c.volumeMapCmd = &cobra.Command{
		Use:   "map",
		Short: "Print the volume mapping(s)",
		Run: func(cmd *cobra.Command, args []string) {

			allBlockDevices, err := c.r.Storage().Volumes(
				c.ctx, &apitypes.VolumesOpts{Attachments: true})
			if err != nil {
				log.Fatalf("Error: %s", err)
			}

			if len(allBlockDevices) > 0 {
				out, err := c.marshalOutput(&allBlockDevices)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Println(out)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumeMapCmd)

	c.volumeGetCmd = &cobra.Command{
		Use:     "get",
		Short:   "Get one or more volumes",
		Aliases: []string{"ls", "list"},
		Run: func(cmd *cobra.Command, args []string) {

			vols, err := c.r.Storage().Volumes(
				c.ctx, &apitypes.VolumesOpts{Attachments: false})
			if err != nil {
				log.Fatal(err)
			}
			if c.volumeID != "" || c.volumeName != "" {
				for _, v := range vols {
					if strings.ToLower(v.ID) == strings.ToLower(c.volumeID) ||
						strings.ToLower(v.Name) == strings.ToLower(c.volumeName) {
						out, err := c.marshalOutput(v)
						if err != nil {
							log.Fatal(err)
						}
						fmt.Println(out)
						return
					}
				}
				return
			}

			if len(vols) > 0 {
				out, err := c.marshalOutput(vols)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Println(out)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumeGetCmd)

	c.volumeCreateCmd = &cobra.Command{
		Use:     "create",
		Short:   "Create a new volume",
		Aliases: []string{"new"},
		Run: func(cmd *cobra.Command, args []string) {

			if c.size == 0 && c.snapshotID == "" && c.volumeID == "" {
				log.Fatalf("missing --size")
			}

			opts := &apitypes.VolumeCreateOpts{
				AvailabilityZone: &c.availabilityZone,
				Size:             &c.size,
				Type:             &c.volumeType,
				IOPS:             &c.iops,
				Opts:             store(),
			}

			var (
				err    error
				volume *apitypes.Volume
			)

			if c.volumeID != "" && c.volumeName != "" {
				volume, err = c.r.Storage().VolumeCopy(
					c.ctx, c.volumeID, c.volumeName, opts.Opts)
			} else if c.snapshotID != "" && c.volumeName != "" {
				volume, err = c.r.Storage().VolumeCreateFromSnapshot(
					c.ctx, c.snapshotID, c.volumeName, opts)
			} else if c.volumeName != "" {
				volume, err = c.r.Storage().VolumeCreate(
					c.ctx, c.volumeName, opts)
			}
			// TODO Get All Volumes
			if err != nil {
				log.Fatal(err)
			}

			out, err := c.marshalOutput(&volume)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(out)

		},
	}
	c.volumeCmd.AddCommand(c.volumeCreateCmd)

	c.volumeRemoveCmd = &cobra.Command{
		Use:     "remove",
		Short:   "Remove a volume",
		Aliases: []string{"rm"},
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeID == "" {
				log.Fatalf("missing --volumeid")
			}

			err := c.r.Storage().VolumeRemove(c.ctx, c.volumeID, store())
			if err != nil {
				log.Fatal(err)
			}

		},
	}
	c.volumeCmd.AddCommand(c.volumeRemoveCmd)

	c.volumeAttachCmd = &cobra.Command{
		Use:   "attach",
		Short: "Attach a volume",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeID == "" {
				log.Fatalf("missing --volumeid")
			}

			vol, _, err := c.r.Storage().VolumeAttach(
				c.ctx, c.volumeID,
				&apitypes.VolumeAttachOpts{Force: c.force})

			if err != nil {
				log.Fatal(err)
			}

			out, err := c.marshalOutput(vol)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(out)

		},
	}
	c.volumeCmd.AddCommand(c.volumeAttachCmd)

	c.volumeDetachCmd = &cobra.Command{
		Use:   "detach",
		Short: "Detach a volume",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeID == "" {
				log.Fatalf("missing --volumeid")
			}

			_, err := c.r.Storage().VolumeDetach(
				c.ctx, c.volumeID, &apitypes.VolumeDetachOpts{Force: c.force})
			if err != nil {
				log.Fatal(err)
			}

		},
	}
	c.volumeCmd.AddCommand(c.volumeDetachCmd)

	c.volumeMountCmd = &cobra.Command{
		Use:   "mount",
		Short: "Mount a volume",
		Run: func(cmd *cobra.Command, args []string) {
			if c.volumeName == "" && c.volumeID == "" {
				log.Fatal("Missing --volumename or --volumeid")
			}

			mountPath, _, err := c.r.Integration().Mount(
				c.ctx, c.volumeID, c.volumeName,
				&apitypes.VolumeMountOpts{
					NewFSType:   c.fsType,
					OverwriteFS: c.overwriteFs,
				})
			if err != nil {
				log.Fatal(err)
			}

			out, err := c.marshalOutput(&mountPath)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(out)

		},
	}
	c.volumeCmd.AddCommand(c.volumeMountCmd)

	c.volumeUnmountCmd = &cobra.Command{
		Use:   "unmount",
		Short: "Unmount a volume",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeName == "" && c.volumeID == "" {
				log.Fatal("Missing --volumename or --volumeid")
			}

			err := c.r.Integration().Unmount(
				c.ctx, c.volumeID, c.volumeName, store())
			if err != nil {
				log.Fatal(err)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumeUnmountCmd)

	c.volumePathCmd = &cobra.Command{
		Use:   "path",
		Short: "Print the volume path",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeName == "" && c.volumeID == "" {
				log.Fatal("Missing --volumename or --volumeid")
			}

			mountPath, err := c.r.Integration().Path(
				c.ctx, c.volumeID, c.volumeName, store())
			if err != nil {
				log.Fatal(err)
			}

			if mountPath != "" {
				out, err := c.marshalOutput(&mountPath)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Println(out)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumePathCmd)
}

func (c *CLI) initVolumeFlags() {
	c.volumeGetCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumeGetCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeCreateCmd.Flags().BoolVar(&c.runAsync, "runasync", false, "runasync")
	c.volumeCreateCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumeCreateCmd.Flags().StringVar(&c.volumeType, "volumetype", "", "volumetype")
	c.volumeCreateCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeCreateCmd.Flags().StringVar(&c.snapshotID, "snapshotid", "", "snapshotid")
	c.volumeCreateCmd.Flags().Int64Var(&c.iops, "iops", 0, "IOPS")
	c.volumeCreateCmd.Flags().Int64Var(&c.size, "size", 0, "size")
	c.volumeCreateCmd.Flags().StringVar(&c.availabilityZone, "availabilityzone", "", "availabilityzone")
	c.volumeRemoveCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeAttachCmd.Flags().BoolVar(&c.runAsync, "runasync", false, "runasync")
	c.volumeAttachCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeAttachCmd.Flags().StringVar(&c.instanceID, "instanceid", "", "instanceid")
	c.volumeAttachCmd.Flags().BoolVar(&c.force, "force", false, "force")
	c.volumeDetachCmd.Flags().BoolVar(&c.runAsync, "runasync", false, "runasync")
	c.volumeDetachCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeDetachCmd.Flags().StringVar(&c.instanceID, "instanceid", "", "instanceid")
	c.volumeDetachCmd.Flags().BoolVar(&c.force, "force", false, "force")
	c.volumeMountCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeMountCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumeMountCmd.Flags().BoolVar(&c.overwriteFs, "overwritefs", false, "overwritefs")
	c.volumeMountCmd.Flags().StringVar(&c.fsType, "fstype", "", "fstype")
	c.volumeUnmountCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeUnmountCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumePathCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumePathCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")

	c.addOutputFormatFlag(c.volumeCmd.Flags())
	c.addOutputFormatFlag(c.volumeGetCmd.Flags())
	c.addOutputFormatFlag(c.volumeCreateCmd.Flags())
	c.addOutputFormatFlag(c.volumeAttachCmd.Flags())
	c.addOutputFormatFlag(c.volumeMountCmd.Flags())
	c.addOutputFormatFlag(c.volumePathCmd.Flags())
	c.addOutputFormatFlag(c.volumeMapCmd.Flags())
}
<commit_msg>Fixed cli commands to for nil exceptions<commit_after>package cli

import (
	"fmt"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	apitypes "github.com/emccode/libstorage/api/types"
)

func (c *CLI) initVolumeCmdsAndFlags() {
	c.initVolumeCmds()
	c.initVolumeFlags()
}

func (c *CLI) initVolumeCmds() {

	c.volumeCmd = &cobra.Command{
		Use:   "volume",
		Short: "The volume manager",
		Run: func(cmd *cobra.Command, args []string) {
			if isHelpFlags(cmd) {
				cmd.Usage()
			} else {
				c.volumeGetCmd.Run(c.volumeGetCmd, args)
			}
		},
	}
	c.c.AddCommand(c.volumeCmd)

	c.volumeMapCmd = &cobra.Command{
		Use:   "map",
		Short: "Print the volume mapping(s)",
		Run: func(cmd *cobra.Command, args []string) {

			allBlockDevices, err := c.r.Storage().Volumes(
				c.ctx, &apitypes.VolumesOpts{Attachments: true})
			if err != nil {
				log.Fatalf("Error: %s", err)
			}

			if len(allBlockDevices) > 0 {
				out, err := c.marshalOutput(&allBlockDevices)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Println(out)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumeMapCmd)

	c.volumeGetCmd = &cobra.Command{
		Use:     "get",
		Short:   "Get one or more volumes",
		Aliases: []string{"ls", "list"},
		Run: func(cmd *cobra.Command, args []string) {

			vols, err := c.r.Storage().Volumes(
				c.ctx, &apitypes.VolumesOpts{Attachments: false})
			if err != nil {
				log.Fatal(err)
			}
			if c.volumeID != "" || c.volumeName != "" {
				for _, v := range vols {
					if strings.ToLower(v.ID) == strings.ToLower(c.volumeID) ||
						strings.ToLower(v.Name) == strings.ToLower(c.volumeName) {
						out, err := c.marshalOutput(v)
						if err != nil {
							log.Fatal(err)
						}
						fmt.Println(out)
						return
					}
				}
				return
			}

			if len(vols) > 0 {
				out, err := c.marshalOutput(vols)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Println(out)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumeGetCmd)

	c.volumeCreateCmd = &cobra.Command{
		Use:     "create",
		Short:   "Create a new volume",
		Aliases: []string{"new"},
		Run: func(cmd *cobra.Command, args []string) {

			if c.size == 0 && c.snapshotID == "" && c.volumeID == "" {
				log.Fatalf("missing --size")
			}

			opts := &apitypes.VolumeCreateOpts{
				AvailabilityZone: &c.availabilityZone,
				Size:             &c.size,
				Type:             &c.volumeType,
				IOPS:             &c.iops,
				Opts:             store(),
			}

			var (
				err    error
				volume *apitypes.Volume
			)

			if c.volumeID != "" && c.volumeName != "" {
				volume, err = c.r.Storage().VolumeCopy(
					c.ctx, c.volumeID, c.volumeName, opts.Opts)
			} else if c.snapshotID != "" && c.volumeName != "" {
				volume, err = c.r.Storage().VolumeCreateFromSnapshot(
					c.ctx, c.snapshotID, c.volumeName, opts)
			} else if c.volumeName != "" {
				volume, err = c.r.Storage().VolumeCreate(
					c.ctx, c.volumeName, opts)
			}
			// TODO Get All Volumes
			if err != nil {
				log.Fatal(err)
			}

			out, err := c.marshalOutput(&volume)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(out)

		},
	}
	c.volumeCmd.AddCommand(c.volumeCreateCmd)

	c.volumeRemoveCmd = &cobra.Command{
		Use:     "remove",
		Short:   "Remove a volume",
		Aliases: []string{"rm"},
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeID == "" {
				log.Fatalf("missing --volumeid")
			}

			err := c.r.Storage().VolumeRemove(c.ctx, c.volumeID, store())
			if err != nil {
				log.Fatal(err)
			}

		},
	}
	c.volumeCmd.AddCommand(c.volumeRemoveCmd)

	c.volumeAttachCmd = &cobra.Command{
		Use:   "attach",
		Short: "Attach a volume",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeID == "" {
				log.Fatalf("missing --volumeid")
			}

			vol, _, err := c.r.Storage().VolumeAttach(
				c.ctx, c.volumeID,
				&apitypes.VolumeAttachOpts{
					Force: c.force,
					Opts:  store(),
				})

			if err != nil {
				log.Fatal(err)
			}

			out, err := c.marshalOutput(vol)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(out)

		},
	}
	c.volumeCmd.AddCommand(c.volumeAttachCmd)

	c.volumeDetachCmd = &cobra.Command{
		Use:   "detach",
		Short: "Detach a volume",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeID == "" {
				log.Fatalf("missing --volumeid")
			}

			_, err := c.r.Storage().VolumeDetach(
				c.ctx, c.volumeID, &apitypes.VolumeDetachOpts{
					Force: c.force,
					Opts:  store(),
				})
			if err != nil {
				log.Fatal(err)
			}

		},
	}
	c.volumeCmd.AddCommand(c.volumeDetachCmd)

	c.volumeMountCmd = &cobra.Command{
		Use:   "mount",
		Short: "Mount a volume",
		Run: func(cmd *cobra.Command, args []string) {
			if c.volumeName == "" && c.volumeID == "" {
				log.Fatal("Missing --volumename or --volumeid")
			}

			mountPath, _, err := c.r.Integration().Mount(
				c.ctx, c.volumeID, c.volumeName,
				&apitypes.VolumeMountOpts{
					NewFSType:   c.fsType,
					OverwriteFS: c.overwriteFs,
				})
			if err != nil {
				log.Fatal(err)
			}

			out, err := c.marshalOutput(&mountPath)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(out)

		},
	}
	c.volumeCmd.AddCommand(c.volumeMountCmd)

	c.volumeUnmountCmd = &cobra.Command{
		Use:   "unmount",
		Short: "Unmount a volume",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeName == "" && c.volumeID == "" {
				log.Fatal("Missing --volumename or --volumeid")
			}

			err := c.r.Integration().Unmount(
				c.ctx, c.volumeID, c.volumeName, store())
			if err != nil {
				log.Fatal(err)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumeUnmountCmd)

	c.volumePathCmd = &cobra.Command{
		Use:   "path",
		Short: "Print the volume path",
		Run: func(cmd *cobra.Command, args []string) {

			if c.volumeName == "" && c.volumeID == "" {
				log.Fatal("Missing --volumename or --volumeid")
			}

			mountPath, err := c.r.Integration().Path(
				c.ctx, c.volumeID, c.volumeName, store())
			if err != nil {
				log.Fatal(err)
			}

			if mountPath != "" {
				out, err := c.marshalOutput(&mountPath)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Println(out)
			}
		},
	}
	c.volumeCmd.AddCommand(c.volumePathCmd)
}

func (c *CLI) initVolumeFlags() {
	c.volumeGetCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumeGetCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeCreateCmd.Flags().BoolVar(&c.runAsync, "runasync", false, "runasync")
	c.volumeCreateCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumeCreateCmd.Flags().StringVar(&c.volumeType, "volumetype", "", "volumetype")
	c.volumeCreateCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeCreateCmd.Flags().StringVar(&c.snapshotID, "snapshotid", "", "snapshotid")
	c.volumeCreateCmd.Flags().Int64Var(&c.iops, "iops", 0, "IOPS")
	c.volumeCreateCmd.Flags().Int64Var(&c.size, "size", 0, "size")
	c.volumeCreateCmd.Flags().StringVar(&c.availabilityZone, "availabilityzone", "", "availabilityzone")
	c.volumeRemoveCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeAttachCmd.Flags().BoolVar(&c.runAsync, "runasync", false, "runasync")
	c.volumeAttachCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeAttachCmd.Flags().StringVar(&c.instanceID, "instanceid", "", "instanceid")
	c.volumeAttachCmd.Flags().BoolVar(&c.force, "force", false, "force")
	c.volumeDetachCmd.Flags().BoolVar(&c.runAsync, "runasync", false, "runasync")
	c.volumeDetachCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeDetachCmd.Flags().StringVar(&c.instanceID, "instanceid", "", "instanceid")
	c.volumeDetachCmd.Flags().BoolVar(&c.force, "force", false, "force")
	c.volumeMountCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeMountCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumeMountCmd.Flags().BoolVar(&c.overwriteFs, "overwritefs", false, "overwritefs")
	c.volumeMountCmd.Flags().StringVar(&c.fsType, "fstype", "", "fstype")
	c.volumeUnmountCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumeUnmountCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")
	c.volumePathCmd.Flags().StringVar(&c.volumeID, "volumeid", "", "volumeid")
	c.volumePathCmd.Flags().StringVar(&c.volumeName, "volumename", "", "volumename")

	c.addOutputFormatFlag(c.volumeCmd.Flags())
	c.addOutputFormatFlag(c.volumeGetCmd.Flags())
	c.addOutputFormatFlag(c.volumeCreateCmd.Flags())
	c.addOutputFormatFlag(c.volumeAttachCmd.Flags())
	c.addOutputFormatFlag(c.volumeMountCmd.Flags())
	c.addOutputFormatFlag(c.volumePathCmd.Flags())
	c.addOutputFormatFlag(c.volumeMapCmd.Flags())
}
<|endoftext|>


<commit_before>package abstract_torrent_client

import (
	"encoding/json"
	"net/http"
)

type Server struct {
}

func (server *Server) Start(client TorrentClient) {
	http.HandleFunc("/magnet/", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodPost {
			magnet := r.PostForm.Get("magnet")
			data := client.HandleMagnet(magnet)
			str, _ := json.Marshal(data)
			w.Write(str)
		}
	})
	err := http.ListenAndServe(":80", nil)

	if err != nil {
		panic(err)
	}
}
<commit_msg>Changed request handling params<commit_after>package abstract_torrent_client

import (
	"encoding/json"
	"net/http"
)

type Server struct {
}

type magnet struct {
	Magnet string `json:"magnet"`
}

func (server *Server) Start(client TorrentClient) {
	http.HandleFunc("/magnet/", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodPost {
			decoder := json.NewDecoder(r.Body)
			magnet := magnet{}

			decoder.Decode(&magnet)
			data := client.HandleMagnet(magnet.Magnet)
			str, _ := json.Marshal(data)
			w.Write(str)
		}
	})
	err := http.ListenAndServe(":80", nil)

	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package filesys

import (
	"bytes"
	"io"
	"math"

	"github.com/chrislusf/seaweedfs/weed/glog"
)

type IntervalNode struct {
	Data   []byte
	Offset int64
	Size   int64
	Next   *IntervalNode
}

type IntervalLinkedList struct {
	Head *IntervalNode
	Tail *IntervalNode
}

type ContinuousIntervals struct {
	lists []*IntervalLinkedList
}

func (list *IntervalLinkedList) Offset() int64 {
	return list.Head.Offset
}
func (list *IntervalLinkedList) Size() int64 {
	return list.Tail.Offset + list.Tail.Size - list.Head.Offset
}
func (list *IntervalLinkedList) addNodeToTail(node *IntervalNode) {
	// glog.V(0).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
	list.Tail.Next = node
	list.Tail = node
}
func (list *IntervalLinkedList) addNodeToHead(node *IntervalNode) {
	// glog.V(0).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
	node.Next = list.Head
	list.Head = node
}

func (list *IntervalLinkedList) ReadData(buf []byte, start, stop int64) {
	t := list.Head
	for {

		nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size)
		if nodeStart < nodeStop {
			glog.V(0).Infof("copying start=%d stop=%d t=[%d,%d) t.data=%d => bufSize=%d nodeStart=%d, nodeStop=%d",
				start, stop, t.Offset, t.Offset+t.Size, len(t.Data),
				len(buf), nodeStart, nodeStop)
			copy(buf[nodeStart-start:], t.Data[nodeStart-t.Offset:nodeStop-t.Offset])
		}

		if t.Next == nil {
			break
		}
		t = t.Next
	}
}

func (c *ContinuousIntervals) TotalSize() (total int64) {
	for _, list := range c.lists {
		total += list.Size()
	}
	return
}

func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) (hasOverlap bool) {
	interval := &IntervalNode{Data: data, Offset: offset, Size: int64(len(data))}

	var prevList, nextList *IntervalLinkedList

	for _, list := range c.lists {
		if list.Head.Offset == interval.Offset+interval.Size {
			nextList = list
			break
		}
	}

	for _, list := range c.lists {
		if list.Head.Offset+list.Size() == offset {
			list.addNodeToTail(interval)
			prevList = list
			break
		}
		if list.Head.Offset <= offset && offset < list.Head.Offset+list.Size() {
			if list.Tail.Offset <= offset {
				dataStartIndex := list.Tail.Offset + list.Tail.Size - offset
				// glog.V(4).Infof("overlap data new [0,%d) same=%v", dataStartIndex, bytes.Compare(interval.Data[0:dataStartIndex], list.Tail.Data[len(list.Tail.Data)-int(dataStartIndex):]))
				interval.Data = interval.Data[dataStartIndex:]
				interval.Size -= dataStartIndex
				interval.Offset = offset + dataStartIndex
				// glog.V(4).Infof("overlapping append as [%d,%d) dataSize=%d", interval.Offset, interval.Offset+interval.Size, len(interval.Data))
				list.addNodeToTail(interval)
				prevList = list
				break
			}
			glog.V(4).Infof("overlapped! interval is [%d,%d) dataSize=%d", interval.Offset, interval.Offset+interval.Size, len(interval.Data))
			hasOverlap = true
			return
		}
	}

	if prevList != nil && nextList != nil {
		// glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size)
		prevList.Tail.Next = nextList.Head
		prevList.Tail = nextList.Tail
		c.removeList(nextList)
	} else if nextList != nil {
		// add to head was not done when checking
		nextList.addNodeToHead(interval)
	}
	if prevList == nil && nextList == nil {
		c.lists = append(c.lists, &IntervalLinkedList{
			Head: interval,
			Tail: interval,
		})
	}

	return
}

func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList {
	var maxSize int64
	maxIndex := -1
	for k, list := range c.lists {
		if maxSize <= list.Size() {
			maxSize = list.Size()
			maxIndex = k
		}
	}
	if maxSize <= 0 {
		return nil
	}

	t := c.lists[maxIndex]
	c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...)
	return t

}

func (c *ContinuousIntervals) removeList(target *IntervalLinkedList) {
	index := -1
	for k, list := range c.lists {
		if list.Offset() == target.Offset() {
			index = k
		}
	}
	if index < 0 {
		return
	}

	c.lists = append(c.lists[0:index], c.lists[index+1:]...)

}

func (c *ContinuousIntervals) ReadData(data []byte, startOffset int64) (offset int64, size int) {
	var minOffset int64 = math.MaxInt64
	var maxStop int64
	for _, list := range c.lists {
		start := max(startOffset, list.Offset())
		stop := min(startOffset+int64(len(data)), list.Offset()+list.Size())
		if start <= stop {
			list.ReadData(data[start-startOffset:], start, stop)
			minOffset = min(minOffset, start)
			maxStop = max(maxStop, stop)
		}
	}

	if minOffset == math.MaxInt64 {
		return 0, 0
	}

	offset = minOffset
	size = int(maxStop - offset)
	return
}

func (l *IntervalLinkedList) ToReader() io.Reader {
	var readers []io.Reader
	t := l.Head
	readers = append(readers, bytes.NewReader(t.Data))
	for t.Next != nil {
		t = t.Next
		readers = append(readers, bytes.NewReader(t.Data))
	}
	return io.MultiReader(readers...)
}
<commit_msg>reduce log<commit_after>package filesys

import (
	"bytes"
	"io"
	"math"

	"github.com/chrislusf/seaweedfs/weed/glog"
)

type IntervalNode struct {
	Data   []byte
	Offset int64
	Size   int64
	Next   *IntervalNode
}

type IntervalLinkedList struct {
	Head *IntervalNode
	Tail *IntervalNode
}

type ContinuousIntervals struct {
	lists []*IntervalLinkedList
}

func (list *IntervalLinkedList) Offset() int64 {
	return list.Head.Offset
}
func (list *IntervalLinkedList) Size() int64 {
	return list.Tail.Offset + list.Tail.Size - list.Head.Offset
}
func (list *IntervalLinkedList) addNodeToTail(node *IntervalNode) {
	// glog.V(0).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
	list.Tail.Next = node
	list.Tail = node
}
func (list *IntervalLinkedList) addNodeToHead(node *IntervalNode) {
	// glog.V(0).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
	node.Next = list.Head
	list.Head = node
}

func (list *IntervalLinkedList) ReadData(buf []byte, start, stop int64) {
	t := list.Head
	for {

		nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size)
		if nodeStart < nodeStop {
			// glog.V(0).Infof("copying start=%d stop=%d t=[%d,%d) t.data=%d => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.Offset, t.Offset+t.Size, len(t.Data), len(buf), nodeStart, nodeStop)
			copy(buf[nodeStart-start:], t.Data[nodeStart-t.Offset:nodeStop-t.Offset])
		}

		if t.Next == nil {
			break
		}
		t = t.Next
	}
}

func (c *ContinuousIntervals) TotalSize() (total int64) {
	for _, list := range c.lists {
		total += list.Size()
	}
	return
}

func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) (hasOverlap bool) {
	interval := &IntervalNode{Data: data, Offset: offset, Size: int64(len(data))}

	var prevList, nextList *IntervalLinkedList

	for _, list := range c.lists {
		if list.Head.Offset == interval.Offset+interval.Size {
			nextList = list
			break
		}
	}

	for _, list := range c.lists {
		if list.Head.Offset+list.Size() == offset {
			list.addNodeToTail(interval)
			prevList = list
			break
		}
		if list.Head.Offset <= offset && offset < list.Head.Offset+list.Size() {
			if list.Tail.Offset <= offset {
				dataStartIndex := list.Tail.Offset + list.Tail.Size - offset
				// glog.V(4).Infof("overlap data new [0,%d) same=%v", dataStartIndex, bytes.Compare(interval.Data[0:dataStartIndex], list.Tail.Data[len(list.Tail.Data)-int(dataStartIndex):]))
				interval.Data = interval.Data[dataStartIndex:]
				interval.Size -= dataStartIndex
				interval.Offset = offset + dataStartIndex
				// glog.V(4).Infof("overlapping append as [%d,%d) dataSize=%d", interval.Offset, interval.Offset+interval.Size, len(interval.Data))
				list.addNodeToTail(interval)
				prevList = list
				break
			}
			glog.V(4).Infof("overlapped! interval is [%d,%d) dataSize=%d", interval.Offset, interval.Offset+interval.Size, len(interval.Data))
			hasOverlap = true
			return
		}
	}

	if prevList != nil && nextList != nil {
		// glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size)
		prevList.Tail.Next = nextList.Head
		prevList.Tail = nextList.Tail
		c.removeList(nextList)
	} else if nextList != nil {
		// add to head was not done when checking
		nextList.addNodeToHead(interval)
	}
	if prevList == nil && nextList == nil {
		c.lists = append(c.lists, &IntervalLinkedList{
			Head: interval,
			Tail: interval,
		})
	}

	return
}

func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList {
	var maxSize int64
	maxIndex := -1
	for k, list := range c.lists {
		if maxSize <= list.Size() {
			maxSize = list.Size()
			maxIndex = k
		}
	}
	if maxSize <= 0 {
		return nil
	}

	t := c.lists[maxIndex]
	c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...)
	return t

}

func (c *ContinuousIntervals) removeList(target *IntervalLinkedList) {
	index := -1
	for k, list := range c.lists {
		if list.Offset() == target.Offset() {
			index = k
		}
	}
	if index < 0 {
		return
	}

	c.lists = append(c.lists[0:index], c.lists[index+1:]...)

}

func (c *ContinuousIntervals) ReadData(data []byte, startOffset int64) (offset int64, size int) {
	var minOffset int64 = math.MaxInt64
	var maxStop int64
	for _, list := range c.lists {
		start := max(startOffset, list.Offset())
		stop := min(startOffset+int64(len(data)), list.Offset()+list.Size())
		if start <= stop {
			list.ReadData(data[start-startOffset:], start, stop)
			minOffset = min(minOffset, start)
			maxStop = max(maxStop, stop)
		}
	}

	if minOffset == math.MaxInt64 {
		return 0, 0
	}

	offset = minOffset
	size = int(maxStop - offset)
	return
}

func (l *IntervalLinkedList) ToReader() io.Reader {
	var readers []io.Reader
	t := l.Head
	readers = append(readers, bytes.NewReader(t.Data))
	for t.Next != nil {
		t = t.Next
		readers = append(readers, bytes.NewReader(t.Data))
	}
	return io.MultiReader(readers...)
}
<|endoftext|>


<commit_before>package s3api

import (
	"crypto/md5"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gorilla/mux"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {

	// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html

	bucket, object := getBucketAndObject(r)

	_, err := validateContentMd5(r.Header)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL)
		return
	}

	dataReader := r.Body
	if s3a.iam.isEnabled() {
		rAuthType := getRequestAuthType(r)
		var s3ErrCode s3err.ErrorCode
		switch rAuthType {
		case authTypeStreamingSigned:
			dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
		case authTypeSignedV2, authTypePresignedV2:
			_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
		case authTypePresigned, authTypeSigned:
			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
		}
		if s3ErrCode != s3err.ErrNone {
			writeErrorResponse(w, s3ErrCode, r.URL)
			return
		}
	}
	defer dataReader.Close()

	if strings.HasSuffix(object, "/") {
		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil {
			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
			return
		}
	} else {
		uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

		etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)

		if errCode != s3err.ErrNone {
			writeErrorResponse(w, errCode, r.URL)
			return
		}

		setEtag(w, etag)
	}

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	if strings.HasSuffix(r.URL.Path, "/") {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	response, _ := s3a.listFilerEntries(bucket, object, 1, "", "/")
	if len(response.Contents) != 0 && strings.HasSuffix(object, "/") {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) {
		for k, v := range proxyResponse.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(http.StatusNoContent)
	})
}

// / ObjectIdentifier carries key name for the object to delete.
type ObjectIdentifier struct {
	ObjectName string `xml:"Key"`
}

// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
	// Element to enable quiet mode for the request
	Quiet bool
	// List of objects to be deleted
	Objects []ObjectIdentifier `xml:"Object"`
}

// DeleteError structure.
type DeleteError struct {
	Code    string
	Message string
	Key     string
}

// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`

	// Collection of all deleted objects
	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`

	// Collection of errors deleting certain objects.
	Errors []DeleteError `xml:"Error,omitempty"`
}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	deleteXMLBytes, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	deleteObjects := &DeleteObjectsRequest{}
	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
		writeErrorResponse(w, s3err.ErrMalformedXML, r.URL)
		return
	}

	var deletedObjects []ObjectIdentifier
	var deleteErrors []DeleteError

	s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		for _, object := range deleteObjects.Objects {
			response, _ := s3a.listFilerEntries(bucket, object.ObjectName, 1, "", "/")
			if len(response.Contents) != 0 && strings.HasSuffix(object.ObjectName, "/") {
				continue
			}

			lastSeparator := strings.LastIndex(object.ObjectName, "/")
			parentDirectoryPath, entryName, isDeleteData, isRecursive := "/", object.ObjectName, true, true
			if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) {
				entryName = object.ObjectName[lastSeparator+1:]
				parentDirectoryPath = "/" + object.ObjectName[:lastSeparator]
			}
			parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)

			err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
			if err == nil {
				deletedObjects = append(deletedObjects, object)
			} else {
				deleteErrors = append(deleteErrors, DeleteError{
					Code:    "",
					Message: err.Error(),
					Key:     object.ObjectName,
				})
			}
		}
		return nil
	})

	deleteResp := DeleteObjectsResponse{}
	if !deleteObjects.Quiet {
		deleteResp.DeletedObjects = deletedObjects
	}
	deleteResp.Errors = deleteErrors

	writeSuccessResponseXML(w, encodeResponse(deleteResp))

}

var passThroughHeaders = []string{
	"response-cache-control",
	"response-content-disposition",
	"response-content-encoding",
	"response-content-language",
	"response-content-type",
	"response-expires",
}

func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter)) {

	glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)

	proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", destUrl, err)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		// handle s3 related headers
		passed := false
		for _, h := range passThroughHeaders {
			if strings.ToLower(header) == h && len(values) > 0 {
				proxyReq.Header.Add(header[len("response-"):], values[0])
				passed = true
				break
			}
		}
		if passed {
			continue
		}
		// handle other headers
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if resp.ContentLength == -1 {
		writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
		return
	}

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	defer util.CloseResponse(resp)

	responseFn(resp, w)

}

func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
	for k, v := range proxyResponse.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(proxyResponse.StatusCode)
	io.Copy(w, proxyResponse.Body)
}

func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) {

	hash := md5.New()
	var body = io.TeeReader(dataReader, hash)

	proxyReq, err := http.NewRequest("PUT", uploadUrl, body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", uploadUrl, err)
		return "", s3err.ErrInternalError
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		return "", s3err.ErrInternalError
	}
	defer resp.Body.Close()

	etag = fmt.Sprintf("%x", hash.Sum(nil))

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.Errorf("upload to filer response read: %v", ra_err)
		return etag, s3err.ErrInternalError
	}
	var ret weed_server.FilerPostResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body))
		return "", s3err.ErrInternalError
	}
	if ret.Error != "" {
		glog.Errorf("upload to filer error: %v", ret.Error)
		return "", s3err.ErrInternalError
	}

	return etag, s3err.ErrNone
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getBucketAndObject(r *http.Request) (bucket, object string) {
	vars := mux.Vars(r)
	bucket = vars["bucket"]
	object = vars["object"]
	if !strings.HasPrefix(object, "/") {
		object = "/" + object
	}

	return
}
<commit_msg>s3: 1.fix spark reading S3 directory wildcard problem 2.fix the problem of the spark history service writing S3 directory<commit_after>package s3api

import (
	"crypto/md5"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gorilla/mux"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {

	// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html

	bucket, object := getBucketAndObject(r)

	_, err := validateContentMd5(r.Header)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL)
		return
	}

	dataReader := r.Body
	if s3a.iam.isEnabled() {
		rAuthType := getRequestAuthType(r)
		var s3ErrCode s3err.ErrorCode
		switch rAuthType {
		case authTypeStreamingSigned:
			dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
		case authTypeSignedV2, authTypePresignedV2:
			_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
		case authTypePresigned, authTypeSigned:
			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
		}
		if s3ErrCode != s3err.ErrNone {
			writeErrorResponse(w, s3ErrCode, r.URL)
			return
		}
	}
	defer dataReader.Close()

	if strings.HasSuffix(object, "/") {
		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil {
			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
			return
		}
	} else {
		uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

		etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)

		if errCode != s3err.ErrNone {
			writeErrorResponse(w, errCode, r.URL)
			return
		}

		setEtag(w, etag)
	}

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	if strings.HasSuffix(r.URL.Path, "/") {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	response, _ := s3a.listFilerEntries(bucket, object, 1, "", "/")
	if len(response.Contents) != 0 && strings.HasSuffix(object, "/") {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) {
		for k, v := range proxyResponse.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(http.StatusNoContent)
	})
}

// / ObjectIdentifier carries key name for the object to delete.
type ObjectIdentifier struct {
	ObjectName string `xml:"Key"`
}

// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
	// Element to enable quiet mode for the request
	Quiet bool
	// List of objects to be deleted
	Objects []ObjectIdentifier `xml:"Object"`
}

// DeleteError structure.
type DeleteError struct {
	Code    string
	Message string
	Key     string
}

// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`

	// Collection of all deleted objects
	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`

	// Collection of errors deleting certain objects.
	Errors []DeleteError `xml:"Error,omitempty"`
}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	deleteXMLBytes, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	deleteObjects := &DeleteObjectsRequest{}
	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
		writeErrorResponse(w, s3err.ErrMalformedXML, r.URL)
		return
	}

	var deletedObjects []ObjectIdentifier
	var deleteErrors []DeleteError

	s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		for _, object := range deleteObjects.Objects {
			response, _ := s3a.listFilerEntries(bucket, object.ObjectName, 1, "", "/")
			if len(response.Contents) != 0 && strings.HasSuffix(object.ObjectName, "/") {
				continue
			}

			lastSeparator := strings.LastIndex(object.ObjectName, "/")
			parentDirectoryPath, entryName, isDeleteData, isRecursive := "/", object.ObjectName, true, true
			if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) {
				entryName = object.ObjectName[lastSeparator+1:]
				parentDirectoryPath = "/" + object.ObjectName[:lastSeparator]
			}
			parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)

			err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
			if err == nil {
				deletedObjects = append(deletedObjects, object)
			} else {
				deleteErrors = append(deleteErrors, DeleteError{
					Code:    "",
					Message: err.Error(),
					Key:     object.ObjectName,
				})
			}
		}
		return nil
	})

	deleteResp := DeleteObjectsResponse{}
	if !deleteObjects.Quiet {
		deleteResp.DeletedObjects = deletedObjects
	}
	deleteResp.Errors = deleteErrors

	writeSuccessResponseXML(w, encodeResponse(deleteResp))

}

var passThroughHeaders = []string{
	"response-cache-control",
	"response-content-disposition",
	"response-content-encoding",
	"response-content-language",
	"response-content-type",
	"response-expires",
}

func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter)) {

	glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)

	proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", destUrl, err)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		// handle s3 related headers
		passed := false
		for _, h := range passThroughHeaders {
			if strings.ToLower(header) == h && len(values) > 0 {
				proxyReq.Header.Add(header[len("response-"):], values[0])
				passed = true
				break
			}
		}
		if passed {
			continue
		}
		// handle other headers
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if resp.ContentLength == -1 && !strings.HasSuffix(destUrl, "/") {
		writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
		return
	}

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	defer util.CloseResponse(resp)

	responseFn(resp, w)

}

func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
	for k, v := range proxyResponse.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(proxyResponse.StatusCode)
	io.Copy(w, proxyResponse.Body)
}

func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) {

	hash := md5.New()
	var body = io.TeeReader(dataReader, hash)

	proxyReq, err := http.NewRequest("PUT", uploadUrl, body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", uploadUrl, err)
		return "", s3err.ErrInternalError
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		return "", s3err.ErrInternalError
	}
	defer resp.Body.Close()

	etag = fmt.Sprintf("%x", hash.Sum(nil))

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.Errorf("upload to filer response read: %v", ra_err)
		return etag, s3err.ErrInternalError
	}
	var ret weed_server.FilerPostResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body))
		return "", s3err.ErrInternalError
	}
	if ret.Error != "" {
		glog.Errorf("upload to filer error: %v", ret.Error)
		return "", s3err.ErrInternalError
	}

	return etag, s3err.ErrNone
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getBucketAndObject(r *http.Request) (bucket, object string) {
	vars := mux.Vars(r)
	bucket = vars["bucket"]
	object = vars["object"]
	if !strings.HasPrefix(object, "/") {
		object = "/" + object
	}

	return
}
<|endoftext|>