<commit_before>// Copyright © 2017 Aeneas Rekkas <aeneas+oss@aeneas.io>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"fmt"
	"net/url"

	"github.com/gorilla/sessions"
	"github.com/julienschmidt/httprouter"
	"github.com/ory/fosite"
	"github.com/ory/fosite/compose"
	"github.com/ory/herodot"
	"github.com/ory/hydra/client"
	"github.com/ory/hydra/config"
	"github.com/ory/hydra/jwk"
	"github.com/ory/hydra/oauth2"
	"github.com/ory/hydra/pkg"
	"github.com/ory/hydra/warden"
)

func injectFositeStore(c *config.Config, clients client.Manager) {
	var ctx = c.Context()
	var store pkg.FositeStorer

	switch con := ctx.Connection.(type) {
	case *config.MemoryConnection:
		store = oauth2.NewFositeMemoryStore(clients, c.GetAccessTokenLifespan())
		break
	case *config.SQLConnection:
		store = oauth2.NewFositeSQLStore(clients, con.GetDatabase(), c.GetLogger(), c.GetAccessTokenLifespan())
		break
	case *config.PluginConnection:
		var err error
		if store, err = con.NewOAuth2Manager(clients); err != nil {
			c.GetLogger().Fatalf("Could not load client manager plugin %s", err)
		}
		break
	default:
		panic("Unknown connection type.")
	}

	ctx.FositeStore = store
}

func newOAuth2Provider(c *config.Config) (fosite.OAuth2Provider, string) {
	var ctx = c.Context()
	var store = ctx.FositeStore

	privateKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "private")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch private signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	publicKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "public")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch public signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	fc := &compose.Config{
		AccessTokenLifespan:            c.GetAccessTokenLifespan(),
		AuthorizeCodeLifespan:          c.GetAuthCodeLifespan(),
		IDTokenLifespan:                c.GetIDTokenLifespan(),
		HashCost:                       c.BCryptWorkFactor,
		ScopeStrategy:                  c.GetScopeStrategy(),
		SendDebugMessagesToClients:     c.SendOAuth2DebugMessagesToClients,
		EnforcePKCE:                    false,
		EnablePKCEPlainChallengeMethod: false,
	}

	return compose.Compose(
		fc,
		store,
		&compose.CommonStrategy{
			CoreStrategy:               compose.NewOAuth2HMACStrategy(fc, c.GetSystemSecret()),
			OpenIDConnectTokenStrategy: compose.NewOpenIDConnectStrategy(jwk.MustRSAPrivate(privateKey)),
		},
		nil,
		compose.OAuth2AuthorizeExplicitFactory,
		compose.OAuth2AuthorizeImplicitFactory,
		compose.OAuth2ClientCredentialsGrantFactory,
		compose.OAuth2RefreshTokenGrantFactory,
		compose.OAuth2PKCEFactory,
		compose.OpenIDConnectExplicitFactory,
		compose.OpenIDConnectHybridFactory,
		compose.OpenIDConnectImplicitFactory,
		compose.OAuth2TokenRevocationFactory,
		warden.OAuth2TokenIntrospectionFactory,
	), publicKey.KeyID
}

func newOAuth2Handler(c *config.Config, router *httprouter.Router, cm oauth2.ConsentRequestManager, o fosite.OAuth2Provider, idTokenKeyID string) *oauth2.Handler {
	if c.ConsentURL == "" {
		proto := "https"
		if c.ForceHTTP {
			proto = "http"
		}
		host := "localhost"
		if c.BindHost != "" {
			host = c.BindHost
		}
		c.ConsentURL = fmt.Sprintf("%s://%s:%d/oauth2/consent", proto, host, c.BindPort)
	}

	consentURL, err := url.Parse(c.ConsentURL)
	pkg.Must(err, "Could not parse consent url %s.", c.ConsentURL)

	handler := &oauth2.Handler{
		ScopesSupported:  c.OpenIDDiscoveryScopesSupported,
		UserinfoEndpoint: c.OpenIDDiscoveryUserinfoEndpoint,
		ClaimsSupported:  c.OpenIDDiscoveryClaimsSupported,
		ForcedHTTP:       c.ForceHTTP,
		OAuth2:           o,
		ScopeStrategy:    c.GetScopeStrategy(),
		Consent: &oauth2.DefaultConsentStrategy{
			Issuer:                   c.Issuer,
			ConsentManager:           c.Context().ConsentManager,
			DefaultChallengeLifespan: c.GetChallengeTokenLifespan(),
			DefaultIDTokenLifespan:   c.GetIDTokenLifespan(),
			KeyID: idTokenKeyID,
		},
		Storage:             c.Context().FositeStore,
		ConsentURL:          *consentURL,
		H:                   herodot.NewJSONWriter(c.GetLogger()),
		AccessTokenLifespan: c.GetAccessTokenLifespan(),
		CookieStore:         sessions.NewCookieStore(c.GetCookieSecret()),
		Issuer:              c.Issuer,
		L:                   c.GetLogger(),
		W:                   c.Context().Warden,
		ResourcePrefix:      c.AccessControlResourcePrefix,
	}

	handler.SetRoutes(router)
	return handler
}
<commit_msg>cmd: Adds OpenID Connect refresh handler (#797)<commit_after>// Copyright © 2017 Aeneas Rekkas <aeneas+oss@aeneas.io>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"fmt"
	"net/url"

	"github.com/gorilla/sessions"
	"github.com/julienschmidt/httprouter"
	"github.com/ory/fosite"
	"github.com/ory/fosite/compose"
	"github.com/ory/herodot"
	"github.com/ory/hydra/client"
	"github.com/ory/hydra/config"
	"github.com/ory/hydra/jwk"
	"github.com/ory/hydra/oauth2"
	"github.com/ory/hydra/pkg"
	"github.com/ory/hydra/warden"
)

func injectFositeStore(c *config.Config, clients client.Manager) {
	var ctx = c.Context()
	var store pkg.FositeStorer

	switch con := ctx.Connection.(type) {
	case *config.MemoryConnection:
		store = oauth2.NewFositeMemoryStore(clients, c.GetAccessTokenLifespan())
		break
	case *config.SQLConnection:
		store = oauth2.NewFositeSQLStore(clients, con.GetDatabase(), c.GetLogger(), c.GetAccessTokenLifespan())
		break
	case *config.PluginConnection:
		var err error
		if store, err = con.NewOAuth2Manager(clients); err != nil {
			c.GetLogger().Fatalf("Could not load client manager plugin %s", err)
		}
		break
	default:
		panic("Unknown connection type.")
	}

	ctx.FositeStore = store
}

func newOAuth2Provider(c *config.Config) (fosite.OAuth2Provider, string) {
	var ctx = c.Context()
	var store = ctx.FositeStore

	privateKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "private")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch private signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	publicKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "public")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch public signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	fc := &compose.Config{
		AccessTokenLifespan:            c.GetAccessTokenLifespan(),
		AuthorizeCodeLifespan:          c.GetAuthCodeLifespan(),
		IDTokenLifespan:                c.GetIDTokenLifespan(),
		HashCost:                       c.BCryptWorkFactor,
		ScopeStrategy:                  c.GetScopeStrategy(),
		SendDebugMessagesToClients:     c.SendOAuth2DebugMessagesToClients,
		EnforcePKCE:                    false,
		EnablePKCEPlainChallengeMethod: false,
	}

	return compose.Compose(
		fc,
		store,
		&compose.CommonStrategy{
			CoreStrategy:               compose.NewOAuth2HMACStrategy(fc, c.GetSystemSecret()),
			OpenIDConnectTokenStrategy: compose.NewOpenIDConnectStrategy(jwk.MustRSAPrivate(privateKey)),
		},
		nil,
		compose.OAuth2AuthorizeExplicitFactory,
		compose.OAuth2AuthorizeImplicitFactory,
		compose.OAuth2ClientCredentialsGrantFactory,
		compose.OAuth2RefreshTokenGrantFactory,
		compose.OAuth2PKCEFactory,
		compose.OpenIDConnectExplicitFactory,
		compose.OpenIDConnectHybridFactory,
		compose.OpenIDConnectImplicitFactory,
		compose.OpenIDConnectRefreshFactory,
		compose.OAuth2TokenRevocationFactory,
		warden.OAuth2TokenIntrospectionFactory,
	), publicKey.KeyID
}

func newOAuth2Handler(c *config.Config, router *httprouter.Router, cm oauth2.ConsentRequestManager, o fosite.OAuth2Provider, idTokenKeyID string) *oauth2.Handler {
	if c.ConsentURL == "" {
		proto := "https"
		if c.ForceHTTP {
			proto = "http"
		}
		host := "localhost"
		if c.BindHost != "" {
			host = c.BindHost
		}
		c.ConsentURL = fmt.Sprintf("%s://%s:%d/oauth2/consent", proto, host, c.BindPort)
	}

	consentURL, err := url.Parse(c.ConsentURL)
	pkg.Must(err, "Could not parse consent url %s.", c.ConsentURL)

	handler := &oauth2.Handler{
		ScopesSupported:  c.OpenIDDiscoveryScopesSupported,
		UserinfoEndpoint: c.OpenIDDiscoveryUserinfoEndpoint,
		ClaimsSupported:  c.OpenIDDiscoveryClaimsSupported,
		ForcedHTTP:       c.ForceHTTP,
		OAuth2:           o,
		ScopeStrategy:    c.GetScopeStrategy(),
		Consent: &oauth2.DefaultConsentStrategy{
			Issuer:                   c.Issuer,
			ConsentManager:           c.Context().ConsentManager,
			DefaultChallengeLifespan: c.GetChallengeTokenLifespan(),
			DefaultIDTokenLifespan:   c.GetIDTokenLifespan(),
			KeyID: idTokenKeyID,
		},
		Storage:             c.Context().FositeStore,
		ConsentURL:          *consentURL,
		H:                   herodot.NewJSONWriter(c.GetLogger()),
		AccessTokenLifespan: c.GetAccessTokenLifespan(),
		CookieStore:         sessions.NewCookieStore(c.GetCookieSecret()),
		Issuer:              c.Issuer,
		L:                   c.GetLogger(),
		W:                   c.Context().Warden,
		ResourcePrefix:      c.AccessControlResourcePrefix,
	}

	handler.SetRoutes(router)
	return handler
}
<|endoftext|>


<commit_before>package files

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"

	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/gin-gonic/gin"
	"github.com/sourcegraph/checkup"
	"github.com/spf13/afero"
	"github.com/stretchr/testify/assert"
)

const COUCHURL = "http://localhost:5984/"

var ts *httptest.Server
var instance *middlewares.Instance

func injectInstance(instance *middlewares.Instance) gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Set("instance", instance)
	}
}

func createDir(t *testing.T, path string) (res *http.Response, v map[string]interface{}) {
	res, err := http.Post(ts.URL+path, "text/plain", strings.NewReader(""))
	if !assert.NoError(t, err) {
		return
	}
	defer res.Body.Close()

	b, err := ioutil.ReadAll(res.Body)
	if !assert.NoError(t, err) {
		return
	}

	if res.StatusCode >= 200 && res.StatusCode < 300 {
		err = json.Unmarshal(b, &v)
		if !assert.NoError(t, err) {
			return
		}
	}

	return
}

func upload(t *testing.T, path, contentType, body, hash string) (res *http.Response, v map[string]interface{}) {
	buf := strings.NewReader(body)
	req, err := http.NewRequest("POST", ts.URL+path, buf)
	if !assert.NoError(t, err) {
		return
	}

	if hash != "" {
		req.Header.Add("Content-MD5", hash)
	}

	res, err = http.DefaultClient.Do(req)
	if !assert.NoError(t, err) {
		return
	}

	defer res.Body.Close()

	b, err := ioutil.ReadAll(res.Body)
	if !assert.NoError(t, err) {
		return
	}

	if res.StatusCode >= 200 && res.StatusCode < 300 {
		err = json.Unmarshal(b, &v)
		if !assert.NoError(t, err) {
			return
		}
	}

	return
}

func TestCreateDirWithNoType(t *testing.T) {
	res, _ := createDir(t, "/files/")
	assert.Equal(t, 422, res.StatusCode)
}

func TestCreateDirWithNoName(t *testing.T) {
	res, _ := createDir(t, "/files/?Type=io.cozy.folders")
	assert.Equal(t, 422, res.StatusCode)
}

func TestCreateDirOnNonExistingParent(t *testing.T) {
	res, _ := createDir(t, "/files/noooop?Name=foo&Type=io.cozy.folders")
	assert.Equal(t, 404, res.StatusCode)
}

func TestCreateDirAlreadyExists(t *testing.T) {
	res1, _ := createDir(t, "/files/?Name=iexist&Type=io.cozy.folders")
	assert.Equal(t, 201, res1.StatusCode)

	res2, _ := createDir(t, "/files/?Name=iexist&Type=io.cozy.folders")
	assert.Equal(t, 409, res2.StatusCode)
}

func TestCreateDirRootSuccess(t *testing.T) {
	res, _ := createDir(t, "/files/?Name=coucou&Type=io.cozy.folders")
	assert.Equal(t, 201, res.StatusCode)

	storage, _ := instance.GetStorageProvider()
	exists, err := afero.DirExists(storage, "/coucou")
	assert.NoError(t, err)
	assert.True(t, exists)
}

func TestCreateDirWithParentSuccess(t *testing.T) {
	res1, data1 := createDir(t, "/files/?Name=dirparent&Type=io.cozy.folders")
	assert.Equal(t, 201, res1.StatusCode)

	var ok bool
	data1, ok = data1["data"].(map[string]interface{})
	assert.True(t, ok)

	parentID, ok := data1["id"].(string)
	assert.True(t, ok)

	res2, _ := createDir(t, "/files/"+parentID+"?Name=child&Type=io.cozy.folders")
	assert.Equal(t, 201, res2.StatusCode)

	storage, _ := instance.GetStorageProvider()
	exists, err := afero.DirExists(storage, "/dirparent/child")
	assert.NoError(t, err)
	assert.True(t, exists)
}

func TestCreateDirWithIllegalCharacter(t *testing.T) {
	res1, _ := createDir(t, "/files/?Name=coucou/les/copains!&Type=io.cozy.folders")
	assert.Equal(t, 422, res1.StatusCode)

	res2, _ := createDir(t, "/files/?Name=j'ai\x00untrou!&Type=io.cozy.folders")
	assert.Equal(t, 422, res2.StatusCode)
}

func TestUploadWithNoType(t *testing.T) {
	res, _ := upload(t, "/files/", "text/plain", "foo", "")
	assert.Equal(t, 422, res.StatusCode)
}

func TestUploadWithNoName(t *testing.T) {
	res, _ := upload(t, "/files/?Type=io.cozy.files", "text/plain", "foo", "")
	assert.Equal(t, 422, res.StatusCode)
}

func TestUploadBadHash(t *testing.T) {
	body := "foo"
	res, _ := upload(t, "/files/?Type=io.cozy.files&Name=badhash", "text/plain", body, "3FbbMXfH+PdjAlWFfVb1dQ==")
	assert.Equal(t, 412, res.StatusCode)

	storage, _ := instance.GetStorageProvider()
	_, err := afero.ReadFile(storage, "123/badhash")
	assert.Error(t, err)
}

func TestUploadAtRootSuccess(t *testing.T) {
	body := "foo"
	res, _ := upload(t, "/files/?Type=io.cozy.files&Name=goodhash", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res.StatusCode)

	storage, _ := instance.GetStorageProvider()
	buf, err := afero.ReadFile(storage, "/goodhash")
	assert.NoError(t, err)
	assert.Equal(t, body, string(buf))
}

func TestUploadWithParentSuccess(t *testing.T) {
	res1, data1 := createDir(t, "/files/?Name=fileparent&Type=io.cozy.folders")
	assert.Equal(t, 201, res1.StatusCode)

	var ok bool
	data1, ok = data1["data"].(map[string]interface{})
	assert.True(t, ok)

	parentID, ok := data1["id"].(string)
	assert.True(t, ok)

	body := "foo"
	res2, _ := upload(t, "/files/"+parentID+"?Type=io.cozy.files&Name=goodhash", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res2.StatusCode)

	storage, _ := instance.GetStorageProvider()
	buf, err := afero.ReadFile(storage, "/fileparent/goodhash")
	assert.NoError(t, err)
	assert.Equal(t, body, string(buf))
}

func TestUploadAtRootAlreadyExists(t *testing.T) {
	body := "foo"
	res1, _ := upload(t, "/files/?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res1.StatusCode)

	res2, _ := upload(t, "/files/?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 409, res2.StatusCode)
}

func TestUploadWithParentAlreadyExists(t *testing.T) {
	_, dirdata := createDir(t, "/files/?Type=io.cozy.folders&Name=container")

	var ok bool
	dirdata, ok = dirdata["data"].(map[string]interface{})
	assert.True(t, ok)

	parentID, ok := dirdata["id"].(string)
	assert.True(t, ok)

	body := "foo"
	res1, _ := upload(t, "/files/"+parentID+"?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res1.StatusCode)

	res2, _ := upload(t, "/files/"+parentID+"?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 409, res2.StatusCode)
}

func TestMain(m *testing.M) {
	// First we make sure couchdb is started
	couchdb, err := checkup.HTTPChecker{URL: COUCHURL}.Check()
	if err != nil || couchdb.Status() != checkup.Healthy {
		fmt.Println("This test need couchdb to run.")
		os.Exit(1)
	}

	gin.SetMode(gin.TestMode)
	instance = &middlewares.Instance{
		Domain:     "test",
		StorageURL: "mem://test",
	}

	router := gin.New()
	router.Use(injectInstance(instance))
	router.POST("/files/", CreationHandler)
	router.POST("/files/:folder-id", CreationHandler)
	ts = httptest.NewServer(router)
	defer ts.Close()
	os.Exit(m.Run())
}
<commit_msg>Mutualize json result unmarshalling in tests<commit_after>package files

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"

	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/gin-gonic/gin"
	"github.com/sourcegraph/checkup"
	"github.com/spf13/afero"
	"github.com/stretchr/testify/assert"
)

const COUCHURL = "http://localhost:5984/"

var ts *httptest.Server
var instance *middlewares.Instance

func injectInstance(instance *middlewares.Instance) gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Set("instance", instance)
	}
}

func extractJSONRes(res *http.Response, mp *map[string]interface{}) (err error) {
	if res.StatusCode >= 300 {
		return
	}

	var b []byte

	b, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	err = json.Unmarshal(b, mp)
	return
}

func createDir(t *testing.T, path string) (res *http.Response, v map[string]interface{}) {
	res, err := http.Post(ts.URL+path, "text/plain", strings.NewReader(""))
	if !assert.NoError(t, err) {
		return
	}
	defer res.Body.Close()

	err = extractJSONRes(res, &v)
	assert.NoError(t, err)

	return
}

func upload(t *testing.T, path, contentType, body, hash string) (res *http.Response, v map[string]interface{}) {
	buf := strings.NewReader(body)
	req, err := http.NewRequest("POST", ts.URL+path, buf)
	if !assert.NoError(t, err) {
		return
	}

	if hash != "" {
		req.Header.Add("Content-MD5", hash)
	}

	res, err = http.DefaultClient.Do(req)
	if !assert.NoError(t, err) {
		return
	}

	defer res.Body.Close()

	err = extractJSONRes(res, &v)
	assert.NoError(t, err)

	return
}

func TestCreateDirWithNoType(t *testing.T) {
	res, _ := createDir(t, "/files/")
	assert.Equal(t, 422, res.StatusCode)
}

func TestCreateDirWithNoName(t *testing.T) {
	res, _ := createDir(t, "/files/?Type=io.cozy.folders")
	assert.Equal(t, 422, res.StatusCode)
}

func TestCreateDirOnNonExistingParent(t *testing.T) {
	res, _ := createDir(t, "/files/noooop?Name=foo&Type=io.cozy.folders")
	assert.Equal(t, 404, res.StatusCode)
}

func TestCreateDirAlreadyExists(t *testing.T) {
	res1, _ := createDir(t, "/files/?Name=iexist&Type=io.cozy.folders")
	assert.Equal(t, 201, res1.StatusCode)

	res2, _ := createDir(t, "/files/?Name=iexist&Type=io.cozy.folders")
	assert.Equal(t, 409, res2.StatusCode)
}

func TestCreateDirRootSuccess(t *testing.T) {
	res, _ := createDir(t, "/files/?Name=coucou&Type=io.cozy.folders")
	assert.Equal(t, 201, res.StatusCode)

	storage, _ := instance.GetStorageProvider()
	exists, err := afero.DirExists(storage, "/coucou")
	assert.NoError(t, err)
	assert.True(t, exists)
}

func TestCreateDirWithParentSuccess(t *testing.T) {
	res1, data1 := createDir(t, "/files/?Name=dirparent&Type=io.cozy.folders")
	assert.Equal(t, 201, res1.StatusCode)

	var ok bool
	data1, ok = data1["data"].(map[string]interface{})
	assert.True(t, ok)

	parentID, ok := data1["id"].(string)
	assert.True(t, ok)

	res2, _ := createDir(t, "/files/"+parentID+"?Name=child&Type=io.cozy.folders")
	assert.Equal(t, 201, res2.StatusCode)

	storage, _ := instance.GetStorageProvider()
	exists, err := afero.DirExists(storage, "/dirparent/child")
	assert.NoError(t, err)
	assert.True(t, exists)
}

func TestCreateDirWithIllegalCharacter(t *testing.T) {
	res1, _ := createDir(t, "/files/?Name=coucou/les/copains!&Type=io.cozy.folders")
	assert.Equal(t, 422, res1.StatusCode)

	res2, _ := createDir(t, "/files/?Name=j'ai\x00untrou!&Type=io.cozy.folders")
	assert.Equal(t, 422, res2.StatusCode)
}

func TestUploadWithNoType(t *testing.T) {
	res, _ := upload(t, "/files/", "text/plain", "foo", "")
	assert.Equal(t, 422, res.StatusCode)
}

func TestUploadWithNoName(t *testing.T) {
	res, _ := upload(t, "/files/?Type=io.cozy.files", "text/plain", "foo", "")
	assert.Equal(t, 422, res.StatusCode)
}

func TestUploadBadHash(t *testing.T) {
	body := "foo"
	res, _ := upload(t, "/files/?Type=io.cozy.files&Name=badhash", "text/plain", body, "3FbbMXfH+PdjAlWFfVb1dQ==")
	assert.Equal(t, 412, res.StatusCode)

	storage, _ := instance.GetStorageProvider()
	_, err := afero.ReadFile(storage, "123/badhash")
	assert.Error(t, err)
}

func TestUploadAtRootSuccess(t *testing.T) {
	body := "foo"
	res, _ := upload(t, "/files/?Type=io.cozy.files&Name=goodhash", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res.StatusCode)

	storage, _ := instance.GetStorageProvider()
	buf, err := afero.ReadFile(storage, "/goodhash")
	assert.NoError(t, err)
	assert.Equal(t, body, string(buf))
}

func TestUploadWithParentSuccess(t *testing.T) {
	res1, data1 := createDir(t, "/files/?Name=fileparent&Type=io.cozy.folders")
	assert.Equal(t, 201, res1.StatusCode)

	var ok bool
	data1, ok = data1["data"].(map[string]interface{})
	assert.True(t, ok)

	parentID, ok := data1["id"].(string)
	assert.True(t, ok)

	body := "foo"
	res2, _ := upload(t, "/files/"+parentID+"?Type=io.cozy.files&Name=goodhash", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res2.StatusCode)

	storage, _ := instance.GetStorageProvider()
	buf, err := afero.ReadFile(storage, "/fileparent/goodhash")
	assert.NoError(t, err)
	assert.Equal(t, body, string(buf))
}

func TestUploadAtRootAlreadyExists(t *testing.T) {
	body := "foo"
	res1, _ := upload(t, "/files/?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res1.StatusCode)

	res2, _ := upload(t, "/files/?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 409, res2.StatusCode)
}

func TestUploadWithParentAlreadyExists(t *testing.T) {
	_, dirdata := createDir(t, "/files/?Type=io.cozy.folders&Name=container")

	var ok bool
	dirdata, ok = dirdata["data"].(map[string]interface{})
	assert.True(t, ok)

	parentID, ok := dirdata["id"].(string)
	assert.True(t, ok)

	body := "foo"
	res1, _ := upload(t, "/files/"+parentID+"?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 201, res1.StatusCode)

	res2, _ := upload(t, "/files/"+parentID+"?Type=io.cozy.files&Name=iexistfile", "text/plain", body, "rL0Y20zC+Fzt72VPzMSk2A==")
	assert.Equal(t, 409, res2.StatusCode)
}

func TestMain(m *testing.M) {
	// First we make sure couchdb is started
	couchdb, err := checkup.HTTPChecker{URL: COUCHURL}.Check()
	if err != nil || couchdb.Status() != checkup.Healthy {
		fmt.Println("This test need couchdb to run.")
		os.Exit(1)
	}

	gin.SetMode(gin.TestMode)
	instance = &middlewares.Instance{
		Domain:     "test",
		StorageURL: "mem://test",
	}

	router := gin.New()
	router.Use(injectInstance(instance))
	router.POST("/files/", CreationHandler)
	router.POST("/files/:folder-id", CreationHandler)
	ts = httptest.NewServer(router)
	defer ts.Close()
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>package chrootarchive

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"syscall"

	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/reexec"
)

type applyLayerResponse struct {
	LayerSize int64 `json:"layerSize"`
}

func applyLayer() {
	runtime.LockOSThread()
	flag.Parse()

	if err := chroot(flag.Arg(0)); err != nil {
		fatal(err)
	}

	// We need to be able to set any perms
	oldmask := syscall.Umask(0)
	defer syscall.Umask(oldmask)
	tmpDir, err := ioutil.TempDir("/", "temp-docker-extract")
	if err != nil {
		fatal(err)
	}

	os.Setenv("TMPDIR", tmpDir)
	size, err := archive.UnpackLayer("/", os.Stdin)
	os.RemoveAll(tmpDir)
	if err != nil {
		fatal(err)
	}

	encoder := json.NewEncoder(os.Stdout)
	if err := encoder.Encode(applyLayerResponse{size}); err != nil {
		fatal(fmt.Errorf("unable to encode layerSize JSON: %s", err))
	}

	flush(os.Stdout)
	flush(os.Stdin)
	os.Exit(0)
}

func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
	dest = filepath.Clean(dest)
	decompressed, err := archive.DecompressStream(layer)
	if err != nil {
		return 0, err
	}

	defer func() {
		if c, ok := decompressed.(io.Closer); ok {
			c.Close()
		}
	}()

	cmd := reexec.Command("docker-applyLayer", dest)
	cmd.Stdin = decompressed

	outBuf, errBuf := new(bytes.Buffer), new(bytes.Buffer)
	cmd.Stdout, cmd.Stderr = outBuf, errBuf

	if err = cmd.Run(); err != nil {
		return 0, fmt.Errorf("ApplyLayer %s stdout: %s stderr: %s", err, outBuf, errBuf)
	}

	// Stdout should be a valid JSON struct representing an applyLayerResponse.
	response := applyLayerResponse{}
	decoder := json.NewDecoder(outBuf)
	if err = decoder.Decode(&response); err != nil {
		return 0, fmt.Errorf("unable to decode ApplyLayer JSON response: %s", err)
	}

	return response.LayerSize, nil
}
<commit_msg>Not doing extra assertion for io.Closer<commit_after>package chrootarchive

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"syscall"

	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/reexec"
)

type applyLayerResponse struct {
	LayerSize int64 `json:"layerSize"`
}

func applyLayer() {
	runtime.LockOSThread()
	flag.Parse()

	if err := chroot(flag.Arg(0)); err != nil {
		fatal(err)
	}

	// We need to be able to set any perms
	oldmask := syscall.Umask(0)
	defer syscall.Umask(oldmask)
	tmpDir, err := ioutil.TempDir("/", "temp-docker-extract")
	if err != nil {
		fatal(err)
	}

	os.Setenv("TMPDIR", tmpDir)
	size, err := archive.UnpackLayer("/", os.Stdin)
	os.RemoveAll(tmpDir)
	if err != nil {
		fatal(err)
	}

	encoder := json.NewEncoder(os.Stdout)
	if err := encoder.Encode(applyLayerResponse{size}); err != nil {
		fatal(fmt.Errorf("unable to encode layerSize JSON: %s", err))
	}

	flush(os.Stdout)
	flush(os.Stdin)
	os.Exit(0)
}

func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
	dest = filepath.Clean(dest)
	decompressed, err := archive.DecompressStream(layer)
	if err != nil {
		return 0, err
	}

	defer decompressed.Close()

	cmd := reexec.Command("docker-applyLayer", dest)
	cmd.Stdin = decompressed

	outBuf, errBuf := new(bytes.Buffer), new(bytes.Buffer)
	cmd.Stdout, cmd.Stderr = outBuf, errBuf

	if err = cmd.Run(); err != nil {
		return 0, fmt.Errorf("ApplyLayer %s stdout: %s stderr: %s", err, outBuf, errBuf)
	}

	// Stdout should be a valid JSON struct representing an applyLayerResponse.
	response := applyLayerResponse{}
	decoder := json.NewDecoder(outBuf)
	if err = decoder.Decode(&response); err != nil {
		return 0, fmt.Errorf("unable to decode ApplyLayer JSON response: %s", err)
	}

	return response.LayerSize, nil
}
<|endoftext|>


<commit_before>package instances

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/sharing"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/labstack/echo/v4"
)

func fsckHandler(c echo.Context) (err error) {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	indexIntegrityCheck, _ := strconv.ParseBool(c.QueryParam("IndexIntegrity"))
	filesConsistencyCheck, _ := strconv.ParseBool(c.QueryParam("FilesConsistency"))
	failFast, _ := strconv.ParseBool(c.QueryParam("FailFast"))

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := i.VFS()
		if indexIntegrityCheck {
			err = fs.CheckIndexIntegrity(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else if filesConsistencyCheck {
			err = fs.CheckFilesConsistency(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else {
			err = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		}
		close(logCh)
	}()

	w := c.Response().Writer
	w.WriteHeader(200)
	encoder := json.NewEncoder(w)
	for log := range logCh {
		// XXX do not serialize to JSON the children and the cozyMetadata, as
		// it can take more than 64ko and scanner will ignore such lines.
		if log.FileDoc != nil {
			log.FileDoc.DirsChildren = nil  // It can be filled on type mismatch
			log.FileDoc.FilesChildren = nil // Idem
			log.FileDoc.FilesChildrenSize = 0
			log.FileDoc.Metadata = nil
		}
		if log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
			log.DirDoc.FilesChildrenSize = 0
			log.DirDoc.Metadata = nil
		}
		if log.VersionDoc != nil {
			log.VersionDoc.Metadata = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	if err != nil {
		log := map[string]string{"error": err.Error()}
		if couchdb.IsNotFoundError(err) {
			log = map[string]string{"type": "no_database", "error": err.Error()}
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return nil
}

func checkTriggers(c echo.Context) error {
	domain := c.Param("domain")
	inst, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	type TriggerInfo struct {
		TID        string          `json:"_id"`
		Type       string          `json:"type"`
		WorkerType string          `json:"worker"`
		Arguments  string          `json:"arguments"`
		Debounce   string          `json:"debounce"`
		Message    json.RawMessage `json:"message"`
	}
	var triggers []*TriggerInfo
	err = couchdb.ForeachDocs(inst, consts.Triggers, func(_ string, data json.RawMessage) error {
		var t *TriggerInfo
		if err := json.Unmarshal(data, &t); err != nil {
			return err
		}
		triggers = append(triggers, t)
		return nil
	})
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"type": "no_database", "error": err.Error()},
			})
		}
		return wrapError(err)
	}

	results := []map[string]interface{}{}
	for i, left := range triggers {
		for j, right := range triggers {
			if i >= j {
				continue
			}
			if left.Type == "@in" {
				// It doesn't make sense to compare @in triggers as they can
				// have scheduled at different times
				continue
			}
			if left.Type != right.Type {
				continue
			}
			if left.WorkerType != right.WorkerType {
				continue
			}
			if left.Arguments != right.Arguments {
				continue
			}
			if left.Debounce != right.Debounce {
				continue
			}
			lHasMessage := left.Message != nil
			rHasMessage := right.Message != nil
			if lHasMessage != rHasMessage {
				continue
			}
			if lHasMessage && rHasMessage {
				if !bytes.Equal(left.Message, right.Message) {
					continue
				}
			}
			results = append(results, map[string]interface{}{
				"type":      "duplicate",
				"_id":       left.TID,
				"other_id":  right.TID,
				"trigger":   left.Type,
				"worker":    left.WorkerType,
				"arguments": left.Arguments,
				"debounce":  left.Debounce,
				"message":   fmt.Sprintf("%s", left.Message),
			})
			break
		}
	}

	for _, t := range triggers {
		if t.Type == "@event" && t.WorkerType == "service" {
			var msg map[string]interface{}
			err := json.Unmarshal(t.Message, &msg)
			if err != nil {
				results = append(results, map[string]interface{}{
					"type":    "invalid_message",
					"_id":     t.TID,
					"trigger": t.Type,
					"worker":  t.WorkerType,
					"message": fmt.Sprintf("%s", t.Message),
				})
				continue
			}
			if name, _ := msg["name"].(string); name == "" {
				results = append(results, map[string]interface{}{
					"type":    "missing_name",
					"_id":     t.TID,
					"trigger": t.Type,
					"worker":  t.WorkerType,
					"message": fmt.Sprintf("%s", t.Message),
				})
			}
		}
	}

	return c.JSON(http.StatusOK, results)
}

func checkShared(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckShared(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"type": "no_database", "error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}

func checkSharings(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckSharings(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"type": "no_database", "error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}
<commit_msg>Detect missing slug for check triggers<commit_after>package instances

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/sharing"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/labstack/echo/v4"
)

func fsckHandler(c echo.Context) (err error) {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	indexIntegrityCheck, _ := strconv.ParseBool(c.QueryParam("IndexIntegrity"))
	filesConsistencyCheck, _ := strconv.ParseBool(c.QueryParam("FilesConsistency"))
	failFast, _ := strconv.ParseBool(c.QueryParam("FailFast"))

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := i.VFS()
		if indexIntegrityCheck {
			err = fs.CheckIndexIntegrity(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else if filesConsistencyCheck {
			err = fs.CheckFilesConsistency(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else {
			err = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		}
		close(logCh)
	}()

	w := c.Response().Writer
	w.WriteHeader(200)
	encoder := json.NewEncoder(w)
	for log := range logCh {
		// XXX do not serialize to JSON the children and the cozyMetadata, as
		// it can take more than 64ko and scanner will ignore such lines.
		if log.FileDoc != nil {
			log.FileDoc.DirsChildren = nil  // It can be filled on type mismatch
			log.FileDoc.FilesChildren = nil // Idem
			log.FileDoc.FilesChildrenSize = 0
			log.FileDoc.Metadata = nil
		}
		if log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
			log.DirDoc.FilesChildrenSize = 0
			log.DirDoc.Metadata = nil
		}
		if log.VersionDoc != nil {
			log.VersionDoc.Metadata = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	if err != nil {
		log := map[string]string{"error": err.Error()}
		if couchdb.IsNotFoundError(err) {
			log = map[string]string{"type": "no_database", "error": err.Error()}
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return nil
}

func checkTriggers(c echo.Context) error {
	domain := c.Param("domain")
	inst, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	type TriggerInfo struct {
		TID        string          `json:"_id"`
		Type       string          `json:"type"`
		WorkerType string          `json:"worker"`
		Arguments  string          `json:"arguments"`
		Debounce   string          `json:"debounce"`
		Message    json.RawMessage `json:"message"`
	}
	var triggers []*TriggerInfo
	err = couchdb.ForeachDocs(inst, consts.Triggers, func(_ string, data json.RawMessage) error {
		var t *TriggerInfo
		if err := json.Unmarshal(data, &t); err != nil {
			return err
		}
		triggers = append(triggers, t)
		return nil
	})
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"type": "no_database", "error": err.Error()},
			})
		}
		return wrapError(err)
	}

	results := []map[string]interface{}{}
	for i, left := range triggers {
		for j, right := range triggers {
			if i >= j {
				continue
			}
			if left.Type == "@in" {
				// It doesn't make sense to compare @in triggers as they can
				// have scheduled at different times
				continue
			}
			if left.Type != right.Type {
				continue
			}
			if left.WorkerType != right.WorkerType {
				continue
			}
			if left.Arguments != right.Arguments {
				continue
			}
			if left.Debounce != right.Debounce {
				continue
			}
			lHasMessage := left.Message != nil
			rHasMessage := right.Message != nil
			if lHasMessage != rHasMessage {
				continue
			}
			if lHasMessage && rHasMessage {
				if !bytes.Equal(left.Message, right.Message) {
					continue
				}
			}
			results = append(results, map[string]interface{}{
				"type":      "duplicate",
				"_id":       left.TID,
				"other_id":  right.TID,
				"trigger":   left.Type,
				"worker":    left.WorkerType,
				"arguments": left.Arguments,
				"debounce":  left.Debounce,
				"message":   fmt.Sprintf("%s", left.Message),
			})
			break
		}
	}

	for _, t := range triggers {
		if t.Type == "@event" && t.WorkerType == "service" {
			var msg map[string]interface{}
			err := json.Unmarshal(t.Message, &msg)
			if err != nil {
				results = append(results, map[string]interface{}{
					"type":    "invalid_message",
					"_id":     t.TID,
					"trigger": t.Type,
					"worker":  t.WorkerType,
					"message": fmt.Sprintf("%s", t.Message),
				})
				continue
			}
			if name, _ := msg["name"].(string); name == "" {
				results = append(results, map[string]interface{}{
					"type":    "missing_name",
					"_id":     t.TID,
					"trigger": t.Type,
					"worker":  t.WorkerType,
					"message": fmt.Sprintf("%s", t.Message),
				})
			}
			if slug, _ := msg["slug"].(string); slug == "" {
				results = append(results, map[string]interface{}{
					"type":    "missing_slug",
					"_id":     t.TID,
					"trigger": t.Type,
					"worker":  t.WorkerType,
					"message": fmt.Sprintf("%s", t.Message),
				})
			}
		}
	}

	return c.JSON(http.StatusOK, results)
}

func checkShared(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckShared(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"type": "no_database", "error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}

func checkSharings(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckSharings(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"type": "no_database", "error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package qos

import (
	"k8s.io/kubernetes/pkg/api"
	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
)

const (
	// PodInfraOOMAdj is very docker specific. For arbitrary runtime, it may not make
	// sense to set sandbox level oom score, e.g. a sandbox could only be a namespace
	// without a process.
	// TODO: Handle infra container oom score adj in a runtime agnostic way.
	// TODO: Should handle critical pod oom score adj with a proper preemption priority.
	// This is the workaround for https://github.com/kubernetes/kubernetes/issues/38322.
	PodInfraOOMAdj        int = -998
	CriticalPodOOMAdj     int = -998
	KubeletOOMScoreAdj    int = -999
	DockerOOMScoreAdj     int = -999
	KubeProxyOOMScoreAdj  int = -999
	guaranteedOOMScoreAdj int = -998
	besteffortOOMScoreAdj int = 1000
)

// GetContainerOOMAdjust returns the amount by which the OOM score of all processes in the
// container should be adjusted.
// The OOM score of a process is the percentage of memory it consumes
// multiplied by 10 (barring exceptional cases) + a configurable quantity which is between -1000
// and 1000. Containers with higher OOM scores are killed if the system runs out of memory.
// See https://lwn.net/Articles/391222/ for more information.
func GetContainerOOMScoreAdjust(pod *api.Pod, container *v1.Container, memoryCapacity int64) int {
	if kubepod.IsCriticalPod(pod) {
		return CriticalPodOOMAdj
	}

	switch GetPodQOS(pod) {
	case Guaranteed:
		// Guaranteed containers should be the last to get killed.
		return guaranteedOOMScoreAdj
	case BestEffort:
		return besteffortOOMScoreAdj
	}

	// Burstable containers are a middle tier, between Guaranteed and Best-Effort. Ideally,
	// we want to protect Burstable containers that consume less memory than requested.
	// The formula below is a heuristic. A container requesting for 10% of a system's
	// memory will have an OOM score adjust of 900. If a process in container Y
	// uses over 10% of memory, its OOM score will be 1000. The idea is that containers
	// which use more than their request will have an OOM score of 1000 and will be prime
	// targets for OOM kills.
	// Note that this is a heuristic, it won't work if a container has many small processes.
	memoryRequest := container.Resources.Requests.Memory().Value()
	oomScoreAdjust := 1000 - (1000*memoryRequest)/memoryCapacity
	// A guaranteed pod using 100% of memory can have an OOM score of 10. Ensure
	// that burstable pods have a higher OOM score adjustment.
	if int(oomScoreAdjust) < (1000 + guaranteedOOMScoreAdj) {
		return (1000 + guaranteedOOMScoreAdj)
	}
	// Give burstable pods a higher chance of survival over besteffort pods.
	if int(oomScoreAdjust) == besteffortOOMScoreAdj {
		return int(oomScoreAdjust - 1)
	}
	return int(oomScoreAdjust)
}
<commit_msg>Fixed an import cycle issue:<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package qos

import (
	"k8s.io/kubernetes/pkg/api"
	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
)

const (
	// PodInfraOOMAdj is very docker specific. For arbitrary runtime, it may not make
	// sense to set sandbox level oom score, e.g. a sandbox could only be a namespace
	// without a process.
	// TODO: Handle infra container oom score adj in a runtime agnostic way.
	// TODO: Should handle critical pod oom score adj with a proper preemption priority.
	// This is the workaround for https://github.com/kubernetes/kubernetes/issues/38322.
	PodInfraOOMAdj        int = -998
	CriticalPodOOMAdj     int = -998
	KubeletOOMScoreAdj    int = -999
	DockerOOMScoreAdj     int = -999
	KubeProxyOOMScoreAdj  int = -999
	guaranteedOOMScoreAdj int = -998
	besteffortOOMScoreAdj int = 1000
)

// GetContainerOOMAdjust returns the amount by which the OOM score of all processes in the
// container should be adjusted.
// The OOM score of a process is the percentage of memory it consumes
// multiplied by 10 (barring exceptional cases) + a configurable quantity which is between -1000
// and 1000. Containers with higher OOM scores are killed if the system runs out of memory.
// See https://lwn.net/Articles/391222/ for more information.
func GetContainerOOMScoreAdjust(pod *api.Pod, container *api.Container, memoryCapacity int64) int {
	if kubepod.IsCriticalPod(pod) {
		return CriticalPodOOMAdj
	}

	switch GetPodQOS(pod) {
	case Guaranteed:
		// Guaranteed containers should be the last to get killed.
		return guaranteedOOMScoreAdj
	case BestEffort:
		return besteffortOOMScoreAdj
	}

	// Burstable containers are a middle tier, between Guaranteed and Best-Effort. Ideally,
	// we want to protect Burstable containers that consume less memory than requested.
	// The formula below is a heuristic. A container requesting for 10% of a system's
	// memory will have an OOM score adjust of 900. If a process in container Y
	// uses over 10% of memory, its OOM score will be 1000. The idea is that containers
	// which use more than their request will have an OOM score of 1000 and will be prime
	// targets for OOM kills.
	// Note that this is a heuristic, it won't work if a container has many small processes.
	memoryRequest := container.Resources.Requests.Memory().Value()
	oomScoreAdjust := 1000 - (1000*memoryRequest)/memoryCapacity
	// A guaranteed pod using 100% of memory can have an OOM score of 10. Ensure
	// that burstable pods have a higher OOM score adjustment.
	if int(oomScoreAdjust) < (1000 + guaranteedOOMScoreAdj) {
		return (1000 + guaranteedOOMScoreAdj)
	}
	// Give burstable pods a higher chance of survival over besteffort pods.
	if int(oomScoreAdjust) == besteffortOOMScoreAdj {
		return int(oomScoreAdjust - 1)
	}
	return int(oomScoreAdjust)
}
<|endoftext|>


<commit_before>package paxos

import (
	"junta/assert"
	"junta/store"
	"testing"
)

func selfRefNewManager(extraNodes ...string) (*Manager, *store.Store) {
	p := make(FakePutterTo, 1)
	st := store.New()
	self := "a"
	st.Apply(uint64(1), mustEncodeSet(membersKey+"/"+self, self+"addr"))
	for i, node := range extraNodes {
		st.Apply(uint64(i+2), mustEncodeSet(membersKey+"/"+node, node+"addr"))
	}
	m := NewManager(self, uint64(len(extraNodes)+1), 1, st, p)
	p[0] = PutPutterTo{m}
	return m, st
}

func TestProposeAndLearn(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp)
	got := ix.Value()

	assert.Equal(t, exp, got, "")
}

func TestProposeAndRecv(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp)
	got := ix.Value()
	assert.Equal(t, exp, got, "")

	seqn, v := m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestProposeAndRecvAltStart(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp)
	got := ix.Value()
	assert.Equal(t, exp, got, "")

	seqn, v := m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestProposeAndRecvMultiple(t *testing.T) {
	exp := []string{"/foo", "/bar"}
	seqnexp := []uint64{2, 3}
	m, st := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp[0])
	got0 := ix.Value()
	assert.Equal(t, exp[0], got0, "")

	seqn0, v0 := m.Recv()
	assert.Equal(t, seqnexp[0], seqn0, "seqn 1")
	assert.Equal(t, exp[0], v0, "")

	st.Apply(seqn0, v0)

	_, ix = m.getInstance(0)
	ix.Propose(exp[1])
	got1 := ix.Value()
	assert.Equal(t, exp[1], got1, "")

	seqn1, v1 := m.Recv()
	assert.Equal(t, seqnexp[1], seqn1, "seqn 1")
	assert.Equal(t, exp[1], v1, "")
}

func TestNewInstanceBecauseOfMessage(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	m.Put(newVoteFrom(1, 1, exp))
	seqn, v := m.Recv()
	assert.Equal(t, uint64(1), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestNewInstanceBecauseOfMessageTriangulate(t *testing.T) {
	exp := "bar"
	m, _ := selfRefNewManager()

	m.Put(newVoteFrom(1, 1, exp))
	seqn, v := m.Recv()
	assert.Equal(t, uint64(1), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestUnusedSeqn(t *testing.T) {
	exp1, exp2 := "foo", "bar"
	m, _ := selfRefNewManager()

	m.Put(newVoteFrom(1, 1, exp1))
	seqn, v := m.Recv()
	assert.Equal(t, uint64(1), seqn, "")
	assert.Equal(t, exp1, v, "")

	_, ix := m.getInstance(0)
	ix.Propose(exp2)
	got := ix.Value()
	assert.Equal(t, exp2, got, "")
	seqn, v = m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, exp2, v, "")
}

func TestIgnoreMalformedMsg(t *testing.T) {
	m, _ := selfRefNewManager()

	m.Put(resize(newVoteFrom(1, 1, ""), -1))

	_, ix := m.getInstance(0)
	ix.Propose("y")
	got := ix.Value()
	assert.Equal(t, "y", got, "")

	seqn, v := m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, "y", v, "")
}

func TestProposeAndStore(t *testing.T) {
	exp := "foo"
	mg, st := selfRefNewManager()

	go func() {
		for {
			st.Apply(mg.Recv())
		}
	}()

	_, got, _ := mg.Propose(exp)
	assert.Equal(t, exp, got, "")
}

func TestProposeBadMutation(t *testing.T) {
	mg, st := selfRefNewManager()

	go func() {
		for {
			st.Apply(mg.Recv())
		}
	}()

	_, _, err := mg.Propose("foo")
	assert.Equal(t, store.ErrBadMutation, err)
}

func mustEncodeSet(k, v string) string {
	m, err := store.EncodeSet(k, v, store.Clobber)
	if err != nil {
		panic(err)
	}
	return m
}

func TestReadFromStore(t *testing.T) {
	// The cluster initially has 1 node (quorum of 1).
	st := store.New()
	p := make(ChanPutCloserTo)
	self := "a"
	st.Apply(1, mustEncodeSet(membersDir+self, ""))
	st.Apply(2, mustEncodeSet(slotDir+"0", self))
	m := NewManager(self, 2, 1, st, p)

	// Fire up a new instance with a vote message. This instance should block
	// trying to read the list of members. If it doesn't wait, it'll
	// immediately learn the value `x`.
	in := newVoteFrom(0, 1, "x")
	in.SetSeqn(5)
	m.Put(in)

	// Satisfy the sync read of data members above. After this, there will be
	// 2 nodes in the cluster, making the quorum 2.
	st.Apply(3, mustEncodeSet(membersDir+"b", ""))
	st.Apply(4, mustEncodeSet(slotDir+"1", "b"))

	// Now try to make it learn a new value with 2 votes to meet the new
	// quorum.
	exp := "y"
	in = newVoteFrom(0, 2, exp)
	in.SetSeqn(5)
	m.Put(in)
	in = newVoteFrom(1, 2, exp)
	in.SetSeqn(5)
	m.Put(in)

	seqn, v := m.Recv()
	assert.Equal(t, uint64(5), seqn, "")
	assert.Equal(t, exp, v, "")
}

type putFunc func(Msg)

func (pf putFunc) Put(msg Msg) {
	go pf(msg)
}

func (pf putFunc) Close() {}

func TestManagerPutFrom(t *testing.T) {
	exp := "bar"
	seqnExp := uint64(7)
	fromAddr := "y"
	fromIndex := 1 // [a, b, c].indexof(b) => 1

	st := store.New()
	self := "a"
	st.Apply(uint64(1), mustEncodeSet(membersDir+self, "x"))
	st.Apply(uint64(2), mustEncodeSet(slotDir+"0", self))
	st.Apply(uint64(3), mustEncodeSet(membersDir+"b", "y"))
	st.Apply(uint64(4), mustEncodeSet(slotDir+"1", "b"))
	st.Apply(uint64(5), mustEncodeSet(membersDir+"c", "z"))
	st.Apply(uint64(6), mustEncodeSet(slotDir+"2", "c"))
	p := make(FakePutterTo, 1)
	m := NewManager(self, 6, 1, st, p)
	p[0] = PutPutterTo{m}

	froms := make(chan int)

	fp := putFunc(func(msg Msg) {
		froms <- msg.From()
	})

	seqn, it := m.getInstance(seqnExp)
	assert.Equal(t, seqnExp, seqn)
	it.cPutter = fp
	it.aPutter = fp
	it.lPutter = fp

	v1 := newVote(1, exp)
	v1.SetSeqn(seqnExp)
	m.PutFrom(fromAddr, v1)

	assert.Equal(t, fromIndex, <-froms, "")
	assert.Equal(t, fromIndex, <-froms, "")
	assert.Equal(t, fromIndex, <-froms, "")
}

func TestManagerGetInstanceForPropose(t *testing.T) {
	m, _ := selfRefNewManager()
	seqn, _ := m.getInstance(0)
	assert.Equal(t, uint64(2), seqn)
}
<commit_msg>Don't hook m to itself; we just set outs anyway.<commit_after>package paxos

import (
	"junta/assert"
	"junta/store"
	"testing"
)

func selfRefNewManager(extraNodes ...string) (*Manager, *store.Store) {
	p := make(FakePutterTo, 1)
	st := store.New()
	self := "a"
	st.Apply(uint64(1), mustEncodeSet(membersKey+"/"+self, self+"addr"))
	for i, node := range extraNodes {
		st.Apply(uint64(i+2), mustEncodeSet(membersKey+"/"+node, node+"addr"))
	}
	m := NewManager(self, uint64(len(extraNodes)+1), 1, st, p)
	p[0] = PutPutterTo{m}
	return m, st
}

func TestProposeAndLearn(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp)
	got := ix.Value()

	assert.Equal(t, exp, got, "")
}

func TestProposeAndRecv(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp)
	got := ix.Value()
	assert.Equal(t, exp, got, "")

	seqn, v := m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestProposeAndRecvAltStart(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp)
	got := ix.Value()
	assert.Equal(t, exp, got, "")

	seqn, v := m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestProposeAndRecvMultiple(t *testing.T) {
	exp := []string{"/foo", "/bar"}
	seqnexp := []uint64{2, 3}
	m, st := selfRefNewManager()

	_, ix := m.getInstance(0)
	ix.Propose(exp[0])
	got0 := ix.Value()
	assert.Equal(t, exp[0], got0, "")

	seqn0, v0 := m.Recv()
	assert.Equal(t, seqnexp[0], seqn0, "seqn 1")
	assert.Equal(t, exp[0], v0, "")

	st.Apply(seqn0, v0)

	_, ix = m.getInstance(0)
	ix.Propose(exp[1])
	got1 := ix.Value()
	assert.Equal(t, exp[1], got1, "")

	seqn1, v1 := m.Recv()
	assert.Equal(t, seqnexp[1], seqn1, "seqn 1")
	assert.Equal(t, exp[1], v1, "")
}

func TestNewInstanceBecauseOfMessage(t *testing.T) {
	exp := "foo"
	m, _ := selfRefNewManager()

	m.Put(newVoteFrom(1, 1, exp))
	seqn, v := m.Recv()
	assert.Equal(t, uint64(1), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestNewInstanceBecauseOfMessageTriangulate(t *testing.T) {
	exp := "bar"
	m, _ := selfRefNewManager()

	m.Put(newVoteFrom(1, 1, exp))
	seqn, v := m.Recv()
	assert.Equal(t, uint64(1), seqn, "")
	assert.Equal(t, exp, v, "")
}

func TestUnusedSeqn(t *testing.T) {
	exp1, exp2 := "foo", "bar"
	m, _ := selfRefNewManager()

	m.Put(newVoteFrom(1, 1, exp1))
	seqn, v := m.Recv()
	assert.Equal(t, uint64(1), seqn, "")
	assert.Equal(t, exp1, v, "")

	_, ix := m.getInstance(0)
	ix.Propose(exp2)
	got := ix.Value()
	assert.Equal(t, exp2, got, "")
	seqn, v = m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, exp2, v, "")
}

func TestIgnoreMalformedMsg(t *testing.T) {
	m, _ := selfRefNewManager()

	m.Put(resize(newVoteFrom(1, 1, ""), -1))

	_, ix := m.getInstance(0)
	ix.Propose("y")
	got := ix.Value()
	assert.Equal(t, "y", got, "")

	seqn, v := m.Recv()
	assert.Equal(t, uint64(2), seqn, "")
	assert.Equal(t, "y", v, "")
}

func TestProposeAndStore(t *testing.T) {
	exp := "foo"
	mg, st := selfRefNewManager()

	go func() {
		for {
			st.Apply(mg.Recv())
		}
	}()

	_, got, _ := mg.Propose(exp)
	assert.Equal(t, exp, got, "")
}

func TestProposeBadMutation(t *testing.T) {
	mg, st := selfRefNewManager()

	go func() {
		for {
			st.Apply(mg.Recv())
		}
	}()

	_, _, err := mg.Propose("foo")
	assert.Equal(t, store.ErrBadMutation, err)
}

func mustEncodeSet(k, v string) string {
	m, err := store.EncodeSet(k, v, store.Clobber)
	if err != nil {
		panic(err)
	}
	return m
}

func TestReadFromStore(t *testing.T) {
	// The cluster initially has 1 node (quorum of 1).
	st := store.New()
	p := make(ChanPutCloserTo)
	self := "a"
	st.Apply(1, mustEncodeSet(membersDir+self, ""))
	st.Apply(2, mustEncodeSet(slotDir+"0", self))
	m := NewManager(self, 2, 1, st, p)

	// Fire up a new instance with a vote message. This instance should block
	// trying to read the list of members. If it doesn't wait, it'll
	// immediately learn the value `x`.
	in := newVoteFrom(0, 1, "x")
	in.SetSeqn(5)
	m.Put(in)

	// Satisfy the sync read of data members above. After this, there will be
	// 2 nodes in the cluster, making the quorum 2.
	st.Apply(3, mustEncodeSet(membersDir+"b", ""))
	st.Apply(4, mustEncodeSet(slotDir+"1", "b"))

	// Now try to make it learn a new value with 2 votes to meet the new
	// quorum.
	exp := "y"
	in = newVoteFrom(0, 2, exp)
	in.SetSeqn(5)
	m.Put(in)
	in = newVoteFrom(1, 2, exp)
	in.SetSeqn(5)
	m.Put(in)

	seqn, v := m.Recv()
	assert.Equal(t, uint64(5), seqn, "")
	assert.Equal(t, exp, v, "")
}

type putFunc func(Msg)

func (pf putFunc) Put(msg Msg) {
	go pf(msg)
}

func (pf putFunc) Close() {}

func TestManagerPutFrom(t *testing.T) {
	exp := "bar"
	seqnExp := uint64(7)
	fromAddr := "y"
	fromIndex := 1 // [a, b, c].indexof(b) => 1

	st := store.New()
	self := "a"
	st.Apply(uint64(1), mustEncodeSet(membersDir+self, "x"))
	st.Apply(uint64(2), mustEncodeSet(slotDir+"0", self))
	st.Apply(uint64(3), mustEncodeSet(membersDir+"b", "y"))
	st.Apply(uint64(4), mustEncodeSet(slotDir+"1", "b"))
	st.Apply(uint64(5), mustEncodeSet(membersDir+"c", "z"))
	st.Apply(uint64(6), mustEncodeSet(slotDir+"2", "c"))
	m := NewManager(self, 6, 1, st, make(FakePutterTo, 0))

	froms := make(chan int)

	fp := putFunc(func(msg Msg) {
		froms <- msg.From()
	})

	seqn, it := m.getInstance(seqnExp)
	assert.Equal(t, seqnExp, seqn)
	it.cPutter = fp
	it.aPutter = fp
	it.lPutter = fp

	v1 := newVote(1, exp)
	v1.SetSeqn(seqnExp)
	m.PutFrom(fromAddr, v1)

	assert.Equal(t, fromIndex, <-froms, "")
	assert.Equal(t, fromIndex, <-froms, "")
	assert.Equal(t, fromIndex, <-froms, "")
}

func TestManagerGetInstanceForPropose(t *testing.T) {
	m, _ := selfRefNewManager()
	seqn, _ := m.getInstance(0)
	assert.Equal(t, uint64(2), seqn)
}
<|endoftext|>


<commit_before>package rpc

import (
	"fmt"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"xd/lib/bittorrent/swarm"
	"xd/lib/config"
	"xd/lib/log"
	"xd/lib/rpc"
	"xd/lib/util"
)

func formatRate(r float64) string {
	str := util.FormatRate(r)
	for len(str) < 12 {
		str += " "
	}
	return str
}

func Run() {
	var args []string
	cmd := "list"
	fname := "torrents.ini"
	if len(os.Args) > 1 {
		cmd = os.Args[1]
		args = os.Args[2:]
	}
	cfg := new(config.Config)
	err := cfg.Load(fname)
	if err != nil {
		log.Errorf("error: %s", err)
		return
	}
	log.SetLevel(cfg.Log.Level)
	u := url.URL{
		Scheme: "http",
		Host:   cfg.RPC.Bind,
		Path:   rpc.RPCPath,
	}
	c := rpc.NewClient(u.String())

	switch strings.ToLower(cmd) {
	case "list":
		listTorrents(c)
	case "add":
		addTorrents(c, args...)
	case "set-piece-window":
		setPieceWindow(c, args[0])
	}
}

func setPieceWindow(c *rpc.Client, str string) {
	n, err := strconv.Atoi(str)
	if err != nil {
		log.Fatalf("error: %s", err.Error())
	}
	c.SetPieceWindow(n)
}

func addTorrents(c *rpc.Client, urls ...string) {
	for idx := range urls {
		fmt.Printf("fetch %s", urls[idx])
		c.AddTorrent(urls[idx])
	}
}

func listTorrents(c *rpc.Client) {
	var err error
	var list swarm.TorrentsList
	list, err = c.ListTorrents()
	if err != nil {
		log.Errorf("rpc error: %s", err)
		return
	}
	var globalTx, globalRx float64

	var torrents swarm.TorrentStatusList
	sort.Stable(&list.Infohashes)

	for _, ih := range list.Infohashes {
		var status swarm.TorrentStatus
		status, err = c.SwarmStatus(ih)

		if err != nil {
			log.Errorf("rpc error: %s", err)
			return
		}

		torrents = append(torrents, status)

	}
	sort.Stable(&torrents)
	for _, status := range torrents {
		var tx, rx float64
		fmt.Printf("%s [%s] %s\n", status.Name, status.Infohash, status.Bitfield.Percent())
		fmt.Println("peers:")
		sort.Stable(&status.Peers)
		for _, peer := range status.Peers {
			pad := peer.ID

			for len(pad) < 65 {
				pad += " "
			}
			fmt.Printf("\t%stx=%s rx=%s\n", pad, formatRate(peer.TX), formatRate(peer.RX))
			tx += peer.TX
			rx += peer.RX
		}
		fmt.Printf("%s tx=%s rx=%s\n", status.State, formatRate(tx), formatRate(rx))
		fmt.Println("files:")
		for idx, f := range status.Files {
			fmt.Printf("\t[%d] %s (%d/%d or %s)\n", idx, f.FileInfo.Path.FilePath(), f.Progress.CountSet(), f.Progress.Length, f.Progress.Percent())
		}
		fmt.Println()
		globalRx += rx
		globalTx += tx
	}
	fmt.Println()
	fmt.Printf("%d torrents: tx=%s rx=%s\n", list.Infohashes.Len(), formatRate(globalTx), formatRate(globalRx))
}
<commit_msg>add minimal help command<commit_after>package rpc

import (
	"fmt"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"xd/lib/bittorrent/swarm"
	"xd/lib/config"
	"xd/lib/log"
	"xd/lib/rpc"
	"xd/lib/util"
)

func formatRate(r float64) string {
	str := util.FormatRate(r)
	for len(str) < 12 {
		str += " "
	}
	return str
}

func Run() {
	var args []string
	cmd := "list"
	fname := "torrents.ini"
	if len(os.Args) > 1 {
		cmd = os.Args[1]
		args = os.Args[2:]
	}
	cfg := new(config.Config)
	err := cfg.Load(fname)
	if err != nil {
		log.Errorf("error: %s", err)
		return
	}
	log.SetLevel(cfg.Log.Level)
	u := url.URL{
		Scheme: "http",
		Host:   cfg.RPC.Bind,
		Path:   rpc.RPCPath,
	}
	c := rpc.NewClient(u.String())

	switch strings.ToLower(cmd) {
	case "list":
		listTorrents(c)
	case "add":
		addTorrents(c, args...)
	case "set-piece-window":
		setPieceWindow(c, args[0])
	case "help":
		printHelp(os.Args[0])
	}
}

func printHelp(cmd string) {
	fmt.Printf("usage: %s [list|add http://somesite.i2p/some.torrent|set-piece-window n|help]", cmd)
	fmt.Println()
}

func setPieceWindow(c *rpc.Client, str string) {
	n, err := strconv.Atoi(str)
	if err != nil {
		log.Fatalf("error: %s", err.Error())
	}
	c.SetPieceWindow(n)
}

func addTorrents(c *rpc.Client, urls ...string) {
	for idx := range urls {
		fmt.Printf("fetch %s", urls[idx])
		c.AddTorrent(urls[idx])
	}
}

func listTorrents(c *rpc.Client) {
	var err error
	var list swarm.TorrentsList
	list, err = c.ListTorrents()
	if err != nil {
		log.Errorf("rpc error: %s", err)
		return
	}
	var globalTx, globalRx float64

	var torrents swarm.TorrentStatusList
	sort.Stable(&list.Infohashes)

	for _, ih := range list.Infohashes {
		var status swarm.TorrentStatus
		status, err = c.SwarmStatus(ih)

		if err != nil {
			log.Errorf("rpc error: %s", err)
			return
		}

		torrents = append(torrents, status)

	}
	sort.Stable(&torrents)
	for _, status := range torrents {
		var tx, rx float64
		fmt.Printf("%s [%s] %s\n", status.Name, status.Infohash, status.Bitfield.Percent())
		fmt.Println("peers:")
		sort.Stable(&status.Peers)
		for _, peer := range status.Peers {
			pad := peer.ID

			for len(pad) < 65 {
				pad += " "
			}
			fmt.Printf("\t%stx=%s rx=%s\n", pad, formatRate(peer.TX), formatRate(peer.RX))
			tx += peer.TX
			rx += peer.RX
		}
		fmt.Printf("%s tx=%s rx=%s\n", status.State, formatRate(tx), formatRate(rx))
		fmt.Println("files:")
		for idx, f := range status.Files {
			fmt.Printf("\t[%d] %s (%d/%d or %s)\n", idx, f.FileInfo.Path.FilePath(), f.Progress.CountSet(), f.Progress.Length, f.Progress.Percent())
		}
		fmt.Println()
		globalRx += rx
		globalTx += tx
	}
	fmt.Println()
	fmt.Printf("%d torrents: tx=%s rx=%s\n", list.Infohashes.Len(), formatRate(globalTx), formatRate(globalRx))
}
<|endoftext|>


<commit_before>package permissions

import "errors"

var (
	// ErrInvalidToken is used when the token is invalid (the signature is not
	// correct, the domain is not the good one, etc.)
	ErrInvalidToken = errors.New("Invalid JWT token")

	// ErrInvalidAudience is used when the audience is not expected
	ErrInvalidAudience = errors.New("Invalid audience for JWT token")
)
<commit_msg>Make invalid token errors return 400 HTTP errors<commit_after>package permissions

import (
	"net/http"

	"github.com/labstack/echo"
)

var (
	// ErrInvalidToken is used when the token is invalid (the signature is not
	// correct, the domain is not the good one, etc.)
	ErrInvalidToken = echo.NewHTTPError(http.StatusBadRequest,
		"Invalid JWT token")

	// ErrInvalidAudience is used when the audience is not expected
	ErrInvalidAudience = echo.NewHTTPError(http.StatusBadRequest,
		"Invalid audience for JWT token")
)
<|endoftext|>


<commit_before>package template

import (
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"
	"text/template"
	"time"
)

var (
	// FuncMap contains the functions exposed to templating engine.
	FuncMap = template.FuncMap{
		// TODO confirmation prompt
		// TODO value prompt
		// TODO encoding utilities (e.g. toBinary)
		// TODO GET, POST utilities
		// TODO Hostname(Also accesible through $HOSTNAME), interface IP addr, etc.
		// TODO add validate for custom regex and expose validate package
		"env":      os.Getenv,
		"time":     CurrentTimeInFmt,
		"hostname": func() string { return os.Getenv("HOSTNAME") },

		"toBinary": func(s string) string {
			n, err := strconv.Atoi(s)
			if err != nil {
				return s
			}

			return fmt.Sprintf("%b", n)
		},

		"formatFilesize": func(value interface{}) string {
			var size float64

			v := reflect.ValueOf(value)
			switch v.Kind() {
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				size = float64(v.Int())
			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				size = float64(v.Uint())
			case reflect.Float32, reflect.Float64:
				size = v.Float()
			default:
				return ""
			}

			var KB float64 = 1 << 10
			var MB float64 = 1 << 20
			var GB float64 = 1 << 30
			var TB float64 = 1 << 40
			var PB float64 = 1 << 50

			filesizeFormat := func(filesize float64, suffix string) string {
				return strings.Replace(fmt.Sprintf("%.1f %s", filesize, suffix), ".0", "", -1)
			}

			var result string
			if size < KB {
				result = filesizeFormat(size, "bytes")
			} else if size < MB {
				result = filesizeFormat(size/KB, "KB")
			} else if size < GB {
				result = filesizeFormat(size/MB, "MB")
			} else if size < TB {
				result = filesizeFormat(size/GB, "GB")
			} else if size < PB {
				result = filesizeFormat(size/TB, "TB")
			} else {
				result = filesizeFormat(size/PB, "PB")
			}

			return result
		},

		// String utilities
		"toLower": strings.ToLower,
		"toUpper": strings.ToUpper,
		"toTitle": strings.ToTitle,
		"title":   strings.Title,

		"trimSpace":  strings.TrimSpace,
		"trimPrefix": strings.TrimPrefix,
		"trimSuffix": strings.TrimSuffix,

		"repeat": strings.Repeat,
	}

	// Options contain the default options for the template execution.
	Options = []string{
		// TODO ignore a field if no value is found instead of writing <no value>
		"missingkey=invalid",
	}
)

// CurrentTimeInFmt returns the current time in the given format.
// See time.Time.Format for more details on the format string.
func CurrentTimeInFmt(fmt string) string {
	t := time.Now()

	return t.Format(fmt)
}
<commit_msg>Added username template function.<commit_after>package template

import (
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"
	"text/template"
	"time"
	"os/user"
)

var (
	// FuncMap contains the functions exposed to templating engine.
	FuncMap = template.FuncMap{
		// TODO confirmation prompt
		// TODO value prompt
		// TODO encoding utilities (e.g. toBinary)
		// TODO GET, POST utilities
		// TODO Hostname(Also accesible through $HOSTNAME), interface IP addr, etc.
		// TODO add validate for custom regex and expose validate package
		"env":      os.Getenv,
		"time":     CurrentTimeInFmt,
		"hostname": func() string { return os.Getenv("HOSTNAME") },
		"username": UserName,

		"toBinary": func(s string) string {
			n, err := strconv.Atoi(s)
			if err != nil {
				return s
			}

			return fmt.Sprintf("%b", n)
		},

		"formatFilesize": func(value interface{}) string {
			var size float64

			v := reflect.ValueOf(value)
			switch v.Kind() {
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				size = float64(v.Int())
			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				size = float64(v.Uint())
			case reflect.Float32, reflect.Float64:
				size = v.Float()
			default:
				return ""
			}

			var KB float64 = 1 << 10
			var MB float64 = 1 << 20
			var GB float64 = 1 << 30
			var TB float64 = 1 << 40
			var PB float64 = 1 << 50

			filesizeFormat := func(filesize float64, suffix string) string {
				return strings.Replace(fmt.Sprintf("%.1f %s", filesize, suffix), ".0", "", -1)
			}

			var result string
			if size < KB {
				result = filesizeFormat(size, "bytes")
			} else if size < MB {
				result = filesizeFormat(size/KB, "KB")
			} else if size < GB {
				result = filesizeFormat(size/MB, "MB")
			} else if size < TB {
				result = filesizeFormat(size/GB, "GB")
			} else if size < PB {
				result = filesizeFormat(size/TB, "TB")
			} else {
				result = filesizeFormat(size/PB, "PB")
			}

			return result
		},

		// String utilities
		"toLower": strings.ToLower,
		"toUpper": strings.ToUpper,
		"toTitle": strings.ToTitle,
		"title":   strings.Title,

		"trimSpace":  strings.TrimSpace,
		"trimPrefix": strings.TrimPrefix,
		"trimSuffix": strings.TrimSuffix,

		"repeat": strings.Repeat,
	}

	// Options contain the default options for the template execution.
	Options = []string{
		// TODO ignore a field if no value is found instead of writing <no value>
		"missingkey=invalid",
	}
)

// CurrentTimeInFmt returns the current time in the given format.
// See time.Time.Format for more details on the format string.
func CurrentTimeInFmt(fmt string) string {
	t := time.Now()

	return t.Format(fmt)
}

// UserName returns the user's name (not the login name).
func UserName() string {
    t, err := user.Current();
    if ( err != nil ) {
        return "Unknown";
    }
    return t.Name;
}
<|endoftext|>


<commit_before>package benchmark

func TestFail(t *testing.T) {
	t.Errorf("force-fail the test")
}
<commit_msg>Revert "add a failing tests, just to confirm go test works as expected".<commit_after><|endoftext|>


<commit_before>package jiralert

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/andygrunwald/go-jira"
	"github.com/free/jiralert/alertmanager"
	log "github.com/golang/glog"
	"github.com/trivago/tgo/tcontainer"
)

// Receiver wraps a JIRA client corresponding to a specific Alertmanager receiver, with its configuration and templates.
type Receiver struct {
	conf   *ReceiverConfig
	tmpl   *Template
	client *jira.Client
}

// NewReceiver creates a Receiver using the provided configuration and template.
func NewReceiver(c *ReceiverConfig, t *Template) (*Receiver, error) {
	client, err := jira.NewClient(http.DefaultClient, c.APIURL)
	if err != nil {
		return nil, err
	}
	client.Authentication.SetBasicAuth(c.User, string(c.Password))

	return &Receiver{conf: c, tmpl: t, client: client}, nil
}

// Notify implements the Notifier interface.
func (r *Receiver) Notify(data *alertmanager.Data) (bool, error) {
	project := r.tmpl.Execute(r.conf.Project, data)
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	// Looks like an ALERT metric name, with spaces removed.
	issueLabel := toIssueLabel(data.GroupLabels)

	issue, retry, err := r.search(project, issueLabel)
	if err != nil {
		return retry, err
	}

	if issue != nil {
		// The set of JIRA status categories is fixed, this is a safe check to make.
		if issue.Fields.Status.StatusCategory.Key != "done" {
			// Issue is in a "to do" or "in progress" state, all done here.
			log.V(1).Infof("Issue %s for %s is unresolved, nothing to do", issue.Key, issueLabel)
			return false, nil
		}
		if r.conf.WontFixResolution != "" && issue.Fields.Resolution.Name == r.conf.WontFixResolution {
			// Issue is resolved as "Won't Fix" or equivalent, log a message just in case.
			log.Infof("Issue %s for %s is resolved as %q, not reopening", issue.Key, issueLabel, issue.Fields.Resolution.Name)
			return false, nil
		}
		log.Infof("Issue %s for %s was resolved, reopening", issue.Key, issueLabel)
		return r.reopen(issue.Key)
	}

	log.Infof("No issue matching %s found, creating new issue", issueLabel)
	issue = &jira.Issue{
		Fields: &jira.IssueFields{
			Project:     jira.Project{Key: project},
			Type:        jira.IssueType{Name: r.tmpl.Execute(r.conf.IssueType, data)},
			Description: r.tmpl.Execute(r.conf.Description, data),
			Summary:     r.tmpl.Execute(r.conf.Summary, data),
			Labels: []string{
				issueLabel,
			},
			Unknowns: tcontainer.NewMarshalMap(),
		},
	}
	if r.conf.Priority != "" {
		issue.Fields.Priority = &jira.Priority{Name: r.tmpl.Execute(r.conf.Priority, data)}
	}

	// Add Components
	issue.Fields.Components = make([]*jira.Component, 0, len(r.conf.Components))
	for _, component := range r.conf.Components {
		issue.Fields.Components = append(issue.Fields.Components, &jira.Component{Name: component})
	}

	// Add Labels
	if r.conf.AddGroupLabels {
		for k, v := range data.GroupLabels {
			issue.Fields.Labels = append(issue.Fields.Labels, fmt.Sprintf("%s=\"%q\"", k, v))
		}
	}

	for key, value := range r.conf.Fields {
		issue.Fields.Unknowns[key] = r.tmpl.Execute(fmt.Sprint(value), data)
	}
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	retry, err = r.create(issue)
	if err != nil {
		log.Infof("Issue created: key=%s ID=%s", issue.Key, issue.ID)
	}
	return retry, err
}

// toIssueLabel returns the group labels in the form of an ALERT metric name, with all spaces removed.
func toIssueLabel(groupLabels alertmanager.KV) string {
	buf := bytes.NewBufferString("ALERT{")
	for _, p := range groupLabels.SortedPairs() {
		buf.WriteString(p.Name)
		buf.WriteString(fmt.Sprintf("=%q,", p.Value))
	}
	buf.Truncate(buf.Len() - 1)
	buf.WriteString("}")
	return strings.Replace(buf.String(), " ", "", -1)
}

func (r *Receiver) search(project, issueLabel string) (*jira.Issue, bool, error) {
	query := fmt.Sprintf("project=%s and labels=%q order by key", project, issueLabel)
	options := &jira.SearchOptions{
		Fields:     []string{"summary", "status", "resolution"},
		MaxResults: 50,
	}
	log.V(1).Infof("search: query=%v options=%+v", query, options)
	issues, resp, err := r.client.Issue.Search(query, options)
	if err != nil {
		retry, err := handleJiraError(resp, err)
		return nil, retry, err
	}
	if len(issues) > 0 {
		if len(issues) > 1 {
			// Swallow it, but log an error.
			log.Errorf("More than one issue matched %s, will only update first: %+v", query, issues)
		}
		log.V(1).Infof("  found: %+v", issues[0])
		return &issues[0], false, nil
	}
	log.V(1).Infof("  no results")
	return nil, false, nil
}

func (r *Receiver) reopen(issueKey string) (bool, error) {
	transitions, resp, err := r.client.Issue.GetTransitions(issueKey)
	if err != nil {
		return handleJiraError(resp, err)
	}
	for _, t := range transitions {
		if t.Name == r.conf.ReopenState {
			log.V(1).Infof("reopen: issueKey=%v transitionID=%v", issueKey, t.ID)
			resp, err = r.client.Issue.DoTransition(issueKey, t.ID)
			if err != nil {
				return handleJiraError(resp, err)
			}
			log.V(1).Infof("  done")
			return false, nil
		}
	}
	return false, fmt.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
}

func (r *Receiver) create(issue *jira.Issue) (bool, error) {
	log.V(1).Infof("create: issue=%v", *issue)
	issue, resp, err := r.client.Issue.Create(issue)
	if err != nil {
		return handleJiraError(resp, err)
	}

	log.V(1).Infof("  done: key=%s ID=%s", issue.Key, issue.ID)
	return false, nil
}

func handleJiraError(resp *jira.Response, err error) (bool, error) {
	log.V(1).Infof("handleJiraError: err=%s, req=%s", err, resp.Request.URL)
	if resp != nil && resp.StatusCode/100 != 2 {
		retry := resp.StatusCode == 500 || resp.StatusCode == 503
		body, _ := ioutil.ReadAll(resp.Body)
		// go-jira error message is not particularly helpful, replace it
		return retry, fmt.Errorf("JIRA request %s returned status %s, body %q", resp.Request.URL, resp.Status, string(body))
	}
	return false, err
}
<commit_msg>Fixing format string<commit_after>package jiralert

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/andygrunwald/go-jira"
	"github.com/free/jiralert/alertmanager"
	log "github.com/golang/glog"
	"github.com/trivago/tgo/tcontainer"
)

// Receiver wraps a JIRA client corresponding to a specific Alertmanager receiver, with its configuration and templates.
type Receiver struct {
	conf   *ReceiverConfig
	tmpl   *Template
	client *jira.Client
}

// NewReceiver creates a Receiver using the provided configuration and template.
func NewReceiver(c *ReceiverConfig, t *Template) (*Receiver, error) {
	client, err := jira.NewClient(http.DefaultClient, c.APIURL)
	if err != nil {
		return nil, err
	}
	client.Authentication.SetBasicAuth(c.User, string(c.Password))

	return &Receiver{conf: c, tmpl: t, client: client}, nil
}

// Notify implements the Notifier interface.
func (r *Receiver) Notify(data *alertmanager.Data) (bool, error) {
	project := r.tmpl.Execute(r.conf.Project, data)
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	// Looks like an ALERT metric name, with spaces removed.
	issueLabel := toIssueLabel(data.GroupLabels)

	issue, retry, err := r.search(project, issueLabel)
	if err != nil {
		return retry, err
	}

	if issue != nil {
		// The set of JIRA status categories is fixed, this is a safe check to make.
		if issue.Fields.Status.StatusCategory.Key != "done" {
			// Issue is in a "to do" or "in progress" state, all done here.
			log.V(1).Infof("Issue %s for %s is unresolved, nothing to do", issue.Key, issueLabel)
			return false, nil
		}
		if r.conf.WontFixResolution != "" && issue.Fields.Resolution.Name == r.conf.WontFixResolution {
			// Issue is resolved as "Won't Fix" or equivalent, log a message just in case.
			log.Infof("Issue %s for %s is resolved as %q, not reopening", issue.Key, issueLabel, issue.Fields.Resolution.Name)
			return false, nil
		}
		log.Infof("Issue %s for %s was resolved, reopening", issue.Key, issueLabel)
		return r.reopen(issue.Key)
	}

	log.Infof("No issue matching %s found, creating new issue", issueLabel)
	issue = &jira.Issue{
		Fields: &jira.IssueFields{
			Project:     jira.Project{Key: project},
			Type:        jira.IssueType{Name: r.tmpl.Execute(r.conf.IssueType, data)},
			Description: r.tmpl.Execute(r.conf.Description, data),
			Summary:     r.tmpl.Execute(r.conf.Summary, data),
			Labels: []string{
				issueLabel,
			},
			Unknowns: tcontainer.NewMarshalMap(),
		},
	}
	if r.conf.Priority != "" {
		issue.Fields.Priority = &jira.Priority{Name: r.tmpl.Execute(r.conf.Priority, data)}
	}

	// Add Components
	issue.Fields.Components = make([]*jira.Component, 0, len(r.conf.Components))
	for _, component := range r.conf.Components {
		issue.Fields.Components = append(issue.Fields.Components, &jira.Component{Name: component})
	}

	// Add Labels
	if r.conf.AddGroupLabels {
		for k, v := range data.GroupLabels {
			issue.Fields.Labels = append(issue.Fields.Labels, fmt.Sprintf("%s=%q", k, v))
		}
	}

	for key, value := range r.conf.Fields {
		issue.Fields.Unknowns[key] = r.tmpl.Execute(fmt.Sprint(value), data)
	}
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	retry, err = r.create(issue)
	if err != nil {
		log.Infof("Issue created: key=%s ID=%s", issue.Key, issue.ID)
	}
	return retry, err
}

// toIssueLabel returns the group labels in the form of an ALERT metric name, with all spaces removed.
func toIssueLabel(groupLabels alertmanager.KV) string {
	buf := bytes.NewBufferString("ALERT{")
	for _, p := range groupLabels.SortedPairs() {
		buf.WriteString(p.Name)
		buf.WriteString(fmt.Sprintf("=%q,", p.Value))
	}
	buf.Truncate(buf.Len() - 1)
	buf.WriteString("}")
	return strings.Replace(buf.String(), " ", "", -1)
}

func (r *Receiver) search(project, issueLabel string) (*jira.Issue, bool, error) {
	query := fmt.Sprintf("project=%s and labels=%q order by key", project, issueLabel)
	options := &jira.SearchOptions{
		Fields:     []string{"summary", "status", "resolution"},
		MaxResults: 50,
	}
	log.V(1).Infof("search: query=%v options=%+v", query, options)
	issues, resp, err := r.client.Issue.Search(query, options)
	if err != nil {
		retry, err := handleJiraError(resp, err)
		return nil, retry, err
	}
	if len(issues) > 0 {
		if len(issues) > 1 {
			// Swallow it, but log an error.
			log.Errorf("More than one issue matched %s, will only update first: %+v", query, issues)
		}
		log.V(1).Infof("  found: %+v", issues[0])
		return &issues[0], false, nil
	}
	log.V(1).Infof("  no results")
	return nil, false, nil
}

func (r *Receiver) reopen(issueKey string) (bool, error) {
	transitions, resp, err := r.client.Issue.GetTransitions(issueKey)
	if err != nil {
		return handleJiraError(resp, err)
	}
	for _, t := range transitions {
		if t.Name == r.conf.ReopenState {
			log.V(1).Infof("reopen: issueKey=%v transitionID=%v", issueKey, t.ID)
			resp, err = r.client.Issue.DoTransition(issueKey, t.ID)
			if err != nil {
				return handleJiraError(resp, err)
			}
			log.V(1).Infof("  done")
			return false, nil
		}
	}
	return false, fmt.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
}

func (r *Receiver) create(issue *jira.Issue) (bool, error) {
	log.V(1).Infof("create: issue=%v", *issue)
	issue, resp, err := r.client.Issue.Create(issue)
	if err != nil {
		return handleJiraError(resp, err)
	}

	log.V(1).Infof("  done: key=%s ID=%s", issue.Key, issue.ID)
	return false, nil
}

func handleJiraError(resp *jira.Response, err error) (bool, error) {
	log.V(1).Infof("handleJiraError: err=%s, req=%s", err, resp.Request.URL)
	if resp != nil && resp.StatusCode/100 != 2 {
		retry := resp.StatusCode == 500 || resp.StatusCode == 503
		body, _ := ioutil.ReadAll(resp.Body)
		// go-jira error message is not particularly helpful, replace it
		return retry, fmt.Errorf("JIRA request %s returned status %s, body %q", resp.Request.URL, resp.Status, string(body))
	}
	return false, err
}
<|endoftext|>


<commit_before>package luddite

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/julienschmidt/httprouter"
)

type SchemaHandler struct {
	fileServer http.Handler
}

func NewSchemaHandler(filePath string) *SchemaHandler {
	return &SchemaHandler{http.FileServer(http.Dir(filePath))}
}

func (h *SchemaHandler) ServeHTTP(rw http.ResponseWriter, req0 *http.Request, params httprouter.Params) {
	versionStr := params.ByName("version")

	version, err := strconv.Atoi(versionStr)
	if err != nil || version < 1 {
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	file := fmt.Sprintf("/v%d/%s", version, params.ByName("filepath"))
	req1, err := http.NewRequest("GET", file, nil)
	if err != nil {
		panic(err)
	}

	rw.Header().Del(HeaderContentType)
	h.fileServer.ServeHTTP(rw, req1)
}
<commit_msg>Force Content-Type header for YAML schema files<commit_after>package luddite

import (
	"fmt"
	"net/http"
	"path"
	"strconv"
	"strings"

	"github.com/julienschmidt/httprouter"
)

type SchemaHandler struct {
	fileServer http.Handler
}

func NewSchemaHandler(filePath string) *SchemaHandler {
	return &SchemaHandler{http.FileServer(http.Dir(filePath))}
}

func (h *SchemaHandler) ServeHTTP(rw http.ResponseWriter, req0 *http.Request, params httprouter.Params) {
	versionStr := params.ByName("version")

	version, err := strconv.Atoi(versionStr)
	if err != nil || version < 1 {
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	filepath := params.ByName("filepath")
	file := fmt.Sprintf("/v%d/%s", version, filepath)
	req1, err := http.NewRequest("GET", file, nil)
	if err != nil {
		panic(err)
	}

	switch strings.ToLower(path.Ext(filepath)) {
	case ".yaml", ".yml":
		rw.Header().Set(HeaderContentType, ContentTypeOctetStream)
	default:
		rw.Header().Del(HeaderContentType)
	}

	h.fileServer.ServeHTTP(rw, req1)
}
<|endoftext|>


<commit_before>// @author Couchbase <info@couchbase.com>
// @copyright 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repository

import (
	"fmt"
	"github.com/couchbase/gometa/common"
	logging "github.com/couchbase/gometa/log"
	"github.com/couchbase/gometa/message"
	// fdb "github.com/couchbase/goforestdb"
	fdb "github.com/couchbase/indexing/secondary/fdb"
	"sync"
	"time"
)

/////////////////////////////////////////////////////////////////////////////
// TransientCommitLog
/////////////////////////////////////////////////////////////////////////////

type TransientCommitLog struct {
	repo    *Repository
	factory *message.ConcreteMsgFactory
	logs    map[common.Txnid]*message.LogEntry
	mutex   sync.Mutex
}

type TransientLogIterator struct {
	repo  *Repository
	txnid common.Txnid
	iter  *RepoIterator

	curTxnid   common.Txnid
	curKey     string
	curContent []byte
	curError   error
}

/////////////////////////////////////////////////////////////////////////////
// CommitLog Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new commit log
//
func NewTransientCommitLog(repo *Repository, lastCommittedTxnid common.Txnid) (CommitLogger, error) {

	log := &TransientCommitLog{
		repo:    repo,
		factory: message.NewConcreteMsgFactory(),
		logs:    make(map[common.Txnid]*message.LogEntry)}

	if lastCommittedTxnid != common.BOOTSTRAP_LAST_COMMITTED_TXID {
		if err := repo.CreateSnapshot(MAIN, lastCommittedTxnid); err != nil {
			logging.Current.Errorf("NewTransientCommitLog: Cannot create initial snapshot")
			return nil, err
		}
	}

	return log, nil
}

//
// Add Entry to commit log
//
func (r *TransientCommitLog) Log(txid common.Txnid, op common.OpCode, key string, content []byte) error {

	msg := r.factory.CreateLogEntry(uint64(txid), uint32(op), key, content)
	r.logs[txid] = msg.(*message.LogEntry)
	return nil
}

//
// Retrieve entry from commit log
//
func (r *TransientCommitLog) Get(txid common.Txnid) (common.OpCode, string, []byte, error) {

	msg, ok := r.logs[txid]
	if !ok || msg == nil {
		err := common.NewError(common.REPO_ERROR, fmt.Sprintf("LogEntry for txid %d does not exist in commit log", txid))
		return common.OPCODE_INVALID, "", nil, err
	}

	return common.GetOpCodeFromInt(msg.GetOpCode()), msg.GetKey(), msg.GetContent(), nil
}

//
// Delete from commit log
//
func (r *TransientCommitLog) Delete(txid common.Txnid) error {
	delete(r.logs, txid)
	return nil
}

//
// Mark a log entry has been committted.   This function is called when there
// is no concurrent commit for the given txid to match the repo snapshot.
//
func (r *TransientCommitLog) MarkCommitted(txid common.Txnid) error {

	delete(r.logs, txid)
	if err := r.repo.CreateSnapshot(MAIN, txid); err != nil {
		return err
	}

	return nil
}

/////////////////////////////////////////////////////////////////////////////
// LogIterator Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new iterator.   This is used by LeaderSyncProxy to replicate log
// entries to a folower/watcher.  txid1 is last logged commited txid from follower/watcher.
// txid2 is the txid of the first packet in the observer queue for this follower/watcher.
// If the observer queue is empty, then txid2 is 0.
//
// For any LogEntry returned form the iterator, the following condition must be satisifed:
// 1) The txid of any LogEntry returned from the iterator must be greater than or equal to txid1.
// 2) The last LogEntry must have txid >= txid2.  In other words, the iterator can gaurantee
//    that it will cover any mutation by txid2, but cannot guarantee that it stops at txid2.
//
// Since TransientCommitLog does not keep track of history, it only supports the following case:
// 1) the beginning txid (txid1) is 0 (sending the full repository)
//
func (r *TransientCommitLog) NewIterator(txid1, txid2 common.Txnid) (CommitLogIterator, error) {

	if txid1 != 0 {
		return nil, common.NewError(common.REPO_ERROR, "TransientLCommitLog.NewIterator: cannot support beginning txid > 0")
	}

	// iter can be nil if nothing has been logged
	retry := true

retryLabel:
	txnid, iter, err := r.repo.AcquireSnapshot(MAIN)
	if err != nil {
		return nil, err
	}

	// The snapshot txnid must be at least match the ending txid (txid2).  If not, it will retry.
	// This is to ensure that it takes care race condition in which the first msg in an observer's
	// queue is a commit, and the caller asks for the state of the repository which is at least as
	// recent as that commit msg (txid2).  If retry fails, return an error.
	if txnid < txid2 {
		if retry {
			time.Sleep(common.XACT_COMMIT_WAIT_TIME)
			retry = false
			goto retryLabel
		}

		return nil, common.NewError(common.REPO_ERROR,
			fmt.Sprintf("TransientLCommitLog.NewIterator: cannot support ending txid > %d", txnid))
	}

	result := &TransientLogIterator{
		iter:       iter,
		txnid:      txnid,
		repo:       r.repo,
		curTxnid:   common.Txnid(txid1 + 1),
		curKey:     "",
		curContent: nil,
		curError:   nil}

	if result.iter != nil {
		result.curKey, result.curContent, result.curError = result.iter.Next()
	}

	return result, nil
}

// Get value from iterator
func (i *TransientLogIterator) Next() (txnid common.Txnid, op common.OpCode, key string, content []byte, err error) {

	if i.iter == nil {
		return 0, common.OPCODE_INVALID, "", nil, fdb.RESULT_ITERATOR_FAIL
	}

	if i.curError != nil {
		return 0, common.OPCODE_INVALID, "", nil, i.curError
	}

	key = i.curKey
	content = i.curContent
	txnid = i.curTxnid

	i.curTxnid = common.Txnid(uint64(i.curTxnid) + 1)
	i.curKey, i.curContent, i.curError = i.iter.Next()

	if i.curError == nil {
		// it is not the last entry. Does not matter what is the actual txnid as long as it is
		// smaller than the snapshot's txnid (TransientLogIterator.txnid).
		return txnid, common.OPCODE_SET, key, content, nil
	}

	// last entry : use the txnid matching the snapshot
	return i.txnid, common.OPCODE_SET, key, content, nil
}

// close iterator
func (i *TransientLogIterator) Close() {

	// TODO: Check if fdb iterator is closed
	if i.iter != nil {
		i.iter.Close()
	}
	i.repo.ReleaseSnapshot(MAIN, i.txnid)
}
<commit_msg>Make cgo happy in go1.6<commit_after>// @author Couchbase <info@couchbase.com>
// @copyright 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repository

import (
	"fmt"
	"github.com/couchbase/gometa/common"
	logging "github.com/couchbase/gometa/log"
	"github.com/couchbase/gometa/message"
	// fdb "github.com/couchbase/goforestdb"
	fdb "github.com/couchbase/indexing/secondary/fdb"
	"sync"
	"time"
)

/////////////////////////////////////////////////////////////////////////////
// TransientCommitLog
/////////////////////////////////////////////////////////////////////////////

type TransientCommitLog struct {
	repo    *Repository
	factory *message.ConcreteMsgFactory
	logs    map[common.Txnid]*message.LogEntry
	mutex   sync.Mutex
}

type TransientLogIterator struct {
	repo  *Repository
	txnid common.Txnid
	iter  *RepoIterator

	curTxnid   common.Txnid
	curKey     string
	curContent []byte
	curError   error
}

/////////////////////////////////////////////////////////////////////////////
// CommitLog Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new commit log
//
func NewTransientCommitLog(repo *Repository, lastCommittedTxnid common.Txnid) (CommitLogger, error) {

	log := &TransientCommitLog{
		repo:    repo,
		factory: message.NewConcreteMsgFactory(),
		logs:    make(map[common.Txnid]*message.LogEntry)}

	if lastCommittedTxnid != common.BOOTSTRAP_LAST_COMMITTED_TXID {
		if err := repo.CreateSnapshot(MAIN, lastCommittedTxnid); err != nil {
			logging.Current.Errorf("NewTransientCommitLog: Cannot create initial snapshot")
			return nil, err
		}
	}

	return log, nil
}

//
// Add Entry to commit log
//
func (r *TransientCommitLog) Log(txid common.Txnid, op common.OpCode, key string, content []byte) error {

	msg := r.factory.CreateLogEntry(uint64(txid), uint32(op), key, content)
	r.logs[txid] = msg.(*message.LogEntry)
	return nil
}

//
// Retrieve entry from commit log
//
func (r *TransientCommitLog) Get(txid common.Txnid) (common.OpCode, string, []byte, error) {

	msg, ok := r.logs[txid]
	if !ok || msg == nil {
		err := common.NewError(common.REPO_ERROR, fmt.Sprintf("LogEntry for txid %d does not exist in commit log", txid))
		return common.OPCODE_INVALID, "", nil, err
	}

	// msg is a protobuf object. Directly pointing slices/strings into a struct may cause problems for cgo calls
	key := string([]byte(msg.GetKey()))
	content := []byte(string(msg.GetContent()))
	return common.GetOpCodeFromInt(msg.GetOpCode()), key, content, nil
}

//
// Delete from commit log
//
func (r *TransientCommitLog) Delete(txid common.Txnid) error {
	delete(r.logs, txid)
	return nil
}

//
// Mark a log entry has been committted.   This function is called when there
// is no concurrent commit for the given txid to match the repo snapshot.
//
func (r *TransientCommitLog) MarkCommitted(txid common.Txnid) error {

	delete(r.logs, txid)
	if err := r.repo.CreateSnapshot(MAIN, txid); err != nil {
		return err
	}

	return nil
}

/////////////////////////////////////////////////////////////////////////////
// LogIterator Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new iterator.   This is used by LeaderSyncProxy to replicate log
// entries to a folower/watcher.  txid1 is last logged commited txid from follower/watcher.
// txid2 is the txid of the first packet in the observer queue for this follower/watcher.
// If the observer queue is empty, then txid2 is 0.
//
// For any LogEntry returned form the iterator, the following condition must be satisifed:
// 1) The txid of any LogEntry returned from the iterator must be greater than or equal to txid1.
// 2) The last LogEntry must have txid >= txid2.  In other words, the iterator can gaurantee
//    that it will cover any mutation by txid2, but cannot guarantee that it stops at txid2.
//
// Since TransientCommitLog does not keep track of history, it only supports the following case:
// 1) the beginning txid (txid1) is 0 (sending the full repository)
//
func (r *TransientCommitLog) NewIterator(txid1, txid2 common.Txnid) (CommitLogIterator, error) {

	if txid1 != 0 {
		return nil, common.NewError(common.REPO_ERROR, "TransientLCommitLog.NewIterator: cannot support beginning txid > 0")
	}

	// iter can be nil if nothing has been logged
	retry := true

retryLabel:
	txnid, iter, err := r.repo.AcquireSnapshot(MAIN)
	if err != nil {
		return nil, err
	}

	// The snapshot txnid must be at least match the ending txid (txid2).  If not, it will retry.
	// This is to ensure that it takes care race condition in which the first msg in an observer's
	// queue is a commit, and the caller asks for the state of the repository which is at least as
	// recent as that commit msg (txid2).  If retry fails, return an error.
	if txnid < txid2 {
		if retry {
			time.Sleep(common.XACT_COMMIT_WAIT_TIME)
			retry = false
			goto retryLabel
		}

		return nil, common.NewError(common.REPO_ERROR,
			fmt.Sprintf("TransientLCommitLog.NewIterator: cannot support ending txid > %d", txnid))
	}

	result := &TransientLogIterator{
		iter:       iter,
		txnid:      txnid,
		repo:       r.repo,
		curTxnid:   common.Txnid(txid1 + 1),
		curKey:     "",
		curContent: nil,
		curError:   nil}

	if result.iter != nil {
		result.curKey, result.curContent, result.curError = result.iter.Next()
	}

	return result, nil
}

// Get value from iterator
func (i *TransientLogIterator) Next() (txnid common.Txnid, op common.OpCode, key string, content []byte, err error) {

	if i.iter == nil {
		return 0, common.OPCODE_INVALID, "", nil, fdb.RESULT_ITERATOR_FAIL
	}

	if i.curError != nil {
		return 0, common.OPCODE_INVALID, "", nil, i.curError
	}

	key = i.curKey
	content = i.curContent
	txnid = i.curTxnid

	i.curTxnid = common.Txnid(uint64(i.curTxnid) + 1)
	i.curKey, i.curContent, i.curError = i.iter.Next()

	if i.curError == nil {
		// it is not the last entry. Does not matter what is the actual txnid as long as it is
		// smaller than the snapshot's txnid (TransientLogIterator.txnid).
		return txnid, common.OPCODE_SET, key, content, nil
	}

	// last entry : use the txnid matching the snapshot
	return i.txnid, common.OPCODE_SET, key, content, nil
}

// close iterator
func (i *TransientLogIterator) Close() {

	// TODO: Check if fdb iterator is closed
	if i.iter != nil {
		i.iter.Close()
	}
	i.repo.ReleaseSnapshot(MAIN, i.txnid)
}
<|endoftext|>