<commit_before>// Copyright 2011 Bobby Powers. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.
package seshcookie

import (
	"http"
	"os"
	"log"
	"time"
	"gob"
	"bytes"
	"sync"
	"sync/atomic"
	"hash"
	"crypto/sha1"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/hmac"
	"encoding/base64"
)

// we want 16 byte blocks, for AES-128
const blockSize = 16

var (
	// if you don't need multiple independent seshcookie
	// instances, you can use this RequestSessions instance to
	// manage & access your sessions.  Simply use it as the final
	// parameter in your call to seshcookie.NewSessionHandler, and
	// whenever you want to access the current session from an
	// embedded http.Handler you can simply call:
	//
	//     seshcookie.Session.Get(req)
	Session = new(RequestSessions)

	// Hash validation of the decrypted cookie failed. Most likely
	// the session was encoded with a different cookie than we're
	// using to decode it, but its possible the client (or someone
	// else) tried to modify the session.
	HashError = os.NewError("Hash validation failed")

	// The cookie is too short, so we must exit decoding early.
	LenError = os.NewError("Bad cookie length")
)

type sessionResponseWriter struct {
	http.ResponseWriter
	h           *SessionHandler
	req         *http.Request
	// int32 so we can use the sync/atomic functions on it
	wroteHeader int32
}

type SessionHandler struct {
	http.Handler
	// The name of the cookie our encoded session will be stored
	// in.
	CookieName string
	RS         *RequestSessions
	encKey     []byte
	hmacKey    []byte
}

type RequestSessions struct {
	lk sync.Mutex
	m  map[*http.Request]map[string]interface{}
	hm map[*http.Request][]byte
}

func (rs *RequestSessions) Get(req *http.Request) map[string]interface{} {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	if rs.m == nil {
		return nil
	}

	return rs.m[req]
}

func (rs *RequestSessions) getHash(req *http.Request) []byte {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	if rs.hm == nil {
		return nil
	}

	return rs.hm[req]
}

func (rs *RequestSessions) Set(req *http.Request, val map[string]interface{}, gobHash []byte) {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	if rs.m == nil {
		rs.m = map[*http.Request]map[string]interface{}{}
		rs.hm = map[*http.Request][]byte{}
	}

	rs.m[req] = val
	rs.hm[req] = gobHash
}

func (rs *RequestSessions) Clear(req *http.Request) {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	rs.m[req] = nil, false
}

func encodeGob(obj interface{}) ([]byte, os.Error) {
	buf := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(obj)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func decodeGob(encoded []byte) (map[string]interface{}, os.Error) {
	buf := bytes.NewBuffer(encoded)
	dec := gob.NewDecoder(buf)
	var out map[string]interface{}
	err := dec.Decode(&out)
	if err != nil {
		return nil, err
	}
	return out, nil
}

// encode uses the given block cipher (in CTR mode) to encrypt the
// data, along with a hash, returning the iv and the ciphertext. What
// is returned looks like:
//
//   encrypted(salt + sessionData) + iv + hmac
//
func encode(block cipher.Block, hmac hash.Hash, data []byte) ([]byte, os.Error) {

	buf := bytes.NewBuffer(nil)

	salt := make([]byte, block.BlockSize())
	if _, err := rand.Read(salt); err != nil {
		return nil, err
	}
	buf.Write(salt)
	buf.Write(data)

	session := buf.Bytes()

	iv := make([]byte, block.BlockSize())
	if _, err := rand.Read(iv); err != nil {
		return nil, err
	}

	stream := cipher.NewCTR(block, iv)
	stream.XORKeyStream(session, session)

	buf.Write(iv)
	hmac.Write(buf.Bytes())
	buf.Write(hmac.Sum())

	return buf.Bytes(), nil
}

func encodeCookie(content interface{}, encKey, hmacKey []byte) (string, []byte, os.Error) {
	encodedGob, err := encodeGob(content)
	if err != nil {
		return "", nil, err
	}

	gobHash := sha1.New()
	gobHash.Write(encodedGob)

	aesCipher, err := aes.NewCipher(encKey)
	if err != nil {
		return "", nil, err
	}

	hmacHash := hmac.NewSHA256(hmacKey)

	sessionBytes, err := encode(aesCipher, hmacHash, encodedGob)
	if err != nil {
		return "", nil, err
	}

	return base64.StdEncoding.EncodeToString(sessionBytes), gobHash.Sum(), nil
}

// decode uses the given block cipher (in CTR mode) to decrypt the
// data, and validate the hash.  If hash validation fails, an error is
// returned.
func decode(block cipher.Block, hmac hash.Hash, ciphertext []byte) ([]byte, os.Error) {
	if len(ciphertext) < 2 * block.BlockSize() + hmac.Size() {
		return nil, LenError
	}

	receivedHmac := ciphertext[len(ciphertext) - hmac.Size():]
	ciphertext = ciphertext[:len(ciphertext) - hmac.Size()]

	hmac.Write(ciphertext)
	if !bytes.Equal(hmac.Sum(), receivedHmac) {
		return nil, HashError
	}

	// split the iv and session bytes
	iv := ciphertext[len(ciphertext) - block.BlockSize():]
	session := ciphertext[:len(ciphertext) - block.BlockSize()]

	stream := cipher.NewCTR(block, iv)
	stream.XORKeyStream(session, session)

	// skip past the iv
	session = session[block.BlockSize():]

	return session, nil
}

func decodeCookie(encoded string, encKey, hmacKey []byte) (map[string]interface{}, []byte, os.Error) {
	sessionBytes, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return nil, nil, err
	}
	aesCipher, err := aes.NewCipher(encKey)
	if err != nil {
		return nil, nil, err
	}

	hmacHash := hmac.NewSHA256(hmacKey)
	gobBytes, err := decode(aesCipher, hmacHash, sessionBytes)
	if err != nil {
		return nil, nil, err
	}

	gobHash := sha1.New()
	gobHash.Write(gobBytes)

	session, err := decodeGob(gobBytes)
	if err != nil {
		log.Printf("decodeGob: %s\n", err)
		return nil, nil, err
	}
	return session, gobHash.Sum(), nil
}

func (s sessionResponseWriter) WriteHeader(code int) {
	if atomic.AddInt32(&s.wroteHeader, 1) == 1 {
		origCookie, err := s.req.Cookie(s.h.CookieName)
		var origCookieVal string
		if err != nil {
			origCookieVal = ""
		} else {
			origCookieVal = origCookie.Value
		}

		session := s.h.RS.Get(s.req)
		if len(session) == 0 {
			// if we have an empty session, but the
			// request didn't start out that way, we
			// assume the user wants us to clear the
			// session
			if origCookieVal != "" {
				log.Println("clearing cookie")
				var cookie http.Cookie
				cookie.Name = s.h.CookieName
				cookie.Value = ""
				cookie.Path = "/"
				// a cookie is expired by setting it
				// with an expiration time in the past
				cookie.Expires = *time.SecondsToUTC(0)
				http.SetCookie(s, &cookie)
			} else {
				log.Println("not setting empty cookie")
			}
			goto write
		}
		encoded, gobHash, err := encodeCookie(session, s.h.encKey, s.h.hmacKey)
		if err != nil {
			log.Printf("createCookie: %s\n", err)
			goto write
		}

		if bytes.Equal(gobHash, s.h.RS.getHash(s.req)) {
			//log.Println("not re-setting identical cookie")
			goto write
		}

		var cookie http.Cookie
		cookie.Name = s.h.CookieName
		cookie.Value = encoded
		cookie.Path = "/"
		http.SetCookie(s, &cookie)
	}
write:
	s.ResponseWriter.WriteHeader(code)
}

func (h *SessionHandler) getCookieSession(req *http.Request) (map[string]interface{}, []byte) {
	cookie, err := req.Cookie(h.CookieName)
	if err != nil {
		log.Printf("getCookieSesh: '%#v' not found\n",
			h.CookieName)
		return map[string]interface{}{}, nil
	}
	session, gobHash, err := decodeCookie(cookie.Value, h.encKey, h.hmacKey)
	if err != nil {
		log.Printf("decodeCookie: %s\n", err)
		return map[string]interface{}{}, nil
	}

	return session, gobHash
}

func (h *SessionHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// get our session a little early, so that we can add our
	// authentication information to it if we get some
	session, gobHash := h.getCookieSession(req)

	h.RS.Set(req, session, gobHash)

	sessionWriter := sessionResponseWriter{rw, h, req, 0}
	h.Handler.ServeHTTP(sessionWriter, req)

	h.RS.Clear(req)
}

func NewSessionHandler(handler http.Handler, cookieName, key string, rs *RequestSessions) *SessionHandler {
	// sha1 sums are 20 bytes long.  we use the first 16 bytes as
	// the aes key.
	encHash := sha1.New()
	encHash.Write([]byte(key))
	encHash.Write([]byte("-encryption"))
	hmacHash := sha1.New()
	hmacHash.Write([]byte(key))
	hmacHash.Write([]byte("-hmac"))

	return &SessionHandler{
		Handler:    handler,
		CookieName: cookieName,
		RS:         rs,
		encKey:     encHash.Sum()[:blockSize],
		hmacKey:    hmacHash.Sum()[:blockSize],
	}
}
<commit_msg>encode: use io.ReadFull to read from rand.Reader<commit_after>// Copyright 2011 Bobby Powers. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.
package seshcookie

import (
	"http"
	"io"
	"os"
	"log"
	"time"
	"gob"
	"bytes"
	"sync"
	"sync/atomic"
	"hash"
	"crypto/sha1"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/hmac"
	"encoding/base64"
)

// we want 16 byte blocks, for AES-128
const blockSize = 16

var (
	// if you don't need multiple independent seshcookie
	// instances, you can use this RequestSessions instance to
	// manage & access your sessions.  Simply use it as the final
	// parameter in your call to seshcookie.NewSessionHandler, and
	// whenever you want to access the current session from an
	// embedded http.Handler you can simply call:
	//
	//     seshcookie.Session.Get(req)
	Session = new(RequestSessions)

	// Hash validation of the decrypted cookie failed. Most likely
	// the session was encoded with a different cookie than we're
	// using to decode it, but its possible the client (or someone
	// else) tried to modify the session.
	HashError = os.NewError("Hash validation failed")

	// The cookie is too short, so we must exit decoding early.
	LenError = os.NewError("Bad cookie length")
)

type sessionResponseWriter struct {
	http.ResponseWriter
	h           *SessionHandler
	req         *http.Request
	// int32 so we can use the sync/atomic functions on it
	wroteHeader int32
}

type SessionHandler struct {
	http.Handler
	// The name of the cookie our encoded session will be stored
	// in.
	CookieName string
	RS         *RequestSessions
	encKey     []byte
	hmacKey    []byte
}

type RequestSessions struct {
	lk sync.Mutex
	m  map[*http.Request]map[string]interface{}
	hm map[*http.Request][]byte
}

func (rs *RequestSessions) Get(req *http.Request) map[string]interface{} {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	if rs.m == nil {
		return nil
	}

	return rs.m[req]
}

func (rs *RequestSessions) getHash(req *http.Request) []byte {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	if rs.hm == nil {
		return nil
	}

	return rs.hm[req]
}

func (rs *RequestSessions) Set(req *http.Request, val map[string]interface{}, gobHash []byte) {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	if rs.m == nil {
		rs.m = map[*http.Request]map[string]interface{}{}
		rs.hm = map[*http.Request][]byte{}
	}

	rs.m[req] = val
	rs.hm[req] = gobHash
}

func (rs *RequestSessions) Clear(req *http.Request) {
	rs.lk.Lock()
	defer rs.lk.Unlock()

	rs.m[req] = nil, false
}

func encodeGob(obj interface{}) ([]byte, os.Error) {
	buf := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(obj)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func decodeGob(encoded []byte) (map[string]interface{}, os.Error) {
	buf := bytes.NewBuffer(encoded)
	dec := gob.NewDecoder(buf)
	var out map[string]interface{}
	err := dec.Decode(&out)
	if err != nil {
		return nil, err
	}
	return out, nil
}

// encode uses the given block cipher (in CTR mode) to encrypt the
// data, along with a hash, returning the iv and the ciphertext. What
// is returned looks like:
//
//   encrypted(salt + sessionData) + iv + hmac
//
func encode(block cipher.Block, hmac hash.Hash, data []byte) ([]byte, os.Error) {

	buf := bytes.NewBuffer(nil)

	salt := make([]byte, block.BlockSize())
	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
		return nil, err
	}
	buf.Write(salt)
	buf.Write(data)

	session := buf.Bytes()

	iv := make([]byte, block.BlockSize())
	if _, err := rand.Read(iv); err != nil {
		return nil, err
	}

	stream := cipher.NewCTR(block, iv)
	stream.XORKeyStream(session, session)

	buf.Write(iv)
	hmac.Write(buf.Bytes())
	buf.Write(hmac.Sum())

	return buf.Bytes(), nil
}

func encodeCookie(content interface{}, encKey, hmacKey []byte) (string, []byte, os.Error) {
	encodedGob, err := encodeGob(content)
	if err != nil {
		return "", nil, err
	}

	gobHash := sha1.New()
	gobHash.Write(encodedGob)

	aesCipher, err := aes.NewCipher(encKey)
	if err != nil {
		return "", nil, err
	}

	hmacHash := hmac.NewSHA256(hmacKey)

	sessionBytes, err := encode(aesCipher, hmacHash, encodedGob)
	if err != nil {
		return "", nil, err
	}

	return base64.StdEncoding.EncodeToString(sessionBytes), gobHash.Sum(), nil
}

// decode uses the given block cipher (in CTR mode) to decrypt the
// data, and validate the hash.  If hash validation fails, an error is
// returned.
func decode(block cipher.Block, hmac hash.Hash, ciphertext []byte) ([]byte, os.Error) {
	if len(ciphertext) < 2 * block.BlockSize() + hmac.Size() {
		return nil, LenError
	}

	receivedHmac := ciphertext[len(ciphertext) - hmac.Size():]
	ciphertext = ciphertext[:len(ciphertext) - hmac.Size()]

	hmac.Write(ciphertext)
	if !bytes.Equal(hmac.Sum(), receivedHmac) {
		return nil, HashError
	}

	// split the iv and session bytes
	iv := ciphertext[len(ciphertext) - block.BlockSize():]
	session := ciphertext[:len(ciphertext) - block.BlockSize()]

	stream := cipher.NewCTR(block, iv)
	stream.XORKeyStream(session, session)

	// skip past the iv
	session = session[block.BlockSize():]

	return session, nil
}

func decodeCookie(encoded string, encKey, hmacKey []byte) (map[string]interface{}, []byte, os.Error) {
	sessionBytes, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return nil, nil, err
	}
	aesCipher, err := aes.NewCipher(encKey)
	if err != nil {
		return nil, nil, err
	}

	hmacHash := hmac.NewSHA256(hmacKey)
	gobBytes, err := decode(aesCipher, hmacHash, sessionBytes)
	if err != nil {
		return nil, nil, err
	}

	gobHash := sha1.New()
	gobHash.Write(gobBytes)

	session, err := decodeGob(gobBytes)
	if err != nil {
		log.Printf("decodeGob: %s\n", err)
		return nil, nil, err
	}
	return session, gobHash.Sum(), nil
}

func (s sessionResponseWriter) WriteHeader(code int) {
	if atomic.AddInt32(&s.wroteHeader, 1) == 1 {
		origCookie, err := s.req.Cookie(s.h.CookieName)
		var origCookieVal string
		if err != nil {
			origCookieVal = ""
		} else {
			origCookieVal = origCookie.Value
		}

		session := s.h.RS.Get(s.req)
		if len(session) == 0 {
			// if we have an empty session, but the
			// request didn't start out that way, we
			// assume the user wants us to clear the
			// session
			if origCookieVal != "" {
				log.Println("clearing cookie")
				var cookie http.Cookie
				cookie.Name = s.h.CookieName
				cookie.Value = ""
				cookie.Path = "/"
				// a cookie is expired by setting it
				// with an expiration time in the past
				cookie.Expires = *time.SecondsToUTC(0)
				http.SetCookie(s, &cookie)
			} else {
				log.Println("not setting empty cookie")
			}
			goto write
		}
		encoded, gobHash, err := encodeCookie(session, s.h.encKey, s.h.hmacKey)
		if err != nil {
			log.Printf("createCookie: %s\n", err)
			goto write
		}

		if bytes.Equal(gobHash, s.h.RS.getHash(s.req)) {
			//log.Println("not re-setting identical cookie")
			goto write
		}

		var cookie http.Cookie
		cookie.Name = s.h.CookieName
		cookie.Value = encoded
		cookie.Path = "/"
		http.SetCookie(s, &cookie)
	}
write:
	s.ResponseWriter.WriteHeader(code)
}

func (h *SessionHandler) getCookieSession(req *http.Request) (map[string]interface{}, []byte) {
	cookie, err := req.Cookie(h.CookieName)
	if err != nil {
		log.Printf("getCookieSesh: '%#v' not found\n",
			h.CookieName)
		return map[string]interface{}{}, nil
	}
	session, gobHash, err := decodeCookie(cookie.Value, h.encKey, h.hmacKey)
	if err != nil {
		log.Printf("decodeCookie: %s\n", err)
		return map[string]interface{}{}, nil
	}

	return session, gobHash
}

func (h *SessionHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// get our session a little early, so that we can add our
	// authentication information to it if we get some
	session, gobHash := h.getCookieSession(req)

	h.RS.Set(req, session, gobHash)

	sessionWriter := sessionResponseWriter{rw, h, req, 0}
	h.Handler.ServeHTTP(sessionWriter, req)

	h.RS.Clear(req)
}

func NewSessionHandler(handler http.Handler, cookieName, key string, rs *RequestSessions) *SessionHandler {
	// sha1 sums are 20 bytes long.  we use the first 16 bytes as
	// the aes key.
	encHash := sha1.New()
	encHash.Write([]byte(key))
	encHash.Write([]byte("-encryption"))
	hmacHash := sha1.New()
	hmacHash.Write([]byte(key))
	hmacHash.Write([]byte("-hmac"))

	return &SessionHandler{
		Handler:    handler,
		CookieName: cookieName,
		RS:         rs,
		encKey:     encHash.Sum()[:blockSize],
		hmacKey:    hmacHash.Sum()[:blockSize],
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package moselserver

import (
	"net/http"
	"github.com/WE-Development/mosel/commons"
	"log"
	"github.com/WE-Development/mosel/config"
	"regexp"
)

// Wrap a http.HandleFunc such that it's authenticated before execution.
// It's build on basic auth or use with sessions as provided by moselserver.sessionCache.
func (server *MoselServer) secure(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if server.Config.Sessions.Enabled {
			authSession(server, fn, w, r)
		} else {
			authDirect(server, fn, w, r)
		}
	}
}

// Authenticate via sessions
func authSession(server *MoselServer, fn http.HandlerFunc, w http.ResponseWriter, r *http.Request) {
	key := r.FormValue("key")
	if key == "" || !server.Context.Sessions.ValidateSession(key) {
		commons.HttpUnauthorized(w)
		return
	}
	fn(w, r)
}

// Authenticate directly via basic auth
func authDirect(server *MoselServer, fn http.HandlerFunc, w http.ResponseWriter, r *http.Request) {
	user, passwd, _ := r.BasicAuth()

	if server.Context.Auth == nil {
		log.Println("No autheticator configured! Denying all requests")
		commons.HttpUnauthorized(w)
		return
	}

	if !server.Context.Auth.Authenticate(user, passwd) {
		commons.HttpUnauthorized(w)
		return
	}

	if ok, err := validateAccessRights(r.URL.Path, server.Config.Users[user], server.Config.Groups); !ok {
		if err != nil {
			log.Println(err)
		}

		commons.HttpUnauthorized(w)
		return
	}

	fn(w, r)
}

func validateAccessRights(path string, userConfig *moselconfig.UserConfig, groupConfigs map[string]*moselconfig.GroupConfig) (bool, error) {
	allow := false;

	rights := make([]moselconfig.AccessRights, 0)

	for groupName, groupConf := range groupConfigs {
		for _, userGroup := range userConfig.Groups {
			if userGroup != groupName {
				continue
			}
			rights = append(rights, groupConf.AccessRights)
		}
	}
	rights = append(rights, userConfig.AccessRights)

	for _, rightConf := range rights {
		var err error
		var match bool
		for _, denyRegex := range rightConf.Deny {
			match, err = regexp.MatchString(denyRegex, path)

			if err != nil {
				return false, err
			}

			allow = allow && !match
		}
		for _, allowRegex := range rightConf.Allow {
			match, err = regexp.MatchString(allowRegex, path)

			if err != nil {
				return false, err
			}

			allow = allow || match
		}

	}

	return allow, nil
}<commit_msg>fix nullptr<commit_after>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package moselserver

import (
	"net/http"
	"github.com/WE-Development/mosel/commons"
	"log"
	"github.com/WE-Development/mosel/config"
	"regexp"
)

// Wrap a http.HandleFunc such that it's authenticated before execution.
// It's build on basic auth or use with sessions as provided by moselserver.sessionCache.
func (server *MoselServer) secure(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if server.Config.Sessions.Enabled {
			authSession(server, fn, w, r)
		} else {
			authDirect(server, fn, w, r)
		}
	}
}

// Authenticate via sessions
func authSession(server *MoselServer, fn http.HandlerFunc, w http.ResponseWriter, r *http.Request) {
	key := r.FormValue("key")
	if key == "" || !server.Context.Sessions.ValidateSession(key) {
		commons.HttpUnauthorized(w)
		return
	}
	fn(w, r)
}

// Authenticate directly via basic auth
func authDirect(server *MoselServer, fn http.HandlerFunc, w http.ResponseWriter, r *http.Request) {
	user, passwd, _ := r.BasicAuth()

	if server.Context.Auth == nil {
		log.Println("No autheticator configured! Denying all requests")
		commons.HttpUnauthorized(w)
		return
	}

	if !server.Context.Auth.Authenticate(user, passwd) {
		commons.HttpUnauthorized(w)
		return
	}

	// if auth true is enabled, it's possible that a positiv authentication occurs without a user configuration
	if !server.Config.AuthTrue.Enabled {
		if ok, err := validateAccessRights(r.URL.Path, server.Config.Users[user], server.Config.Groups); !ok {
			if err != nil {
				log.Println(err)
			}

			commons.HttpUnauthorized(w)
			return
		}
	}

	fn(w, r)
}

func validateAccessRights(path string, userConfig *moselconfig.UserConfig, groupConfigs map[string]*moselconfig.GroupConfig) (bool, error) {
	allow := false;

	rights := make([]moselconfig.AccessRights, 0)

	for groupName, groupConf := range groupConfigs {
		for _, userGroup := range userConfig.Groups {
			if userGroup != groupName {
				continue
			}
			rights = append(rights, groupConf.AccessRights)
		}
	}
	rights = append(rights, userConfig.AccessRights)

	for _, rightConf := range rights {
		var err error
		var match bool
		for _, denyRegex := range rightConf.Deny {
			match, err = regexp.MatchString(denyRegex, path)

			if err != nil {
				return false, err
			}

			allow = allow && !match
		}
		for _, allowRegex := range rightConf.Allow {
			match, err = regexp.MatchString(allowRegex, path)

			if err != nil {
				return false, err
			}

			allow = allow || match
		}

	}

	return allow, nil
}<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

// channel holds logic for finding and communicating with members of a
// channel.
//
// Usage:
//  // Construct a new channel.
//  c := newChannel(ctx, mounttable, proxy, "path/to/channel/name")
//
//  // Join the channel.
//  err := c.join()
//
//  // Get all members in the channel.
//  members, err := c.getMembers()
//
//  // Send a message to a member.
//  c.sendMessageTo(member, "message")
//
//  // Send a message to all members in the channel.
//  c.broadcastMessage("message")
//
//  // Leave the channel.
//  c.leave()

import (
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"sort"
	"time"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/security/access"
	mt "v.io/v23/services/mounttable"
	"v.io/x/chat/vdl"
	"v.io/x/ref/lib/xrpc"
	_ "v.io/x/ref/runtime/factories/roaming"
)

// message is a message that will be displayed in the UI.
type message struct {
	SenderName string
	Text       string
	Timestamp  time.Time
}

// chatServerMethods implements the chat server VDL interface.
type chatServerMethods struct {
	// Incoming messages get sent to messages channel.
	messages chan<- message
}

var _ vdl.ChatServerMethods = (*chatServerMethods)(nil)

func newChatServerMethods(messages chan<- message) *chatServerMethods {
	return &chatServerMethods{
		messages: messages,
	}
}

// SendMessage is called by clients to send a message to the server.
func (cs *chatServerMethods) SendMessage(ctx *context.T, call rpc.ServerCall, IncomingMessage string) error {
	remoteb, _ := security.RemoteBlessingNames(ctx, call.Security())
	cs.messages <- message{
		SenderName: firstShortName(remoteb),
		Text:       IncomingMessage,
		Timestamp:  time.Now(),
	}
	return nil
}

// member is a member of the channel.
type member struct {
	// Blessings is the remote blessings of the member.  There could
	// potentially be multiple.
	Blessings []string
	// Name is the name we will display for this member.
	Name string
	// Path is the path in the mounttable where the member is mounted.
	Path string
}

// members are sortable by Name.
type byName []*member

func (b byName) Len() int           { return len(b) }
func (b byName) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b byName) Less(i, j int) bool { return b[i].Name < b[j].Name }

// channel interface.
type channel struct {
	// Vanadium context.
	ctx *context.T
	// The location where we mount ourselves and look for other users.
	path string
	// The implementation of the chat server.
	chatServerMethods *chatServerMethods
	// The chat server.
	server rpc.XServer
	// Channel that emits incoming messages.
	messages chan message
	// Cached list of channel members.
	members []*member
}

func newChannel(ctx *context.T, mounttable, proxy, path string) (*channel, error) {
	// Set the namespace root to the mounttable passed on the command line.
	newCtx, _, err := v23.WithNewNamespace(ctx, mounttable)
	if err != nil {
		return nil, err
	}

	// Set the proxy that will be used to listen.
	listenSpec := v23.GetListenSpec(ctx)
	listenSpec.Proxy = proxy

	messages := make(chan message)

	return &channel{
		chatServerMethods: newChatServerMethods(messages),
		messages:          messages,
		path:              path,
		ctx:               newCtx,
		server:            nil,
	}, nil
}

// UserName returns a short, human-friendly representation of the chat client.
func (cr *channel) UserName() string {
	// TODO(ashankar): It is wrong to assume that
	// v23.GetPrincipal(ctx).BlessingStore().Default() returns a valid
	// "sender". Think about the "who-am-I" API and use that here instead.
	userName := fmt.Sprint(v23.GetPrincipal(cr.ctx).BlessingStore().Default())
	if sn := shortName(userName); sn != "" {
		userName = sn
	}
	return userName
}

// getLockedName picks a random name inside the channel's mounttable path and
// tries to "lock" it by settings restrictive permissions on the name.  It
// tries repeatedly until it finds an unused name that can be locked, and
// returns the locked name.
func (cr *channel) getLockedName() (string, error) {
	myPatterns := security.DefaultBlessingPatterns(v23.GetPrincipal(cr.ctx))

	// myACL is an ACL that only allows my blessing.
	myACL := access.AccessList{
		In: myPatterns,
	}
	// openACL is an ACL that allows anybody.
	openACL := access.AccessList{
		In: []security.BlessingPattern{security.AllPrincipals},
	}

	permissions := access.Permissions{
		// Give everybody the ability to read and resolve the name.
		string(mt.Resolve): openACL,
		string(mt.Read):    openACL,
		// All other permissions are only for us.
		string(mt.Admin):  myACL,
		string(mt.Create): myACL,
		string(mt.Mount):  myACL,
	}

	// Repeatedly try to SetPermissions under random names until we find a free
	// one.

	// Collisions should be rare.  25 times should be enough to find a free
	// one
	maxTries := 25
	for i := 0; i < maxTries; i++ {
		// Pick a random suffix, the hash of our default blessing and the time.
		now := time.Now().UnixNano()
		hash := sha256.Sum256([]byte(fmt.Sprintf("%s-%d", cr.UserName(), now)))
		suffix := base64.URLEncoding.EncodeToString(hash[:])

		name := naming.Join(cr.path, suffix)

		ns := v23.GetNamespace(cr.ctx)

		if err := ns.SetPermissions(cr.ctx, name, permissions, ""); err != nil {
			// Try again with a different name.
			continue
		}

		// SetPermissions succeeded!  We now own the name.
		return name, nil
	}
	return "", fmt.Errorf("Error getting a locked name.  Tried %v times but did not succeed.", maxTries)
}

// join starts a chat server and mounts it in the channel path.
func (cr *channel) join() error {
	// Get a locked name in the mounttable that we can mount our server on.
	name, err := cr.getLockedName()
	if err != nil {
		return err
	}
	// Serve the chat server on the locked name.
	serverChat := vdl.ChatServer(cr.chatServerMethods)

	// Create a new server.
	cr.server, err = xrpc.NewServer(cr.ctx, name, serverChat, security.AllowEveryone())
	return err
}

// leave stops the chat server and removes our mounted name from the
// mounttable.
func (cr *channel) leave() error {
	// Stop serving.
	cr.server.Stop()

	// Get the names we are mounted at.  Should only be one.
	names := cr.server.Status().Mounts.Names()
	// Delete the name and all sub-names in the hierarchy.
	ns := v23.GetNamespace(cr.ctx)
	for _, name := range names {
		if err := ns.Delete(cr.ctx, name, true); err != nil {
			return err
		}
	}

	cr.server = nil

	return nil
}

// newMember creates a new member object.
func (cr *channel) newMember(blessings []string, path string) *member {
	name := "unknown"
	if len(blessings) > 0 {
		// Arbitrarily choose the first blessing as the display name.
		name = shortName(blessings[0])
	}
	return &member{
		Name:      name,
		Blessings: blessings,
		Path:      path,
	}
}

// getMembers gets a list of members in the channel.
func (cr *channel) getMembers() ([]*member, error) {
	ctx, cancel := context.WithTimeout(cr.ctx, 5*time.Second)
	defer cancel()

	// Glob on the channel path for mounted members.
	globPath := cr.path + "/*"
	globChan, err := v23.GetNamespace(ctx).Glob(ctx, globPath)
	if err != nil {
		return nil, err
	}

	members := []*member{}

	for reply := range globChan {
		switch v := reply.(type) {
		case *naming.GlobReplyEntry:
			blessings := blessingNamesFromMountEntry(&v.Value)
			if len(blessings) == 0 {
				// No servers mounted at that name, likely only a
				// lonely ACL.  Safe to ignore.
				// TODO(nlacasse): Should there be a time-limit
				// on ACLs in the namespace?  Seems like we'll
				// have an ACL graveyard before too long.
				continue
			}
			member := cr.newMember(blessings, v.Value.Name)
			members = append(members, member)
		}
	}

	sort.Sort(byName(members))

	cr.members = members
	return members, nil
}

// broadcastMessage sends a message to all members in the channel.
func (cr *channel) broadcastMessage(messageText string) error {
	for _, member := range cr.members {
		// TODO(nlacasse): Sending messages async means they might get sent out of
		// order. Consider either sending them sync or maintain a queue.
		go cr.sendMessageTo(member, messageText)
	}
	return nil
}

// sendMessageTo sends a message to a particular member.  It ensures that the
// receiving server has the same blessings that the member does.
func (cr *channel) sendMessageTo(member *member, messageText string) {
	ctx, cancel := context.WithTimeout(cr.ctx, 5*time.Second)
	defer cancel()

	s := vdl.ChatClient(member.Path)

	// The AllowedServersPolicy options require that the server matches the
	// blessings we got when we globbed it.
	opts := make([]rpc.CallOpt, len(member.Blessings))
	for i, blessing := range member.Blessings {
		opts[i] = options.AllowedServersPolicy{security.BlessingPattern(blessing)}
	}

	if err := s.SendMessage(ctx, messageText, opts...); err != nil {
		return // member has disconnected.
	}
}

func blessingNamesFromMountEntry(me *naming.MountEntry) []string {
	names := me.Names()
	if len(names) == 0 {
		return nil
	}
	// Using the first valid mount entry for now.
	// TODO(nlacasse): How should we deal with multiple members mounted on
	// a single mountpoint?
	for _, name := range names {
		addr, _ := naming.SplitAddressName(name)
		ep, err := v23.NewEndpoint(addr)
		if err != nil {
			// TODO(nlacasse): Log this or bubble up?
			continue
		}
		return ep.BlessingNames()
	}
	return nil
}
<commit_msg>v23: Promote the XWithNew*Server APIs out of experimental\ .<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

// channel holds logic for finding and communicating with members of a
// channel.
//
// Usage:
//  // Construct a new channel.
//  c := newChannel(ctx, mounttable, proxy, "path/to/channel/name")
//
//  // Join the channel.
//  err := c.join()
//
//  // Get all members in the channel.
//  members, err := c.getMembers()
//
//  // Send a message to a member.
//  c.sendMessageTo(member, "message")
//
//  // Send a message to all members in the channel.
//  c.broadcastMessage("message")
//
//  // Leave the channel.
//  c.leave()

import (
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"sort"
	"time"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/security/access"
	mt "v.io/v23/services/mounttable"
	"v.io/x/chat/vdl"
	"v.io/x/ref/lib/xrpc"
	_ "v.io/x/ref/runtime/factories/roaming"
)

// message is a message that will be displayed in the UI.
type message struct {
	SenderName string
	Text       string
	Timestamp  time.Time
}

// chatServerMethods implements the chat server VDL interface.
type chatServerMethods struct {
	// Incoming messages get sent to messages channel.
	messages chan<- message
}

var _ vdl.ChatServerMethods = (*chatServerMethods)(nil)

func newChatServerMethods(messages chan<- message) *chatServerMethods {
	return &chatServerMethods{
		messages: messages,
	}
}

// SendMessage is called by clients to send a message to the server.
func (cs *chatServerMethods) SendMessage(ctx *context.T, call rpc.ServerCall, IncomingMessage string) error {
	remoteb, _ := security.RemoteBlessingNames(ctx, call.Security())
	cs.messages <- message{
		SenderName: firstShortName(remoteb),
		Text:       IncomingMessage,
		Timestamp:  time.Now(),
	}
	return nil
}

// member is a member of the channel.
type member struct {
	// Blessings is the remote blessings of the member.  There could
	// potentially be multiple.
	Blessings []string
	// Name is the name we will display for this member.
	Name string
	// Path is the path in the mounttable where the member is mounted.
	Path string
}

// members are sortable by Name.
type byName []*member

func (b byName) Len() int           { return len(b) }
func (b byName) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b byName) Less(i, j int) bool { return b[i].Name < b[j].Name }

// channel interface.
type channel struct {
	// Vanadium context.
	ctx *context.T
	// The location where we mount ourselves and look for other users.
	path string
	// The implementation of the chat server.
	chatServerMethods *chatServerMethods
	// The chat server.
	server rpc.Server
	// Channel that emits incoming messages.
	messages chan message
	// Cached list of channel members.
	members []*member
}

func newChannel(ctx *context.T, mounttable, proxy, path string) (*channel, error) {
	// Set the namespace root to the mounttable passed on the command line.
	newCtx, _, err := v23.WithNewNamespace(ctx, mounttable)
	if err != nil {
		return nil, err
	}

	// Set the proxy that will be used to listen.
	listenSpec := v23.GetListenSpec(ctx)
	listenSpec.Proxy = proxy

	messages := make(chan message)

	return &channel{
		chatServerMethods: newChatServerMethods(messages),
		messages:          messages,
		path:              path,
		ctx:               newCtx,
		server:            nil,
	}, nil
}

// UserName returns a short, human-friendly representation of the chat client.
func (cr *channel) UserName() string {
	// TODO(ashankar): It is wrong to assume that
	// v23.GetPrincipal(ctx).BlessingStore().Default() returns a valid
	// "sender". Think about the "who-am-I" API and use that here instead.
	userName := fmt.Sprint(v23.GetPrincipal(cr.ctx).BlessingStore().Default())
	if sn := shortName(userName); sn != "" {
		userName = sn
	}
	return userName
}

// getLockedName picks a random name inside the channel's mounttable path and
// tries to "lock" it by settings restrictive permissions on the name.  It
// tries repeatedly until it finds an unused name that can be locked, and
// returns the locked name.
func (cr *channel) getLockedName() (string, error) {
	myPatterns := security.DefaultBlessingPatterns(v23.GetPrincipal(cr.ctx))

	// myACL is an ACL that only allows my blessing.
	myACL := access.AccessList{
		In: myPatterns,
	}
	// openACL is an ACL that allows anybody.
	openACL := access.AccessList{
		In: []security.BlessingPattern{security.AllPrincipals},
	}

	permissions := access.Permissions{
		// Give everybody the ability to read and resolve the name.
		string(mt.Resolve): openACL,
		string(mt.Read):    openACL,
		// All other permissions are only for us.
		string(mt.Admin):  myACL,
		string(mt.Create): myACL,
		string(mt.Mount):  myACL,
	}

	// Repeatedly try to SetPermissions under random names until we find a free
	// one.

	// Collisions should be rare.  25 times should be enough to find a free
	// one
	maxTries := 25
	for i := 0; i < maxTries; i++ {
		// Pick a random suffix, the hash of our default blessing and the time.
		now := time.Now().UnixNano()
		hash := sha256.Sum256([]byte(fmt.Sprintf("%s-%d", cr.UserName(), now)))
		suffix := base64.URLEncoding.EncodeToString(hash[:])

		name := naming.Join(cr.path, suffix)

		ns := v23.GetNamespace(cr.ctx)

		if err := ns.SetPermissions(cr.ctx, name, permissions, ""); err != nil {
			// Try again with a different name.
			continue
		}

		// SetPermissions succeeded!  We now own the name.
		return name, nil
	}
	return "", fmt.Errorf("Error getting a locked name.  Tried %v times but did not succeed.", maxTries)
}

// join starts a chat server and mounts it in the channel path.
func (cr *channel) join() error {
	// Get a locked name in the mounttable that we can mount our server on.
	name, err := cr.getLockedName()
	if err != nil {
		return err
	}
	// Serve the chat server on the locked name.
	serverChat := vdl.ChatServer(cr.chatServerMethods)

	// Create a new server.
	cr.server, err = xrpc.NewServer(cr.ctx, name, serverChat, security.AllowEveryone())
	return err
}

// leave stops the chat server and removes our mounted name from the
// mounttable.
func (cr *channel) leave() error {
	// Stop serving.
	cr.server.Stop()

	// Get the names we are mounted at.  Should only be one.
	names := cr.server.Status().Mounts.Names()
	// Delete the name and all sub-names in the hierarchy.
	ns := v23.GetNamespace(cr.ctx)
	for _, name := range names {
		if err := ns.Delete(cr.ctx, name, true); err != nil {
			return err
		}
	}

	cr.server = nil

	return nil
}

// newMember creates a new member object.
func (cr *channel) newMember(blessings []string, path string) *member {
	name := "unknown"
	if len(blessings) > 0 {
		// Arbitrarily choose the first blessing as the display name.
		name = shortName(blessings[0])
	}
	return &member{
		Name:      name,
		Blessings: blessings,
		Path:      path,
	}
}

// getMembers gets a list of members in the channel.
func (cr *channel) getMembers() ([]*member, error) {
	ctx, cancel := context.WithTimeout(cr.ctx, 5*time.Second)
	defer cancel()

	// Glob on the channel path for mounted members.
	globPath := cr.path + "/*"
	globChan, err := v23.GetNamespace(ctx).Glob(ctx, globPath)
	if err != nil {
		return nil, err
	}

	members := []*member{}

	for reply := range globChan {
		switch v := reply.(type) {
		case *naming.GlobReplyEntry:
			blessings := blessingNamesFromMountEntry(&v.Value)
			if len(blessings) == 0 {
				// No servers mounted at that name, likely only a
				// lonely ACL.  Safe to ignore.
				// TODO(nlacasse): Should there be a time-limit
				// on ACLs in the namespace?  Seems like we'll
				// have an ACL graveyard before too long.
				continue
			}
			member := cr.newMember(blessings, v.Value.Name)
			members = append(members, member)
		}
	}

	sort.Sort(byName(members))

	cr.members = members
	return members, nil
}

// broadcastMessage sends a message to all members in the channel.
func (cr *channel) broadcastMessage(messageText string) error {
	for _, member := range cr.members {
		// TODO(nlacasse): Sending messages async means they might get sent out of
		// order. Consider either sending them sync or maintain a queue.
		go cr.sendMessageTo(member, messageText)
	}
	return nil
}

// sendMessageTo sends a message to a particular member.  It ensures that the
// receiving server has the same blessings that the member does.
func (cr *channel) sendMessageTo(member *member, messageText string) {
	ctx, cancel := context.WithTimeout(cr.ctx, 5*time.Second)
	defer cancel()

	s := vdl.ChatClient(member.Path)

	// The AllowedServersPolicy options require that the server matches the
	// blessings we got when we globbed it.
	opts := make([]rpc.CallOpt, len(member.Blessings))
	for i, blessing := range member.Blessings {
		opts[i] = options.AllowedServersPolicy{security.BlessingPattern(blessing)}
	}

	if err := s.SendMessage(ctx, messageText, opts...); err != nil {
		return // member has disconnected.
	}
}

func blessingNamesFromMountEntry(me *naming.MountEntry) []string {
	names := me.Names()
	if len(names) == 0 {
		return nil
	}
	// Using the first valid mount entry for now.
	// TODO(nlacasse): How should we deal with multiple members mounted on
	// a single mountpoint?
	for _, name := range names {
		addr, _ := naming.SplitAddressName(name)
		ep, err := v23.NewEndpoint(addr)
		if err != nil {
			// TODO(nlacasse): Log this or bubble up?
			continue
		}
		return ep.BlessingNames()
	}
	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"

	"github.com/golang/protobuf/ptypes"
	"github.com/nsheridan/cashier/lib"
	"github.com/nsheridan/cashier/proto"
	"github.com/pkg/errors"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"golang.org/x/net/context"
)

// SavePublicFiles installs the public part of the cert and key.
func SavePublicFiles(prefix string, cert *ssh.Certificate, pub ssh.PublicKey) error {
	if prefix == "" {
		return nil
	}
	pubTxt := ssh.MarshalAuthorizedKey(pub)
	certPubTxt := []byte(cert.Type() + " " + base64.StdEncoding.EncodeToString(cert.Marshal()))

	if err := ioutil.WriteFile(prefix+".pub", pubTxt, 0644); err != nil {
		return err
	}
	err := ioutil.WriteFile(prefix+"-cert.pub", certPubTxt, 0644)

	return err
}

// InstallCert adds the private key and signed certificate to the ssh agent.
func InstallCert(a agent.Agent, cert *ssh.Certificate, key Key) error {
	t := time.Unix(int64(cert.ValidBefore), 0)
	lifetime := t.Sub(time.Now()).Seconds()
	comment := fmt.Sprintf("%s [Expires %s]", cert.KeyId, t)
	pubcert := agent.AddedKey{
		PrivateKey:   key,
		Certificate:  cert,
		Comment:      comment,
		LifetimeSecs: uint32(lifetime),
	}
	if err := a.Add(pubcert); err != nil {
		return errors.Wrap(err, "unable to add cert to ssh agent")
	}
	privkey := agent.AddedKey{
		PrivateKey:   key,
		Comment:      comment,
		LifetimeSecs: uint32(lifetime),
	}
	if err := a.Add(privkey); err != nil {
		return errors.Wrap(err, "unable to add private key to ssh agent")
	}
	return nil
}

// send the signing request to the CA.
func send(s []byte, token, ca string, ValidateTLSCertificate bool) (*lib.SignResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: !ValidateTLSCertificate},
	}
	client := &http.Client{Transport: transport}
	u, err := url.Parse(ca)
	if err != nil {
		return nil, errors.Wrap(err, "unable to parse CA url")
	}
	u.Path = path.Join(u.Path, "/sign")
	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(s))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Bad response from server: %s", resp.Status)
	}
	defer resp.Body.Close()
	c := &lib.SignResponse{}
	if err := json.NewDecoder(resp.Body).Decode(c); err != nil {
		return nil, errors.Wrap(err, "unable to decode server response")
	}
	return c, nil
}

// Sign sends the public key to the CA to be signed.
func Sign(pub ssh.PublicKey, token string, conf *Config) (*ssh.Certificate, error) {
	validity, err := time.ParseDuration(conf.Validity)
	if err != nil {
		return nil, err
	}
	s, err := json.Marshal(&lib.SignRequest{
		Key:        string(lib.GetPublicKey(pub)),
		ValidUntil: time.Now().Add(validity),
	})
	if err != nil {
		return nil, errors.Wrap(err, "unable to create sign request")
	}
	resp, err := send(s, token, conf.CA, conf.ValidateTLSCertificate)
	if err != nil {
		return nil, errors.Wrap(err, "error sending request to CA")
	}
	if resp.Status != "ok" {
		return nil, fmt.Errorf("bad response from CA: %s", resp.Response)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey([]byte(resp.Response))
	if err != nil {
		return nil, errors.Wrap(err, "unable to parse response")
	}
	cert, ok := k.(*ssh.Certificate)
	if !ok {
		return nil, fmt.Errorf("did not receive a valid certificate from server")
	}
	return cert, nil
}

// RPCSign sends the public key to the CA to be signed.
func RPCSign(pub ssh.PublicKey, token string, conf *Config) (*ssh.Certificate, error) {
	var opts []grpc.DialOption
	var srv string
	if strings.HasPrefix(conf.CA, "https://") {
		srv = strings.TrimPrefix(conf.CA, "https://")
	} else {
		srv = strings.TrimPrefix(conf.CA, "http://")
		opts = append(opts, grpc.WithInsecure())
	}
	conn, err := grpc.Dial(srv, opts...)
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	stub := proto.NewSignerClient(conn)
	lifetime, err := time.ParseDuration(conf.Validity)
	if err != nil {
		return nil, err
	}
	deadline := time.Now().Add(lifetime)
	ts, err := ptypes.TimestampProto(deadline)
	if err != nil {
		return nil, err
	}
	req := &proto.SignRequest{
		Key:        lib.GetPublicKey(pub),
		ValidUntil: ts,
	}
	md := metadata.New(map[string]string{
		"security": "authorization",
		"payload":  token,
	})
	r, err := stub.Sign(metadata.NewContext(context.TODO(), md), req)
	if err != nil {
		return nil, err
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(r.Cert)
	if err != nil {
		return nil, err
	}
	cert, ok := k.(*ssh.Certificate)
	if !ok {
		return nil, errors.New("did not receive a valid certificate from server")
	}
	return cert, nil
}
<commit_msg>Ensure resp.Body is closed<commit_after>package client

import (
	"bytes"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"

	"github.com/golang/protobuf/ptypes"
	"github.com/nsheridan/cashier/lib"
	"github.com/nsheridan/cashier/proto"
	"github.com/pkg/errors"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"golang.org/x/net/context"
)

// SavePublicFiles installs the public part of the cert and key.
func SavePublicFiles(prefix string, cert *ssh.Certificate, pub ssh.PublicKey) error {
	if prefix == "" {
		return nil
	}
	pubTxt := ssh.MarshalAuthorizedKey(pub)
	certPubTxt := []byte(cert.Type() + " " + base64.StdEncoding.EncodeToString(cert.Marshal()))

	if err := ioutil.WriteFile(prefix+".pub", pubTxt, 0644); err != nil {
		return err
	}
	err := ioutil.WriteFile(prefix+"-cert.pub", certPubTxt, 0644)

	return err
}

// InstallCert adds the private key and signed certificate to the ssh agent.
func InstallCert(a agent.Agent, cert *ssh.Certificate, key Key) error {
	t := time.Unix(int64(cert.ValidBefore), 0)
	lifetime := t.Sub(time.Now()).Seconds()
	comment := fmt.Sprintf("%s [Expires %s]", cert.KeyId, t)
	pubcert := agent.AddedKey{
		PrivateKey:   key,
		Certificate:  cert,
		Comment:      comment,
		LifetimeSecs: uint32(lifetime),
	}
	if err := a.Add(pubcert); err != nil {
		return errors.Wrap(err, "unable to add cert to ssh agent")
	}
	privkey := agent.AddedKey{
		PrivateKey:   key,
		Comment:      comment,
		LifetimeSecs: uint32(lifetime),
	}
	if err := a.Add(privkey); err != nil {
		return errors.Wrap(err, "unable to add private key to ssh agent")
	}
	return nil
}

// send the signing request to the CA.
func send(s []byte, token, ca string, ValidateTLSCertificate bool) (*lib.SignResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: !ValidateTLSCertificate},
	}
	client := &http.Client{Transport: transport}
	u, err := url.Parse(ca)
	if err != nil {
		return nil, errors.Wrap(err, "unable to parse CA url")
	}
	u.Path = path.Join(u.Path, "/sign")
	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(s))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Bad response from server: %s", resp.Status)
	}
	c := &lib.SignResponse{}
	if err := json.NewDecoder(resp.Body).Decode(c); err != nil {
		return nil, errors.Wrap(err, "unable to decode server response")
	}
	return c, nil
}

// Sign sends the public key to the CA to be signed.
func Sign(pub ssh.PublicKey, token string, conf *Config) (*ssh.Certificate, error) {
	validity, err := time.ParseDuration(conf.Validity)
	if err != nil {
		return nil, err
	}
	s, err := json.Marshal(&lib.SignRequest{
		Key:        string(lib.GetPublicKey(pub)),
		ValidUntil: time.Now().Add(validity),
	})
	if err != nil {
		return nil, errors.Wrap(err, "unable to create sign request")
	}
	resp, err := send(s, token, conf.CA, conf.ValidateTLSCertificate)
	if err != nil {
		return nil, errors.Wrap(err, "error sending request to CA")
	}
	if resp.Status != "ok" {
		return nil, fmt.Errorf("bad response from CA: %s", resp.Response)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey([]byte(resp.Response))
	if err != nil {
		return nil, errors.Wrap(err, "unable to parse response")
	}
	cert, ok := k.(*ssh.Certificate)
	if !ok {
		return nil, fmt.Errorf("did not receive a valid certificate from server")
	}
	return cert, nil
}

// RPCSign sends the public key to the CA to be signed.
func RPCSign(pub ssh.PublicKey, token string, conf *Config) (*ssh.Certificate, error) {
	var opts []grpc.DialOption
	var srv string
	if strings.HasPrefix(conf.CA, "https://") {
		srv = strings.TrimPrefix(conf.CA, "https://")
	} else {
		srv = strings.TrimPrefix(conf.CA, "http://")
		opts = append(opts, grpc.WithInsecure())
	}
	conn, err := grpc.Dial(srv, opts...)
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	stub := proto.NewSignerClient(conn)
	lifetime, err := time.ParseDuration(conf.Validity)
	if err != nil {
		return nil, err
	}
	deadline := time.Now().Add(lifetime)
	ts, err := ptypes.TimestampProto(deadline)
	if err != nil {
		return nil, err
	}
	req := &proto.SignRequest{
		Key:        lib.GetPublicKey(pub),
		ValidUntil: ts,
	}
	md := metadata.New(map[string]string{
		"security": "authorization",
		"payload":  token,
	})
	r, err := stub.Sign(metadata.NewContext(context.TODO(), md), req)
	if err != nil {
		return nil, err
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(r.Cert)
	if err != nil {
		return nil, err
	}
	cert, ok := k.(*ssh.Certificate)
	if !ok {
		return nil, errors.New("did not receive a valid certificate from server")
	}
	return cert, nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"github.com/opennetworkinglab/onos-warden/warden"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
	"io"
	"fmt"
)

func main() {
	conn, err := grpc.Dial("127.0.0.1:1234", grpc.WithInsecure())
	if err != nil {
		grpclog.Fatalf("fail to dial: %v", err)
	}
	defer conn.Close()
	client := warden.NewClusterClientServiceClient(conn)

	stream, err := client.ServerClusters(context.Background())
	if err != nil {
		grpclog.Fatalf("%v.ServerClusters(_) = _, %v", client, err)
	}

	count := 0
	waitc := make(chan struct{})
	go func() {
		for {
			in, err := stream.Recv()
			if err == io.EOF {
				// read done.
				close(waitc)
				return
			}
			if err != nil {
				grpclog.Fatalf("Failed to receive: %v", err)
			}
			grpclog.Println("Got message:", in)
			if in.State == warden.ClusterAdvertisement_AVAILABLE {
				count++
				stream.Send(&warden.ClusterRequest{
					RequestId: fmt.Sprintf("res-%d", count),
					Type: warden.ClusterRequest_RESERVE,
					ClusterId: in.ClusterId,
					ClusterType: in.ClusterType,
					Duration: 0,
					Spec: &warden.ClusterRequest_Spec{
						ControllerNodes: uint32(3),
						UserName: "client",
						UserKey: "myawesomepubkey-rsa",
					},
				})
				stream.CloseSend()
				close(waitc)
				break
			}
		}
	}()

	//stream.CloseSend()
	<-waitc
}
<commit_msg>Making the client smarter<commit_after>package main

import (
	"context"
	"github.com/opennetworkinglab/onos-warden/warden"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
	"io"
	"fmt"
	"os"
	"os/user"
	"flag"
	"math/rand"
	"time"
)

// Request the first available cell, return when cell reservation is completed
func main() {
	rand.Seed(time.Now().Unix())
	currUser, err := user.Current()
	if err != nil {
		panic(err)
	}
	username := flag.String("user", currUser.Username, "username for reservation; defaults to $USER")
	key := flag.String("key", "", "public key for SSH")
	duration := flag.Int64("duration", -1, "duration of reservation in minutes; -1 is unlimited")
	nodes := flag.Uint64("nodes", 3, "number of nodes in cell; defaults to 3")

	conn, err := grpc.Dial("127.0.0.1:1234", grpc.WithInsecure())
	if err != nil {
		grpclog.Fatalf("fail to dial: %v", err)
	}
	defer conn.Close()
	client := warden.NewClusterClientServiceClient(conn)

	stream, err := client.ServerClusters(context.Background())
	if err != nil {
		grpclog.Fatalf("%v.ServerClusters(_) = _, %v", client, err)
	}
	defer stream.CloseSend()

	var cluster *warden.ClusterAdvertisement
	var reqId string
	for {
		ad, err := stream.Recv()
		if err == io.EOF {
			// stream closed
			grpclog.Fatalln("Stream closed unexpectedly")
			os.Exit(1)
		}
		if err != nil {
			grpclog.Fatalf("Failed to receive: %v", err)
		}
		grpclog.Println("Got message:", ad)
		if cluster == nil {
			// pick the first available cluster
			if ad.State == warden.ClusterAdvertisement_AVAILABLE {
				cluster = ad
				reqId = fmt.Sprintf("res-%x", rand.Int())
				stream.Send(&warden.ClusterRequest{
					RequestId:   reqId,
					Type:        warden.ClusterRequest_RESERVE,
					ClusterId:   ad.ClusterId,
					ClusterType: ad.ClusterType,
					Duration:    int32(*duration),
					Spec: &warden.ClusterRequest_Spec{
						ControllerNodes: uint32(*nodes),
						UserName: *username,
						UserKey: *key,
					},
				})
			}

		} else if cluster.ClusterId == ad.ClusterId && cluster.ClusterType == cluster.ClusterType {
			if ad.RequestId != reqId {
				// cluster was assigned to someone else; try again...
				cluster = nil
				reqId = ""
			} else if ad.State == warden.ClusterAdvertisement_RESERVED {
				fmt.Println("Got cluster:", ad);
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>package client

import (
	"context"
	"io"
	"log"

	"github.com/gokitter/kitter/kitter"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

// KitterCallback allows a method to be called on the native code
type KitterCallback interface {
	NewMessage(message string)
}

// KitterClient creates a new client that can be used for sending and receiving messages
// to a Kitter server
type KitterClient struct {
	client kitter.KitterClient
}

// NewKitterClient creates a new Kitter client and connets to the given server
func NewKitterClient(server string) *KitterClient {
	conn, err := grpc.Dial(server, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("did not connect: %v", err)
	}

	c := kitter.NewKitterClient(conn)

	return &KitterClient{client: c}
}

// WriteMessage sends a message to the kitter server
func (k *KitterClient) WriteMessage(message string) {
	k.client.Keet(context.Background(), &kitter.Message{From: "Nic", Content: message})
}

// ReadStream reads a stream of messages from the kitter server.
// When a new message is received the NewMessage method is called on the
// provided callback
func (k *KitterClient) ReadStream(callback KitterCallback) {
	stream, err := k.client.KeetStream(context.Background(), &kitter.Filter{})
	if err != nil {
		grpclog.Fatalf("%v.ListFeatures(_) = _, %v", k.client, err)
	}
	for {
		message, err := stream.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			grpclog.Fatalf("%v.ListFeatures(_) = _, %v", k.client, err)
		}

		callback.NewMessage(message.Content)
	}
}
<commit_msg>added close method<commit_after>package client

import (
	"context"
	"io"
	"log"

	"github.com/gokitter/kitter/kitter"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

// KitterCallback allows a method to be called on the native code
type KitterCallback interface {
	NewMessage(message string)
}

// KitterClient creates a new client that can be used for sending and receiving messages
// to a Kitter server
type KitterClient struct {
	client kitter.KitterClient
}

// NewKitterClient creates a new Kitter client and connets to the given server
func NewKitterClient(server string) *KitterClient {
	conn, err := grpc.Dial(server, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("did not connect: %v", err)
	}

	c := kitter.NewKitterClient(conn)

	return &KitterClient{client: c}
}

// WriteMessage sends a message to the kitter server
func (k *KitterClient) WriteMessage(message string) {
	k.client.Keet(context.Background(), &kitter.Message{From: "Nic", Content: message})
}

// ReadStream reads a stream of messages from the kitter server.
// When a new message is received the NewMessage method is called on the
// provided callback
func (k *KitterClient) ReadStream(callback KitterCallback) {
	stream, err := k.client.KeetStream(context.Background(), &kitter.Filter{})
	if err != nil {
		grpclog.Fatalf("%v.ListFeatures(_) = _, %v", k.client, err)
	}
	for {
		message, err := stream.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			grpclog.Fatalf("%v.ListFeatures(_) = _, %v", k.client, err)
		}

		callback.NewMessage(message.Content)
	}
}

// Close closes the connection to the server
func (k *KitterClient) Close() {
	k.Close()
}
<|endoftext|>


<commit_before>package game

// Act is different things a player can do.
type Act int

// The acts.
const (
	ActNoOp      Act = iota // Do nothing, just tell everybody the state
	ActStartGame            // Go from lobby state to ingame state
	ActPlayCard
	ActDiscard
	ActReturnToLobby // Go from game over state to lobby state
)

// Action is all info needed from the frontend to act.
type Action struct {
	Act    Act `json:"act"`
	Player int `json:"player"`
	Card   int `json:"card"`
}
<commit_msg>You don't have to send me which player you are<commit_after>package game

// Act is different things a player can do.
type Act int

// The acts.
const (
	ActNoOp      Act = iota // Do nothing, just tell everybody the state
	ActStartGame            // Go from lobby state to ingame state
	ActPlayCard
	ActDiscard
	ActReturnToLobby // Go from game over state to lobby state
)

// Action is all info needed from the frontend to act.
type Action struct {
	Act  Act `json:"act"`
	Card int `json:"card"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"
	"volkan.io/ds"
)

func main() {
	tree := ds.Tree{nil, 42, nil}

	fmt.Println("Tree:")
	fmt.Printf("Typeof tree: “%T”.\n", tree)
	fmt.Printf("Typeof tree: “%s”.\n", reflect.TypeOf(tree))
	fmt.Println("")
}
<commit_msg>minor modification<commit_after>package main

import (
	"fmt"
	"reflect"
	"volkan.io/ds"
)

func main() {
	tree := ds.BTree{nil, 42, nil}

	fmt.Println("Tree:")
	fmt.Printf("Typeof tree: “%T”.\n", tree)
	fmt.Printf("Typeof tree: “%s”.\n", reflect.TypeOf(tree))
	fmt.Println("")
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/nomad/structs"
	"log"
	"net/url"
	"sync"
	"time"
)

const (
	syncInterval = 5 * time.Second
)

type trackedService struct {
	allocId string
	task    *structs.Task
	service *structs.Service
}

func (t *trackedService) IsServiceValid() bool {
	for _, service := range t.task.Services {
		if service.Id == t.service.Id {
			return true
		}
	}

	return false
}

type ConsulClient struct {
	client     *consul.Client
	logger     *log.Logger
	shutdownCh chan struct{}

	trackedServices map[string]*trackedService // Service ID to Tracked Service Map
	trackedSrvLock  sync.Mutex
}

func NewConsulClient(logger *log.Logger, consulAddr string) (*ConsulClient, error) {
	var err error
	var c *consul.Client
	cfg := consul.DefaultConfig()
	cfg.Address = consulAddr
	if c, err = consul.NewClient(cfg); err != nil {
		return nil, err
	}

	consulClient := ConsulClient{
		client:          c,
		logger:          logger,
		trackedServices: make(map[string]*trackedService),
		shutdownCh:      make(chan struct{}),
	}

	return &consulClient, nil
}

func (c *ConsulClient) Register(task *structs.Task, allocID string) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		c.logger.Printf("[INFO] consul: Registering service %s with Consul.", service.Name)
		if err := c.registerService(service, task, allocID); err != nil {
			mErr.Errors = append(mErr.Errors, err)
		}
	}

	return mErr.ErrorOrNil()
}

func (c *ConsulClient) Deregister(task *structs.Task) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		c.logger.Printf("[INFO] consul: De-Registering service %v with Consul", service.Name)
		if err := c.deregisterService(service.Id); err != nil {
			c.logger.Printf("[ERROR] consul: Error in de-registering service %v from Consul", service.Name)
			mErr.Errors = append(mErr.Errors, err)
		}
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) ShutDown() {
	close(c.shutdownCh)
}

func (c *ConsulClient) findPortAndHostForLabel(portLabel string, task *structs.Task) (string, int) {
	for _, network := range task.Resources.Networks {
		if p, ok := network.MapLabelToValues()[portLabel]; ok {
			return network.IP, p
		}
	}
	return "", 0
}

func (c *ConsulClient) SyncWithConsul() {
	sync := time.After(syncInterval)
	agent := c.client.Agent()

	for {
		select {
		case <-sync:
			sync = time.After(syncInterval)
			var consulServices map[string]*consul.AgentService
			var err error

			for serviceId, ts := range c.trackedServices {
				if !ts.IsServiceValid() {
					c.logger.Printf("[INFO] Removing service: %s since the task doesn't have it anymore", ts.service.Name)
					c.deregisterService(serviceId)
				}
			}

			// Get the list of the services that Consul knows about
			if consulServices, err = agent.Services(); err != nil {
				c.logger.Printf("[DEBUG] consul: Error while syncing services with Consul: %v", err)
				continue
			}

			// See if we have services that Consul doesn't know about yet.
			// Register with Consul the services which are not registered
			for serviceId := range c.trackedServices {
				if _, ok := consulServices[serviceId]; !ok {
					ts := c.trackedServices[serviceId]
					c.registerService(ts.service, ts.task, ts.allocId)
				}
			}

			// See if consul thinks we have some services which are not running
			// anymore on the node. We de-register those services
			for serviceId := range consulServices {
				if serviceId == "consul" {
					continue
				}
				if _, ok := c.trackedServices[serviceId]; !ok {
					if err := c.deregisterService(serviceId); err != nil {
						c.logger.Printf("[DEBUG] consul: Error while de-registering service with ID: %s", serviceId)
					}
				}
			}
		case <-c.shutdownCh:
			c.logger.Printf("[INFO] Shutting down Consul Client")
			return
		}
	}
}

func (c *ConsulClient) registerService(service *structs.Service, task *structs.Task, allocID string) error {
	var mErr multierror.Error
	service.Id = fmt.Sprintf("%s-%s", allocID, task.Name)
	host, port := c.findPortAndHostForLabel(service.PortLabel, task)
	if host == "" || port == 0 {
		return fmt.Errorf("consul: The port:%s marked for registration of service: %s couldn't be found", service.PortLabel, service.Name)
	}
	checks := c.makeChecks(service, host, port)
	asr := &consul.AgentServiceRegistration{
		ID:      service.Id,
		Name:    service.Name,
		Tags:    service.Tags,
		Port:    port,
		Address: host,
		Checks:  checks,
	}
	ts := &trackedService{
		allocId: allocID,
		task:    task,
		service: service,
	}
	c.trackedSrvLock.Lock()
	c.trackedServices[service.Id] = ts
	c.trackedSrvLock.Unlock()

	if err := c.client.Agent().ServiceRegister(asr); err != nil {
		c.logger.Printf("[ERROR] consul: Error while registering service %v with Consul: %v", service.Name, err)
		mErr.Errors = append(mErr.Errors, err)
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) deregisterService(serviceId string) error {
	c.trackedSrvLock.Lock()
	delete(c.trackedServices, serviceId)
	c.trackedSrvLock.Unlock()

	if err := c.client.Agent().ServiceDeregister(serviceId); err != nil {
		return err
	}
	return nil
}

func (c *ConsulClient) makeChecks(service *structs.Service, ip string, port int) []*consul.AgentServiceCheck {
	var checks []*consul.AgentServiceCheck
	for _, check := range service.Checks {
		c := &consul.AgentServiceCheck{
			Interval: check.Interval.String(),
			Timeout:  check.Timeout.String(),
		}
		switch check.Type {
		case structs.ServiceCheckHTTP:
			if check.Protocol == "" {
				check.Protocol = "http"
			}
			url := url.URL{
				Scheme: check.Protocol,
				Host:   fmt.Sprintf("%s:%d", ip, port),
				Path:   check.Path,
			}
			c.HTTP = url.String()
		case structs.ServiceCheckTCP:
			c.TCP = fmt.Sprintf("%s:%d", ip, port)
		case structs.ServiceCheckScript:
			c.Script = check.Script // TODO This needs to include the path of the alloc dir and based on driver types
		}
		checks = append(checks, c)
	}
	return checks
}
<commit_msg>Triggering the next sync after we have finished the current one<commit_after>package client

import (
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/nomad/structs"
	"log"
	"net/url"
	"sync"
	"time"
)

const (
	syncInterval = 5 * time.Second
)

type trackedService struct {
	allocId string
	task    *structs.Task
	service *structs.Service
}

func (t *trackedService) IsServiceValid() bool {
	for _, service := range t.task.Services {
		if service.Id == t.service.Id {
			return true
		}
	}

	return false
}

type ConsulClient struct {
	client     *consul.Client
	logger     *log.Logger
	shutdownCh chan struct{}

	trackedServices map[string]*trackedService // Service ID to Tracked Service Map
	trackedSrvLock  sync.Mutex
}

func NewConsulClient(logger *log.Logger, consulAddr string) (*ConsulClient, error) {
	var err error
	var c *consul.Client
	cfg := consul.DefaultConfig()
	cfg.Address = consulAddr
	if c, err = consul.NewClient(cfg); err != nil {
		return nil, err
	}

	consulClient := ConsulClient{
		client:          c,
		logger:          logger,
		trackedServices: make(map[string]*trackedService),
		shutdownCh:      make(chan struct{}),
	}

	return &consulClient, nil
}

func (c *ConsulClient) Register(task *structs.Task, allocID string) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		c.logger.Printf("[INFO] consul: Registering service %s with Consul.", service.Name)
		if err := c.registerService(service, task, allocID); err != nil {
			mErr.Errors = append(mErr.Errors, err)
		}
	}

	return mErr.ErrorOrNil()
}

func (c *ConsulClient) Deregister(task *structs.Task) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		c.logger.Printf("[INFO] consul: De-Registering service %v with Consul", service.Name)
		if err := c.deregisterService(service.Id); err != nil {
			c.logger.Printf("[ERROR] consul: Error in de-registering service %v from Consul", service.Name)
			mErr.Errors = append(mErr.Errors, err)
		}
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) ShutDown() {
	close(c.shutdownCh)
}

func (c *ConsulClient) findPortAndHostForLabel(portLabel string, task *structs.Task) (string, int) {
	for _, network := range task.Resources.Networks {
		if p, ok := network.MapLabelToValues()[portLabel]; ok {
			return network.IP, p
		}
	}
	return "", 0
}

func (c *ConsulClient) SyncWithConsul() {
	sync := time.After(syncInterval)
	agent := c.client.Agent()

	for {
		select {
		case <-sync:
			var consulServices map[string]*consul.AgentService
			var err error

			for serviceId, ts := range c.trackedServices {
				if !ts.IsServiceValid() {
					c.logger.Printf("[INFO] consul: Removing service: %s since the task doesn't have it anymore", ts.service.Name)
					c.deregisterService(serviceId)
				}
			}

			// Get the list of the services that Consul knows about
			if consulServices, err = agent.Services(); err != nil {
				c.logger.Printf("[DEBUG] consul: Error while syncing services with Consul: %v", err)
				continue
			}

			// See if we have services that Consul doesn't know about yet.
			// Register with Consul the services which are not registered
			for serviceId := range c.trackedServices {
				if _, ok := consulServices[serviceId]; !ok {
					ts := c.trackedServices[serviceId]
					c.registerService(ts.service, ts.task, ts.allocId)
				}
			}

			// See if consul thinks we have some services which are not running
			// anymore on the node. We de-register those services
			for serviceId := range consulServices {
				if serviceId == "consul" {
					continue
				}
				if _, ok := c.trackedServices[serviceId]; !ok {
					if err := c.deregisterService(serviceId); err != nil {
						c.logger.Printf("[DEBUG] consul: Error while de-registering service with ID: %s", serviceId)
					}
				}
			}
			sync = time.After(syncInterval)
		case <-c.shutdownCh:
			c.logger.Printf("[INFO] Shutting down Consul Client")
			return
		}
	}
}

func (c *ConsulClient) registerService(service *structs.Service, task *structs.Task, allocID string) error {
	var mErr multierror.Error
	service.Id = fmt.Sprintf("%s-%s", allocID, task.Name)
	host, port := c.findPortAndHostForLabel(service.PortLabel, task)
	if host == "" || port == 0 {
		return fmt.Errorf("consul: The port:%s marked for registration of service: %s couldn't be found", service.PortLabel, service.Name)
	}
	checks := c.makeChecks(service, host, port)
	asr := &consul.AgentServiceRegistration{
		ID:      service.Id,
		Name:    service.Name,
		Tags:    service.Tags,
		Port:    port,
		Address: host,
		Checks:  checks,
	}
	ts := &trackedService{
		allocId: allocID,
		task:    task,
		service: service,
	}
	c.trackedSrvLock.Lock()
	c.trackedServices[service.Id] = ts
	c.trackedSrvLock.Unlock()

	if err := c.client.Agent().ServiceRegister(asr); err != nil {
		c.logger.Printf("[ERROR] consul: Error while registering service %v with Consul: %v", service.Name, err)
		mErr.Errors = append(mErr.Errors, err)
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) deregisterService(serviceId string) error {
	c.trackedSrvLock.Lock()
	delete(c.trackedServices, serviceId)
	c.trackedSrvLock.Unlock()

	if err := c.client.Agent().ServiceDeregister(serviceId); err != nil {
		return err
	}
	return nil
}

func (c *ConsulClient) makeChecks(service *structs.Service, ip string, port int) []*consul.AgentServiceCheck {
	var checks []*consul.AgentServiceCheck
	for _, check := range service.Checks {
		c := &consul.AgentServiceCheck{
			Interval: check.Interval.String(),
			Timeout:  check.Timeout.String(),
		}
		switch check.Type {
		case structs.ServiceCheckHTTP:
			if check.Protocol == "" {
				check.Protocol = "http"
			}
			url := url.URL{
				Scheme: check.Protocol,
				Host:   fmt.Sprintf("%s:%d", ip, port),
				Path:   check.Path,
			}
			c.HTTP = url.String()
		case structs.ServiceCheckTCP:
			c.TCP = fmt.Sprintf("%s:%d", ip, port)
		case structs.ServiceCheckScript:
			c.Script = check.Script // TODO This needs to include the path of the alloc dir and based on driver types
		}
		checks = append(checks, c)
	}
	return checks
}
<|endoftext|>


<commit_before>package fastly

import (
	"fmt"
	"net/http"
	"sort"
)

type S3Config config

// https://docs.fastly.com/api/logging#logging_s3
type S3 struct {
	ServiceID string `json:"service_id,omitempty"`
	Version   uint   `json:"version,string,omitempty"`

	Name              string      `json:"name,omitempty"`
	BucketName        string      `json:"bucket_name,omitempty"`
	Domain            string      `json:"domain"`
	AccessKey         string      `json:"access_key,omitempty"`
	SecretKey         string      `json:"secret_key,omitempty"`
	Path              string      `json:"path"`
	Period            uint        `json:"period,string,omitempty"`
	GzipLevel         uint        `json:"gzip_level,string"`
	Format            string      `json:"format"`
	ResponseCondition string      `json:"response_condition"`
	TimestampFormat   string      `json:"timestamp_format"`
	Redundancy        string      `json:"redundancy"`
	MessageType       MessageType `json:"message_type"`
}

type MessageType int

const (
	_                              = iota
	MessageTypeClassic MessageType = iota
	MessageTypeLoggly
	MessageTypeLogplex
	MessageTypeBlank
)

func (s *MessageType) UnmarshalText(b []byte) error {
	switch string(b) {
	case "classic":
		*s = MessageTypeClassic
	case "loggly":
		*s = MessageTypeLoggly
	case "logplex":
		*s = MessageTypeLogplex
	case "blank":
		*s = MessageTypeBlank
	}
	return nil
}

func (s *MessageType) MarshalText() ([]byte, error) {
	switch *s {
	case MessageTypeClassic:
		return []byte("classic"), nil
	case MessageTypeLoggly:
		return []byte("loggly"), nil
	case MessageTypeLogplex:
		return []byte("logplex"), nil
	case MessageTypeBlank:
		return []byte("blank"), nil
	}
	return nil, nil
}

// s3sByName is a sortable list of s3s.
type s3sByName []*S3

// Len, Swap, and Less implement the sortable interface.
func (s s3sByName) Len() int      { return len(s) }
func (s s3sByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s s3sByName) Less(i, j int) bool {
	return s[i].Name < s[j].Name
}

// List s3s for a specific service and version.
func (c *S3Config) List(serviceID string, version uint) ([]*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3", serviceID, version)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	s3s := new([]*S3)
	resp, err := c.client.Do(req, s3s)
	if err != nil {
		return nil, resp, err
	}

	sort.Stable(s3sByName(*s3s))

	return *s3s, resp, nil
}

// Get fetches a specific s3 by name.
func (c *S3Config) Get(serviceID string, version uint, name string) (*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3/%s", serviceID, version, name)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	s3 := new(S3)
	resp, err := c.client.Do(req, s3)
	if err != nil {
		return nil, resp, err
	}
	return s3, resp, nil
}

// Create a new s3.
func (c *S3Config) Create(serviceID string, version uint, s3 *S3) (*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3", serviceID, version)

	req, err := c.client.NewJSONRequest("POST", u, s3)
	if err != nil {
		return nil, nil, err
	}

	b := new(S3)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Update a s3
func (c *S3Config) Update(serviceID string, version uint, name string, s3 *S3) (*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3/%s", serviceID, version, name)

	req, err := c.client.NewJSONRequest("PUT", u, s3)
	if err != nil {
		return nil, nil, err
	}

	b := new(S3)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Delete a s3
func (c *S3Config) Delete(serviceID string, version uint, name string) (*http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3/%s", serviceID, version, name)

	req, err := c.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}
<commit_msg>Allow mesage_type to be empty on S3 logging<commit_after>package fastly

import (
	"fmt"
	"net/http"
	"sort"
)

type S3Config config

// https://docs.fastly.com/api/logging#logging_s3
type S3 struct {
	ServiceID string `json:"service_id,omitempty"`
	Version   uint   `json:"version,string,omitempty"`

	Name              string      `json:"name,omitempty"`
	BucketName        string      `json:"bucket_name,omitempty"`
	Domain            string      `json:"domain"`
	AccessKey         string      `json:"access_key,omitempty"`
	SecretKey         string      `json:"secret_key,omitempty"`
	Path              string      `json:"path"`
	Period            uint        `json:"period,string,omitempty"`
	GzipLevel         uint        `json:"gzip_level,string"`
	Format            string      `json:"format"`
	ResponseCondition string      `json:"response_condition"`
	TimestampFormat   string      `json:"timestamp_format"`
	Redundancy        string      `json:"redundancy"`
	MessageType       MessageType `json:"message_type,omitempty"`
}

type MessageType int

const (
	_                              = iota
	MessageTypeClassic MessageType = iota
	MessageTypeLoggly
	MessageTypeLogplex
	MessageTypeBlank
)

func (s *MessageType) UnmarshalText(b []byte) error {
	switch string(b) {
	case "classic":
		*s = MessageTypeClassic
	case "loggly":
		*s = MessageTypeLoggly
	case "logplex":
		*s = MessageTypeLogplex
	case "blank":
		*s = MessageTypeBlank
	}
	return nil
}

func (s *MessageType) MarshalText() ([]byte, error) {
	switch *s {
	case MessageTypeClassic:
		return []byte("classic"), nil
	case MessageTypeLoggly:
		return []byte("loggly"), nil
	case MessageTypeLogplex:
		return []byte("logplex"), nil
	case MessageTypeBlank:
		return []byte("blank"), nil
	}
	return nil, nil
}

// s3sByName is a sortable list of s3s.
type s3sByName []*S3

// Len, Swap, and Less implement the sortable interface.
func (s s3sByName) Len() int      { return len(s) }
func (s s3sByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s s3sByName) Less(i, j int) bool {
	return s[i].Name < s[j].Name
}

// List s3s for a specific service and version.
func (c *S3Config) List(serviceID string, version uint) ([]*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3", serviceID, version)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	s3s := new([]*S3)
	resp, err := c.client.Do(req, s3s)
	if err != nil {
		return nil, resp, err
	}

	sort.Stable(s3sByName(*s3s))

	return *s3s, resp, nil
}

// Get fetches a specific s3 by name.
func (c *S3Config) Get(serviceID string, version uint, name string) (*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3/%s", serviceID, version, name)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	s3 := new(S3)
	resp, err := c.client.Do(req, s3)
	if err != nil {
		return nil, resp, err
	}
	return s3, resp, nil
}

// Create a new s3.
func (c *S3Config) Create(serviceID string, version uint, s3 *S3) (*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3", serviceID, version)

	req, err := c.client.NewJSONRequest("POST", u, s3)
	if err != nil {
		return nil, nil, err
	}

	b := new(S3)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Update a s3
func (c *S3Config) Update(serviceID string, version uint, name string, s3 *S3) (*S3, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3/%s", serviceID, version, name)

	req, err := c.client.NewJSONRequest("PUT", u, s3)
	if err != nil {
		return nil, nil, err
	}

	b := new(S3)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Delete a s3
func (c *S3Config) Delete(serviceID string, version uint, name string) (*http.Response, error) {
	u := fmt.Sprintf("/service/%s/version/%d/logging/s3/%s", serviceID, version, name)

	req, err := c.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package repo

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/rs/zerolog/log"

	"github.com/kopia/kopia/manifest"

	"github.com/kopia/kopia/object"

	"github.com/kopia/kopia/auth"
	"github.com/kopia/kopia/block"
	"github.com/kopia/kopia/internal/config"
	"github.com/kopia/kopia/internal/units"
	"github.com/kopia/kopia/storage"
	"github.com/kopia/kopia/storage/logging"

	// Register well-known blob storage providers
	_ "github.com/kopia/kopia/storage/filesystem"
	_ "github.com/kopia/kopia/storage/gcs"
)

// Options provides configuration parameters for connection to a repository.
type Options struct {
	CredentialsCallback func() (auth.Credentials, error)    // Provides credentials required to open the repository if not persisted.
	TraceStorage        func(f string, args ...interface{}) // Logs all storage access using provided Printf-style function

	DisableCache         bool // disable caching
	DisableListCache     bool // disable list caching
	ObjectManagerOptions object.ManagerOptions
}

// Open opens a Repository specified in the configuration file.
func Open(ctx context.Context, configFile string, options *Options) (rep *Repository, err error) {
	t0 := time.Now()
	log.Debug().Msgf("opening repository from %v", configFile)
	defer func() {
		if err == nil {
			log.Debug().Dur("duration_ms", time.Since(t0)).Msg("opened repository")
		} else {
			log.Error().Dur("duration_ms", time.Since(t0)).Msg("failed to open repository")
		}
	}()

	if options == nil {
		options = &Options{}
	}

	configFile, err = filepath.Abs(configFile)
	if err != nil {
		return nil, err
	}

	log.Debug().Str("file", configFile).Msg("loading config")
	lc, err := config.LoadFromFile(configFile)
	if err != nil {
		return nil, err
	}

	var creds auth.Credentials
	if len(lc.Connection.Key) > 0 {
		log.Debug().Msg("getting credentials from master key")
		creds, err = auth.MasterKey(lc.Connection.Key)
	} else {
		if options.CredentialsCallback == nil {
			return nil, errors.New("key not persisted and no credentials specified")
		}
		log.Debug().Msg("getting credentials using callback")
		creds, err = options.CredentialsCallback()
	}

	if err != nil {
		return nil, fmt.Errorf("invalid credentials: %v", err)
	}

	log.Debug().Str("connection", lc.Connection.ConnectionInfo.Type).Interface("params", lc.Connection.ConnectionInfo.Config).Msg("opening storage")

	st, err := storage.NewStorage(ctx, lc.Connection.ConnectionInfo)
	if err != nil {
		return nil, fmt.Errorf("cannot open storage: %v", err)
	}

	caching := lc.Caching
	if options.DisableCache {
		caching = block.CachingOptions{}
	}
	if options.DisableListCache {
		caching.IgnoreListCache = true
	}

	r, err := connect(ctx, st, creds, options, caching)
	if err != nil {
		st.Close()
		return nil, err
	}

	r.ConfigFile = configFile

	return r, nil
}

// SetCachingConfig changes caching configuration for a given repository config file.
func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingOptions) error {
	configFile, err := filepath.Abs(configFile)
	if err != nil {
		return err
	}

	log.Debug().Str("file", configFile).Msg("loading config")
	lc, err := config.LoadFromFile(configFile)
	if err != nil {
		return err
	}

	st, err := storage.NewStorage(ctx, lc.Connection.ConnectionInfo)
	if err != nil {
		return fmt.Errorf("cannot open storage: %v", err)
	}

	f, err := readAndCacheFormatBlock(st, "")
	if err != nil {
		return fmt.Errorf("can't read format block: %v", err)
	}

	if opt.MaxCacheSizeBytes > 0 {
		if opt.CacheDirectory == "" {
			// derive cache directory from config
			absConfig, err := filepath.Abs(configFile)
			if err != nil {
				return err
			}
			lc.Caching.CacheDirectory = filepath.Join(filepath.Dir(absConfig), fmt.Sprintf("cache-%x", f.UniqueID))
		} else {
			absCacheDir, err := filepath.Abs(opt.CacheDirectory)
			if err != nil {
				return err
			}

			lc.Caching.CacheDirectory = absCacheDir
		}
		lc.Caching.MaxCacheSizeBytes = opt.MaxCacheSizeBytes
		lc.Caching.MaxListCacheDurationSec = opt.MaxListCacheDurationSec

		log.Printf("Enabling cache directory '%v' with max size %v", lc.Caching.CacheDirectory, units.BytesStringBase2(lc.Caching.MaxCacheSizeBytes))
		os.MkdirAll(lc.Caching.CacheDirectory, 0700)
	} else {
		log.Printf("Disabling caching")
		lc.Caching = block.CachingOptions{}
	}

	d, err := json.MarshalIndent(&lc, "", "  ")
	if err != nil {
		return err
	}

	if err := ioutil.WriteFile(configFile, d, 0600); err != nil {
		return nil
	}

	return nil
}

func readAndCacheFormatBlock(st storage.Storage, cacheDirectory string) (*formatBlock, error) {
	cachedFile := filepath.Join(cacheDirectory, "kopia.repository")
	if cacheDirectory != "" {
		b, err := ioutil.ReadFile(cachedFile)
		if err == nil {
			// read from cache.
			return parseFormatBlock(b)
		}
	}

	b, err := st.GetBlock(formatBlockID, 0, -1)
	if err != nil {
		return nil, err
	}

	// block successfully read from storage.
	f, err := parseFormatBlock(b)
	if err != nil {
		return nil, err
	}

	if cacheDirectory != "" {
		if err := ioutil.WriteFile(cachedFile, b, 0600); err != nil {
			log.Printf("warning: unable to write cache: %v", err)
		}
	}

	return f, nil
}

func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, options *Options, caching block.CachingOptions) (*Repository, error) {
	if options == nil {
		options = &Options{}
	}
	if options.TraceStorage != nil {
		st = logging.NewWrapper(st, logging.Prefix("[STORAGE] "), logging.Output(options.TraceStorage))
	}

	log.Debug().Msg("reading encrypted format block")
	// Read cache block, potentially from cache.
	f, err := readAndCacheFormatBlock(st, caching.CacheDirectory)
	if err != nil {
		return nil, fmt.Errorf("unable to read format block: %v", err)
	}

	km, err := auth.NewKeyManager(creds, f.SecurityOptions)
	if err != nil {
		return nil, err
	}

	repoConfig, err := decryptFormatBytes(f, km)
	if err != nil {
		return nil, fmt.Errorf("unable to decrypt repository config: %v", err)
	}

	fo := repoConfig.FormattingOptions
	if fo.MaxPackSize == 0 {
		fo.MaxPackSize = repoConfig.MaxBlockSize
	}

	caching.HMACSecret = km.DeriveKey([]byte("local-cache-integrity"), 16)

	log.Debug().Msg("initializing block manager")
	bm, err := block.NewManager(st, fo, caching)
	if err != nil {
		return nil, fmt.Errorf("unable to open block manager: %v", err)
	}

	log.Debug().Msg("initializing object manager")
	om, err := object.NewObjectManager(bm, *repoConfig, options.ObjectManagerOptions)
	if err != nil {
		return nil, fmt.Errorf("unable to open object manager: %v", err)
	}

	log.Debug().Msg("initializing manifest manager")
	manifests, err := manifest.NewManager(bm)
	if err != nil {
		return nil, fmt.Errorf("unable to open manifests: %v", err)
	}

	return &Repository{
		Blocks:         bm,
		Objects:        om,
		Storage:        st,
		KeyManager:     km,
		Manifests:      manifests,
		Security:       f.SecurityOptions,
		CacheDirectory: caching.CacheDirectory,
	}, nil
}
<commit_msg>removed depedendency on actual providers from 'repo'<commit_after>package repo

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/rs/zerolog/log"

	"github.com/kopia/kopia/auth"
	"github.com/kopia/kopia/block"
	"github.com/kopia/kopia/internal/config"
	"github.com/kopia/kopia/internal/units"
	"github.com/kopia/kopia/manifest"
	"github.com/kopia/kopia/object"
	"github.com/kopia/kopia/storage"
	"github.com/kopia/kopia/storage/logging"
)

// Options provides configuration parameters for connection to a repository.
type Options struct {
	CredentialsCallback func() (auth.Credentials, error)    // Provides credentials required to open the repository if not persisted.
	TraceStorage        func(f string, args ...interface{}) // Logs all storage access using provided Printf-style function

	DisableCache         bool // disable caching
	DisableListCache     bool // disable list caching
	ObjectManagerOptions object.ManagerOptions
}

// Open opens a Repository specified in the configuration file.
func Open(ctx context.Context, configFile string, options *Options) (rep *Repository, err error) {
	t0 := time.Now()
	log.Debug().Msgf("opening repository from %v", configFile)
	defer func() {
		if err == nil {
			log.Debug().Dur("duration_ms", time.Since(t0)).Msg("opened repository")
		} else {
			log.Error().Dur("duration_ms", time.Since(t0)).Msg("failed to open repository")
		}
	}()

	if options == nil {
		options = &Options{}
	}

	configFile, err = filepath.Abs(configFile)
	if err != nil {
		return nil, err
	}

	log.Debug().Str("file", configFile).Msg("loading config")
	lc, err := config.LoadFromFile(configFile)
	if err != nil {
		return nil, err
	}

	var creds auth.Credentials
	if len(lc.Connection.Key) > 0 {
		log.Debug().Msg("getting credentials from master key")
		creds, err = auth.MasterKey(lc.Connection.Key)
	} else {
		if options.CredentialsCallback == nil {
			return nil, errors.New("key not persisted and no credentials specified")
		}
		log.Debug().Msg("getting credentials using callback")
		creds, err = options.CredentialsCallback()
	}

	if err != nil {
		return nil, fmt.Errorf("invalid credentials: %v", err)
	}

	log.Debug().Str("connection", lc.Connection.ConnectionInfo.Type).Msg("opening storage")

	st, err := storage.NewStorage(ctx, lc.Connection.ConnectionInfo)
	if err != nil {
		return nil, fmt.Errorf("cannot open storage: %v", err)
	}

	caching := lc.Caching
	if options.DisableCache {
		caching = block.CachingOptions{}
	}
	if options.DisableListCache {
		caching.IgnoreListCache = true
	}

	r, err := connect(ctx, st, creds, options, caching)
	if err != nil {
		st.Close()
		return nil, err
	}

	r.ConfigFile = configFile

	return r, nil
}

// SetCachingConfig changes caching configuration for a given repository config file.
func SetCachingConfig(ctx context.Context, configFile string, opt block.CachingOptions) error {
	configFile, err := filepath.Abs(configFile)
	if err != nil {
		return err
	}

	log.Debug().Str("file", configFile).Msg("loading config")
	lc, err := config.LoadFromFile(configFile)
	if err != nil {
		return err
	}

	st, err := storage.NewStorage(ctx, lc.Connection.ConnectionInfo)
	if err != nil {
		return fmt.Errorf("cannot open storage: %v", err)
	}

	f, err := readAndCacheFormatBlock(st, "")
	if err != nil {
		return fmt.Errorf("can't read format block: %v", err)
	}

	if opt.MaxCacheSizeBytes > 0 {
		if opt.CacheDirectory == "" {
			// derive cache directory from config
			absConfig, err := filepath.Abs(configFile)
			if err != nil {
				return err
			}
			lc.Caching.CacheDirectory = filepath.Join(filepath.Dir(absConfig), fmt.Sprintf("cache-%x", f.UniqueID))
		} else {
			absCacheDir, err := filepath.Abs(opt.CacheDirectory)
			if err != nil {
				return err
			}

			lc.Caching.CacheDirectory = absCacheDir
		}
		lc.Caching.MaxCacheSizeBytes = opt.MaxCacheSizeBytes
		lc.Caching.MaxListCacheDurationSec = opt.MaxListCacheDurationSec

		log.Printf("Enabling cache directory '%v' with max size %v", lc.Caching.CacheDirectory, units.BytesStringBase2(lc.Caching.MaxCacheSizeBytes))
		os.MkdirAll(lc.Caching.CacheDirectory, 0700)
	} else {
		log.Printf("Disabling caching")
		lc.Caching = block.CachingOptions{}
	}

	d, err := json.MarshalIndent(&lc, "", "  ")
	if err != nil {
		return err
	}

	if err := ioutil.WriteFile(configFile, d, 0600); err != nil {
		return nil
	}

	return nil
}

func readAndCacheFormatBlock(st storage.Storage, cacheDirectory string) (*formatBlock, error) {
	cachedFile := filepath.Join(cacheDirectory, "kopia.repository")
	if cacheDirectory != "" {
		b, err := ioutil.ReadFile(cachedFile)
		if err == nil {
			// read from cache.
			return parseFormatBlock(b)
		}
	}

	b, err := st.GetBlock(formatBlockID, 0, -1)
	if err != nil {
		return nil, err
	}

	// block successfully read from storage.
	f, err := parseFormatBlock(b)
	if err != nil {
		return nil, err
	}

	if cacheDirectory != "" {
		if err := ioutil.WriteFile(cachedFile, b, 0600); err != nil {
			log.Printf("warning: unable to write cache: %v", err)
		}
	}

	return f, nil
}

func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, options *Options, caching block.CachingOptions) (*Repository, error) {
	if options == nil {
		options = &Options{}
	}
	if options.TraceStorage != nil {
		st = logging.NewWrapper(st, logging.Prefix("[STORAGE] "), logging.Output(options.TraceStorage))
	}

	log.Debug().Msg("reading encrypted format block")
	// Read cache block, potentially from cache.
	f, err := readAndCacheFormatBlock(st, caching.CacheDirectory)
	if err != nil {
		return nil, fmt.Errorf("unable to read format block: %v", err)
	}

	km, err := auth.NewKeyManager(creds, f.SecurityOptions)
	if err != nil {
		return nil, err
	}

	repoConfig, err := decryptFormatBytes(f, km)
	if err != nil {
		return nil, fmt.Errorf("unable to decrypt repository config: %v", err)
	}

	fo := repoConfig.FormattingOptions
	if fo.MaxPackSize == 0 {
		fo.MaxPackSize = repoConfig.MaxBlockSize
	}

	caching.HMACSecret = km.DeriveKey([]byte("local-cache-integrity"), 16)

	log.Debug().Msg("initializing block manager")
	bm, err := block.NewManager(st, fo, caching)
	if err != nil {
		return nil, fmt.Errorf("unable to open block manager: %v", err)
	}

	log.Debug().Msg("initializing object manager")
	om, err := object.NewObjectManager(bm, *repoConfig, options.ObjectManagerOptions)
	if err != nil {
		return nil, fmt.Errorf("unable to open object manager: %v", err)
	}

	log.Debug().Msg("initializing manifest manager")
	manifests, err := manifest.NewManager(bm)
	if err != nil {
		return nil, fmt.Errorf("unable to open manifests: %v", err)
	}

	return &Repository{
		Blocks:         bm,
		Objects:        om,
		Storage:        st,
		KeyManager:     km,
		Manifests:      manifests,
		Security:       f.SecurityOptions,
		CacheDirectory: caching.CacheDirectory,
	}, nil
}
<|endoftext|>