<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sessions

import (
	"encoding/base32"
	"io"
	"net/http"
	"os"
	"strings"
	"sync"

	"code.google.com/p/gorilla/securecookie"
)

// Store is an interface for custom session stores.
type Store interface {
	Get(r *http.Request, name string) (*Session, error)
	New(r *http.Request, name string) (*Session, error)
	Save(r *http.Request, w http.ResponseWriter, s *Session) error
}

// CookieStore ----------------------------------------------------------------

// NewCookieStore returns a new CookieStore.
//
// Keys are defined in pairs to allow key rotation, but the common case is
// to set a single authentication key and optionally an encryption key.
//
// The first key in a pair is used for authentication and the second for
// encryption. The encryption key can be set to nil or omitted in the last
// pair, but the authentication key is required in all pairs.
//
// It is recommended to use an authentication key with 32 or 64 bytes.
// The encryption key, if set, must be either 16, 24, or 32 bytes to select
// AES-128, AES-192, or AES-256 modes.
//
// Use the convenience function securecookie.GenerateRandomKey() to create
// strong keys.
func NewCookieStore(keyPairs ...[]byte) *CookieStore {
	return &CookieStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
	}
}

// CookieStore stores sessions using secure cookies.
type CookieStore struct {
	Codecs  []securecookie.Codec
	Options *Options // default configuration
}

// Get returns a session for the given name after adding it to the registry.
//
// It returns a new session if the sessions doesn't exist. Access IsNew on
// the session to check if it is an existing session or a new one.
//
// It returns a new session and an error if the session exists but could
// not be decoded.
func (s *CookieStore) Get(r *http.Request, name string) (*Session, error) {
	return GetRegistry(r).Get(s, name)
}

// New returns a session for the given name without adding it to the registry.
//
// The difference between New() and Get() is that calling New() twice will
// decode the session data twice, while Get() registers and reuses the same
// decoded session after the first call.
func (s *CookieStore) New(r *http.Request, name string) (*Session, error) {
	session := NewSession(s, name)
	session.Options = &(*s.Options)
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.Values,
			s.Codecs...)
		if err == nil {
			session.IsNew = false
		}
	}
	return session, err
}

// Save adds a single session to the response.
func (s *CookieStore) Save(r *http.Request, w http.ResponseWriter,
	session *Session) error {
	encoded, err := securecookie.EncodeMulti(session.Name(), session.Values,
		s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, NewCookie(session.Name(), encoded, session.Options))
	return nil
}

// FilesystemStore ------------------------------------------------------------

var fileMutex sync.RWMutex

// NewFilesystemStore returns a new FilesystemStore.
//
// The path argument is the directory where sessions will be saved. If empty
// it will use os.TempDir().
//
// See NewCookieStore() for a description of the other parameters.
func NewFilesystemStore(path string, keyPairs ...[]byte) *FilesystemStore {
	if path == "" {
		path = os.TempDir()
	}
	if path[len(path)-1] != '/' {
		path += "/"
	}
	return &FilesystemStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
		path: path,
	}
}

// FilesystemStore stores sessions in the filesystem.
//
// It also serves as a referece for custom stores.
//
// This store is still experimental and not well tested. Feedback is welcome.
type FilesystemStore struct {
	Codecs  []securecookie.Codec
	Options *Options // default configuration
	path    string
}

// Get returns a session for the given name after adding it to the registry.
//
// See CookieStore.Get().
func (s *FilesystemStore) Get(r *http.Request, name string) (*Session, error) {
	return GetRegistry(r).Get(s, name)
}

// New returns a session for the given name without adding it to the registry.
//
// See CookieStore.New().
func (s *FilesystemStore) New(r *http.Request, name string) (*Session, error) {
	session := NewSession(s, name)
	session.Options = &(*s.Options)
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...)
		if err == nil {
			err = s.load(session)
			if err == nil {
				session.IsNew = false
			}
		}
	}
	return session, err
}

// Save adds a single session to the response.
func (s *FilesystemStore) Save(r *http.Request, w http.ResponseWriter,
	session *Session) error {
	if session.ID == "" {
		// Because the ID is used in the filename, encode it to
		// use alphanumeric characters only.
		session.ID = strings.TrimRight(
			base32.StdEncoding.EncodeToString(
				securecookie.GenerateRandomKey(32)), "=")
	}
	if err := s.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID,
		s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, NewCookie(session.Name(), encoded, session.Options))
	return nil
}

// save writes encoded session.Values to a file.
func (s *FilesystemStore) save(session *Session) error {
	encoded, err := securecookie.EncodeMulti(session.Name(), session.Values,
		s.Codecs...)
	if err != nil {
		return err
	}
	filename := s.path + "session_" + session.ID
	fileMutex.Lock()
	defer fileMutex.Unlock()
	fp, err2 := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0600)
	if err2 != nil {
		return err2
	}
	if _, err = fp.Write([]byte(encoded)); err != nil {
		return err
	}
	fp.Close()
	return nil
}

// load reads a file and decodes its content into session.Values.
func (s *FilesystemStore) load(session *Session) error {
	filename := s.path + "session_" + session.ID
	fp, err := os.OpenFile(filename, os.O_RDONLY, 0400)
	if err != nil {
		return err
	}
	defer fp.Close()
	var fdata []byte
	buf := make([]byte, 128)
	for {
		var n int
		n, err = fp.Read(buf[0:])
		fdata = append(fdata, buf[0:n]...)
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
	}
	if err = securecookie.DecodeMulti(session.Name(), string(fdata),
		&session.Values, s.Codecs...); err != nil {
		return err
	}
	return nil
}
<commit_msg>Truncate session files when writing, otherwise we might not be able to parse them later on. Patch from Michael Stapelberg (thanks!). Fixes issue #53<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sessions

import (
	"encoding/base32"
	"io"
	"net/http"
	"os"
	"strings"
	"sync"

	"github.com/gorilla/securecookie"
)

// Store is an interface for custom session stores.
type Store interface {
	Get(r *http.Request, name string) (*Session, error)
	New(r *http.Request, name string) (*Session, error)
	Save(r *http.Request, w http.ResponseWriter, s *Session) error
}

// CookieStore ----------------------------------------------------------------

// NewCookieStore returns a new CookieStore.
//
// Keys are defined in pairs to allow key rotation, but the common case is
// to set a single authentication key and optionally an encryption key.
//
// The first key in a pair is used for authentication and the second for
// encryption. The encryption key can be set to nil or omitted in the last
// pair, but the authentication key is required in all pairs.
//
// It is recommended to use an authentication key with 32 or 64 bytes.
// The encryption key, if set, must be either 16, 24, or 32 bytes to select
// AES-128, AES-192, or AES-256 modes.
//
// Use the convenience function securecookie.GenerateRandomKey() to create
// strong keys.
func NewCookieStore(keyPairs ...[]byte) *CookieStore {
	return &CookieStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
	}
}

// CookieStore stores sessions using secure cookies.
type CookieStore struct {
	Codecs  []securecookie.Codec
	Options *Options // default configuration
}

// Get returns a session for the given name after adding it to the registry.
//
// It returns a new session if the sessions doesn't exist. Access IsNew on
// the session to check if it is an existing session or a new one.
//
// It returns a new session and an error if the session exists but could
// not be decoded.
func (s *CookieStore) Get(r *http.Request, name string) (*Session, error) {
	return GetRegistry(r).Get(s, name)
}

// New returns a session for the given name without adding it to the registry.
//
// The difference between New() and Get() is that calling New() twice will
// decode the session data twice, while Get() registers and reuses the same
// decoded session after the first call.
func (s *CookieStore) New(r *http.Request, name string) (*Session, error) {
	session := NewSession(s, name)
	session.Options = &(*s.Options)
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.Values,
			s.Codecs...)
		if err == nil {
			session.IsNew = false
		}
	}
	return session, err
}

// Save adds a single session to the response.
func (s *CookieStore) Save(r *http.Request, w http.ResponseWriter,
	session *Session) error {
	encoded, err := securecookie.EncodeMulti(session.Name(), session.Values,
		s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, NewCookie(session.Name(), encoded, session.Options))
	return nil
}

// FilesystemStore ------------------------------------------------------------

var fileMutex sync.RWMutex

// NewFilesystemStore returns a new FilesystemStore.
//
// The path argument is the directory where sessions will be saved. If empty
// it will use os.TempDir().
//
// See NewCookieStore() for a description of the other parameters.
func NewFilesystemStore(path string, keyPairs ...[]byte) *FilesystemStore {
	if path == "" {
		path = os.TempDir()
	}
	if path[len(path)-1] != '/' {
		path += "/"
	}
	return &FilesystemStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
		path: path,
	}
}

// FilesystemStore stores sessions in the filesystem.
//
// It also serves as a referece for custom stores.
//
// This store is still experimental and not well tested. Feedback is welcome.
type FilesystemStore struct {
	Codecs  []securecookie.Codec
	Options *Options // default configuration
	path    string
}

// Get returns a session for the given name after adding it to the registry.
//
// See CookieStore.Get().
func (s *FilesystemStore) Get(r *http.Request, name string) (*Session, error) {
	return GetRegistry(r).Get(s, name)
}

// New returns a session for the given name without adding it to the registry.
//
// See CookieStore.New().
func (s *FilesystemStore) New(r *http.Request, name string) (*Session, error) {
	session := NewSession(s, name)
	session.Options = &(*s.Options)
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...)
		if err == nil {
			err = s.load(session)
			if err == nil {
				session.IsNew = false
			}
		}
	}
	return session, err
}

// Save adds a single session to the response.
func (s *FilesystemStore) Save(r *http.Request, w http.ResponseWriter,
	session *Session) error {
	if session.ID == "" {
		// Because the ID is used in the filename, encode it to
		// use alphanumeric characters only.
		session.ID = strings.TrimRight(
			base32.StdEncoding.EncodeToString(
				securecookie.GenerateRandomKey(32)), "=")
	}
	if err := s.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID,
		s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, NewCookie(session.Name(), encoded, session.Options))
	return nil
}

// save writes encoded session.Values to a file.
func (s *FilesystemStore) save(session *Session) error {
	encoded, err := securecookie.EncodeMulti(session.Name(), session.Values,
		s.Codecs...)
	if err != nil {
		return err
	}
	filename := s.path + "session_" + session.ID
	fileMutex.Lock()
	defer fileMutex.Unlock()
	fp, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	if _, err = fp.Write([]byte(encoded)); err != nil {
		return err
	}
	fp.Close()
	return nil
}

// load reads a file and decodes its content into session.Values.
func (s *FilesystemStore) load(session *Session) error {
	filename := s.path + "session_" + session.ID
	fp, err := os.OpenFile(filename, os.O_RDONLY, 0400)
	if err != nil {
		return err
	}
	defer fp.Close()
	var fdata []byte
	buf := make([]byte, 128)
	for {
		var n int
		n, err = fp.Read(buf[0:])
		fdata = append(fdata, buf[0:n]...)
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
	}
	if err = securecookie.DecodeMulti(session.Name(), string(fdata),
		&session.Values, s.Codecs...); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package benchdb

import (
	"bytes"
	"crypto/rand"
	"database/sql"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	_ "github.com/lib/pq"

	"golang.org/x/tools/benchmark/parse"
)

// A BenchDB manges the execution of benchmark tests using go test and
// writing a parse.Set to a database.
//
// Implementations of BenchDB for different databases are allowed by way
// of the WriteSet method.
type BenchDB interface {
	// Run executes go test bench for benchmarks matching regex in the current
	// directory. By default it does not run unit tests by way of setting test.run
	// to XXX in the call to go test. It also parses the benchSet and calls WriteSet
	// to write the benchmark data to a database. It returns any error encountered.
	Run(regex string) error

	// WriteSet is responsible for opening a postgres database connection and writing
	// a parsed benchSet to a db table. It closes the connection, returns the number of
	// benchmark tests written, and any error encountered.
	WriteSet(parse.Set) (int, error)
}

// BenchPSQL represents a BenchDB that writes benchmarks to a postgres
// database.
type BenchPSQL struct {
	Driver    string // database driver name
	ConnStr   string // sql connection string
	TableName string // database table name

	dbConn *sql.DB
}

// Run runs go test benchmarks matching regex in the current directory and writes
// benchmark data to a PSQL database by calling WriteSet. It returns any error
// encountered.
func (benchdb *BenchPSQL) Run(regex string) error {
	// Exec a subprocess for go test bench and write
	// to both stdout and a byte buffer.
	cmd := exec.Command("go", "test", "-bench", regex,
		"-test.run", "XXX", "-benchmem")
	var out bytes.Buffer
	cmd.Stdout = io.MultiWriter(os.Stdout, &out)
	cmd.Stderr = io.Writer(os.Stderr)
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("command failed: %v", err)
	}

	benchSet, err := parse.ParseSet(&out)
	if err != nil {
		return fmt.Errorf("failed to parse benchmark data: %v", err)
	}

	// Writes parse set to sql database.
	_, err = benchdb.WriteSet(benchSet)
	if err != nil {
		return fmt.Errorf("failed to write benchSet to db: %v", err)
	}
	return nil
}

const nsha = 7

func latestGitSha() (string, error) {
	out, err := exec.Command("git", "rev-parse", "HEAD").Output()
	if err != nil {
		return "", fmt.Errorf("failed to get latest git sha: %v\n", err)
	}
	return string(out[:nsha]), nil
}

func uuid() (string, error) {
	b := make([]byte, 16)
	if _, err := io.ReadFull(rand.Reader, b); err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

// WriteSet is responsible for opening a postgres database connection and writing
// a parsed benchSet to a db table. It closes the connection, returns the number of
// benchmark tests written, and any error encountered.
//
// A new sql transaction is created and committed per Benchmark in benchSet. This way if a
// db failure occurs all data from the benchSet is not lost.
func (benchdb *BenchPSQL) WriteSet(benchSet parse.Set) (int, error) {
	sqlDB, err := sql.Open(benchdb.Driver, benchdb.ConnStr)
	if err != nil {
		return 0, fmt.Errorf("could not connect to db: %v", err)
	}
	defer sqlDB.Close()
	benchdb.dbConn = sqlDB

	batchId, err := uuid()
	if err != nil {
		return 0, fmt.Errorf("could not generate batch id: %v\n", err)
	}

	cnt := 0
	for _, b := range benchSet {
		n := len(b)
		for i := 0; i < n; i++ {
			val := b[i]
			err := saveBenchmark(benchdb.dbConn, benchdb.TableName, batchId, *val)
			if err != nil {
				return 0, fmt.Errorf("failed to save benchmark: %v", err)
			}
			cnt++
		}
	}
	return cnt, nil
}

func saveBenchmark(dbConn *sql.DB, table, batchId string, b parse.Benchmark) error {
	// Create a transaction per Benchmark.
	tx, err := dbConn.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	sha, err := latestGitSha()
	if err != nil {
		return err
	}

	q := fmt.Sprintf(`
        INSERT INTO %s
        (batch_id, latest_sha, datetime, name, n, ns_op, allocated_bytes_op, allocs_op)
        VALUES
        ($1, $2, $3, $4, $5, $6, $7, $8)
        `, table)

	// Strips of leading Benchmark string in Benchmark.Name
	name := strings.TrimPrefix(strings.TrimSpace(b.Name), "Benchmark")
	ts := time.Now().UTC()

	_, err = tx.Exec(q,
		batchId, sha, ts, name, b.N, b.NsPerOp, b.AllocedBytesPerOp, b.AllocsPerOp)
	if err != nil {
		return err
	}
	return tx.Commit()
}
<commit_msg>fixed typo in comment<commit_after>package benchdb

import (
	"bytes"
	"crypto/rand"
	"database/sql"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	_ "github.com/lib/pq"

	"golang.org/x/tools/benchmark/parse"
)

// A BenchDB manges the execution of benchmark tests using go test and
// writing a parse.Set to a database.
//
// Implementations of BenchDB for different databases are allowed by way
// of the WriteSet method.
type BenchDB interface {
	// Run executes go test bench for benchmarks matching regex in the current
	// directory. By default it does not run unit tests by way of setting test.run
	// to XXX in the call to go test. It also parses the benchSet and calls WriteSet
	// to write the benchmark data to a database. It returns any error encountered.
	Run(regex string) error

	// WriteSet is responsible for opening a postgres database connection and writing
	// a parsed benchSet to a db table. It closes the connection, returns the number of
	// benchmark tests written, and any error encountered.
	WriteSet(parse.Set) (int, error)
}

// BenchPSQL represents a BenchDB that writes benchmarks to a postgres
// database.
type BenchPSQL struct {
	Driver    string // database driver name
	ConnStr   string // sql connection string
	TableName string // database table name

	dbConn *sql.DB
}

// Run runs go test benchmarks matching regex in the current directory and writes
// benchmark data to a PSQL database by calling WriteSet. It returns any error
// encountered.
func (benchdb *BenchPSQL) Run(regex string) error {
	// Exec a subprocess for go test bench and write
	// to both stdout and a byte buffer.
	cmd := exec.Command("go", "test", "-bench", regex,
		"-test.run", "XXX", "-benchmem")
	var out bytes.Buffer
	cmd.Stdout = io.MultiWriter(os.Stdout, &out)
	cmd.Stderr = io.Writer(os.Stderr)
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("command failed: %v", err)
	}

	benchSet, err := parse.ParseSet(&out)
	if err != nil {
		return fmt.Errorf("failed to parse benchmark data: %v", err)
	}

	// Writes parse set to sql database.
	_, err = benchdb.WriteSet(benchSet)
	if err != nil {
		return fmt.Errorf("failed to write benchSet to db: %v", err)
	}
	return nil
}

const nsha = 7

func latestGitSha() (string, error) {
	out, err := exec.Command("git", "rev-parse", "HEAD").Output()
	if err != nil {
		return "", fmt.Errorf("failed to get latest git sha: %v\n", err)
	}
	return string(out[:nsha]), nil
}

func uuid() (string, error) {
	b := make([]byte, 16)
	if _, err := io.ReadFull(rand.Reader, b); err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

// WriteSet is responsible for opening a postgres database connection and writing
// a parsed benchSet to a db table. It closes the connection, returns the number of
// benchmark tests written, and any error encountered.
//
// A new sql transaction is created and committed per Benchmark in benchSet. This way if a
// db failure occurs all data from the benchSet is not lost.
func (benchdb *BenchPSQL) WriteSet(benchSet parse.Set) (int, error) {
	sqlDB, err := sql.Open(benchdb.Driver, benchdb.ConnStr)
	if err != nil {
		return 0, fmt.Errorf("could not connect to db: %v", err)
	}
	defer sqlDB.Close()
	benchdb.dbConn = sqlDB

	batchId, err := uuid()
	if err != nil {
		return 0, fmt.Errorf("could not generate batch id: %v\n", err)
	}

	cnt := 0
	for _, b := range benchSet {
		n := len(b)
		for i := 0; i < n; i++ {
			val := b[i]
			err := saveBenchmark(benchdb.dbConn, benchdb.TableName, batchId, *val)
			if err != nil {
				return 0, fmt.Errorf("failed to save benchmark: %v", err)
			}
			cnt++
		}
	}
	return cnt, nil
}

func saveBenchmark(dbConn *sql.DB, table, batchId string, b parse.Benchmark) error {
	// Create a transaction per Benchmark.
	tx, err := dbConn.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	sha, err := latestGitSha()
	if err != nil {
		return err
	}

	q := fmt.Sprintf(`
        INSERT INTO %s
        (batch_id, latest_sha, datetime, name, n, ns_op, allocated_bytes_op, allocs_op)
        VALUES
        ($1, $2, $3, $4, $5, $6, $7, $8)
        `, table)

	// Strips leading Benchmark string in Benchmark.Name
	name := strings.TrimPrefix(strings.TrimSpace(b.Name), "Benchmark")
	ts := time.Now().UTC()

	_, err = tx.Exec(q,
		batchId, sha, ts, name, b.N, b.NsPerOp, b.AllocedBytesPerOp, b.AllocsPerOp)
	if err != nil {
		return err
	}
	return tx.Commit()
}
<|endoftext|>


<commit_before>package xpi

import (
	"archive/zip"
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"github.com/pkg/errors"
	"io"
	"io/ioutil"
	"strings"
)

func makePKCS7Manifest(input []byte, metafiles []Metafile) (manifest []byte, err error) {
	for _, f := range metafiles {
		if !f.IsNameValid() {
			err = errors.Errorf("Cannot pack metafile with invalid path %s", f.Name)
			return
		}
	}

	manifest, err = makeJARManifest(input)
	if err != nil {
		return
	}

	mw := bytes.NewBuffer(manifest)
	for _, f := range metafiles {
		fmt.Fprintf(mw, "Name: %s\nDigest-Algorithms: SHA1 SHA256\n", f.Name)
		h1 := sha1.New()
		h1.Write(f.Body)
		fmt.Fprintf(mw, "SHA1-Digest: %s\n", base64.StdEncoding.EncodeToString(h1.Sum(nil)))
		h2 := sha256.New()
		h2.Write(f.Body)
		fmt.Fprintf(mw, "SHA256-Digest: %s\n\n", base64.StdEncoding.EncodeToString(h2.Sum(nil)))
	}

	return mw.Bytes(), err
}

// makeJARManifestAndSignatureFile writes hashes for all entries in a zip to a
// manifest file then hashes the manifest file to write a signature
// file and returns both
func makeJARManifestAndSignatureFile(input []byte) (manifest, sigfile []byte, err error) {
	manifest, err = makeJARManifest(input)
	if err != nil {
		return
	}

	sigfile, err = makeJARSignatureFile(manifest)
	if err != nil {
		return
	}

	return
}

// makeJARManifest calculates a sha1 and sha256 hash for each zip entry and writes them to a manifest file
func makeJARManifest(input []byte) (manifest []byte, err error) {
	inputReader := bytes.NewReader(input)
	r, err := zip.NewReader(inputReader, int64(len(input)))
	if err != nil {
		return
	}

	// generate the manifest file by calculating an MD5, sha1, and sha256 hash for each zip entry
	mw := bytes.NewBuffer(manifest)
	manifest = []byte(fmt.Sprintf("Manifest-Version: 1.0\n\n"))

	for _, f := range r.File {
		if isJARSignatureFile(f.Name) {
			// reserved signature files do not get included in the manifest
			continue
		}
		if f.FileInfo().IsDir() {
			// directories do not get included
			continue
		}
		rc, err := f.Open()
		if err != nil {
			return manifest, err
		}
		data, err := ioutil.ReadAll(rc)
		if err != nil {
			return manifest, err
		}
		fmt.Fprintf(mw, "Name: %s\nDigest-Algorithms: MD5 SHA1 SHA256\n", f.Name)
		h0 := md5.New()
		h0.Write(data)
		fmt.Fprintf(mw, "MD5-Digest: %s\n", base64.StdEncoding.EncodeToString(h0.Sum(nil)))
		h1 := sha1.New()
		h1.Write(data)
		fmt.Fprintf(mw, "SHA1-Digest: %s\n", base64.StdEncoding.EncodeToString(h1.Sum(nil)))
		h2 := sha256.New()
		h2.Write(data)
		fmt.Fprintf(mw, "SHA256-Digest: %s\n\n", base64.StdEncoding.EncodeToString(h2.Sum(nil)))
	}
	manifestBody := mw.Bytes()
	manifest = append(manifest, manifestBody...)

	return
}

// makeJARSignatureFile calculates a signature file by hashing the manifest with sha1 and sha256
func makeJARSignatureFile(manifest []byte) (sigfile []byte, err error) {
	sw := bytes.NewBuffer(sigfile)
	fmt.Fprint(sw, "Signature-Version: 1.0\n")
	h0 := md5.New()
	h0.Write(manifest)
	fmt.Fprintf(sw, "MD5-Digest-Manifest: %s\n", base64.StdEncoding.EncodeToString(h0.Sum(nil)))
	h1 := sha1.New()
	h1.Write(manifest)
	fmt.Fprintf(sw, "SHA1-Digest-Manifest: %s\n", base64.StdEncoding.EncodeToString(h1.Sum(nil)))
	h2 := sha256.New()
	h2.Write(manifest)
	fmt.Fprintf(sw, "SHA256-Digest-Manifest: %s\n\n", base64.StdEncoding.EncodeToString(h2.Sum(nil)))
	sigfile = sw.Bytes()

	return
}

// Metafile is a file to pack into a JAR at .Name with contents .Body
type Metafile struct {
	Name string
	Body []byte
}

// IsNameValid checks whether a Metafile.Name is non-nil and begins
// with "META-INF/" functions taking Metafile args should validate
// names before reading or writing them to JARs
func (m *Metafile) IsNameValid() bool {
	return m != nil && strings.HasPrefix(m.Name, "META-INF/")
}

// repackJARWithMetafiles inserts metafiles in the input JAR file and returns a JAR ZIP archive
func repackJARWithMetafiles(input []byte, metafiles []Metafile) (output []byte, err error) {
	for _, f := range metafiles {
		if !f.IsNameValid() {
			err = errors.Errorf("Cannot pack metafile with invalid path %s", f.Name)
			return
		}
	}
	var (
		rc     io.ReadCloser
		fwhead *zip.FileHeader
		fw     io.Writer
		data   []byte
	)
	inputReader := bytes.NewReader(input)
	r, err := zip.NewReader(inputReader, int64(len(input)))
	if err != nil {
		return
	}
	// Create a buffer to write our archive to.
	buf := new(bytes.Buffer)

	// Create a new zip archive.
	w := zip.NewWriter(buf)

	// Iterate through the files in the archive,
	for _, f := range r.File {
		// skip signature files, we have new ones we'll add at the end
		if isJARSignatureFile(f.Name) {
			continue
		}
		rc, err = f.Open()
		if err != nil {
			return
		}
		fwhead := &zip.FileHeader{
			Name:   f.Name,
			Method: zip.Deflate,
		}
		// insert the file into the archive
		fw, err = w.CreateHeader(fwhead)
		if err != nil {
			return
		}
		data, err = ioutil.ReadAll(rc)
		if err != nil {
			return
		}
		_, err = fw.Write(data)
		if err != nil {
			return
		}
		rc.Close()
	}
	// insert the signature files. Those will be compressed
	// so we don't have to worry about their alignment
	for _, meta := range metafiles {
		fwhead = &zip.FileHeader{
			Name:   meta.Name,
			Method: zip.Deflate,
		}
		fw, err = w.CreateHeader(fwhead)
		if err != nil {
			return
		}
		_, err = fw.Write(meta.Body)
		if err != nil {
			return
		}
	}
	// Make sure to check the error on Close.
	err = w.Close()
	if err != nil {
		return
	}

	output = buf.Bytes()
	return
}

// repackJAR inserts the manifest, signature file and pkcs7 signature in the input JAR file,
// and return a JAR ZIP archive
func repackJAR(input, manifest, sigfile, signature []byte) (output []byte, err error) {
	var metas = []Metafile{
		{pkcs7ManifestPath, manifest},
		{pkcs7SignatureFilePath, sigfile},
		{pkcs7SigPath, signature},
	}
	return repackJARWithMetafiles(input, metas)
}

// The JAR format defines a number of signature files stored under the META-INF directory
// META-INF/MANIFEST.MF
// META-INF/*.SF
// META-INF/*.DSA
// META-INF/*.RSA
// META-INF/SIG-*
// and their lowercase variants
// https://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Signed_JAR_File
func isJARSignatureFile(name string) bool {
	if strings.HasPrefix(name, "META-INF/") {
		name = strings.TrimPrefix(name, "META-INF/")
		if name == "MANIFEST.MF" || name == "manifest.mf" ||
			strings.HasSuffix(name, ".SF") || strings.HasSuffix(name, ".sf") ||
			strings.HasSuffix(name, ".RSA") || strings.HasSuffix(name, ".rsa") ||
			strings.HasSuffix(name, ".DSA") || strings.HasSuffix(name, ".dsa") ||
			strings.HasPrefix(name, "SIG-") || strings.HasPrefix(name, "sig-") {
			return true
		}
	}
	return false
}

// readFileFromZIP reads a given filename out of a ZIP and returns it or an error
func readFileFromZIP(signedXPI []byte, filename string) ([]byte, error) {
	zipReader := bytes.NewReader(signedXPI)
	r, err := zip.NewReader(zipReader, int64(len(signedXPI)))
	if err != nil {
		return nil, errors.Wrap(err, "Error reading ZIP")
	}

	for _, f := range r.File {
		if f.Name == filename {
			rc, err := f.Open()
			defer rc.Close()
			if err != nil {
				return nil, errors.Wrapf(err, "Error opening file %s in ZIP", filename)
			}
			data, err := ioutil.ReadAll(rc)
			if err != nil {
				return nil, errors.Wrapf(err, "Error reading file %s in ZIP", filename)
			}
			return data, nil
		}
	}
	return nil, errors.Errorf("failed to find %s in ZIP", filename)
}
<commit_msg>signers: Add missing MD5 digest to xpi metafiles<commit_after>package xpi

import (
	"archive/zip"
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"github.com/pkg/errors"
	"io"
	"io/ioutil"
	"strings"
)

func makePKCS7Manifest(input []byte, metafiles []Metafile) (manifest []byte, err error) {
	for _, f := range metafiles {
		if !f.IsNameValid() {
			err = errors.Errorf("Cannot pack metafile with invalid path %s", f.Name)
			return
		}
	}

	manifest, err = makeJARManifest(input)
	if err != nil {
		return
	}

	mw := bytes.NewBuffer(manifest)
	for _, f := range metafiles {
		fmt.Fprintf(mw, "Name: %s\nDigest-Algorithms: MD5 SHA1 SHA256\n", f.Name)
		h0 := md5.New()
		h0.Write(f.Body)
		fmt.Fprintf(mw, "MD5-Digest: %s\n", base64.StdEncoding.EncodeToString(h0.Sum(nil)))
		h1 := sha1.New()
		h1.Write(f.Body)
		fmt.Fprintf(mw, "SHA1-Digest: %s\n", base64.StdEncoding.EncodeToString(h1.Sum(nil)))
		h2 := sha256.New()
		h2.Write(f.Body)
		fmt.Fprintf(mw, "SHA256-Digest: %s\n\n", base64.StdEncoding.EncodeToString(h2.Sum(nil)))
	}

	return mw.Bytes(), err
}

// makeJARManifestAndSignatureFile writes hashes for all entries in a zip to a
// manifest file then hashes the manifest file to write a signature
// file and returns both
func makeJARManifestAndSignatureFile(input []byte) (manifest, sigfile []byte, err error) {
	manifest, err = makeJARManifest(input)
	if err != nil {
		return
	}

	sigfile, err = makeJARSignatureFile(manifest)
	if err != nil {
		return
	}

	return
}

// makeJARManifest calculates a sha1 and sha256 hash for each zip entry and writes them to a manifest file
func makeJARManifest(input []byte) (manifest []byte, err error) {
	inputReader := bytes.NewReader(input)
	r, err := zip.NewReader(inputReader, int64(len(input)))
	if err != nil {
		return
	}

	// generate the manifest file by calculating an MD5, sha1, and sha256 hash for each zip entry
	mw := bytes.NewBuffer(manifest)
	manifest = []byte(fmt.Sprintf("Manifest-Version: 1.0\n\n"))

	for _, f := range r.File {
		if isJARSignatureFile(f.Name) {
			// reserved signature files do not get included in the manifest
			continue
		}
		if f.FileInfo().IsDir() {
			// directories do not get included
			continue
		}
		rc, err := f.Open()
		if err != nil {
			return manifest, err
		}
		data, err := ioutil.ReadAll(rc)
		if err != nil {
			return manifest, err
		}
		fmt.Fprintf(mw, "Name: %s\nDigest-Algorithms: MD5 SHA1 SHA256\n", f.Name)
		h0 := md5.New()
		h0.Write(data)
		fmt.Fprintf(mw, "MD5-Digest: %s\n", base64.StdEncoding.EncodeToString(h0.Sum(nil)))
		h1 := sha1.New()
		h1.Write(data)
		fmt.Fprintf(mw, "SHA1-Digest: %s\n", base64.StdEncoding.EncodeToString(h1.Sum(nil)))
		h2 := sha256.New()
		h2.Write(data)
		fmt.Fprintf(mw, "SHA256-Digest: %s\n\n", base64.StdEncoding.EncodeToString(h2.Sum(nil)))
	}
	manifestBody := mw.Bytes()
	manifest = append(manifest, manifestBody...)

	return
}

// makeJARSignatureFile calculates a signature file by hashing the manifest with sha1 and sha256
func makeJARSignatureFile(manifest []byte) (sigfile []byte, err error) {
	sw := bytes.NewBuffer(sigfile)
	fmt.Fprint(sw, "Signature-Version: 1.0\n")
	h0 := md5.New()
	h0.Write(manifest)
	fmt.Fprintf(sw, "MD5-Digest-Manifest: %s\n", base64.StdEncoding.EncodeToString(h0.Sum(nil)))
	h1 := sha1.New()
	h1.Write(manifest)
	fmt.Fprintf(sw, "SHA1-Digest-Manifest: %s\n", base64.StdEncoding.EncodeToString(h1.Sum(nil)))
	h2 := sha256.New()
	h2.Write(manifest)
	fmt.Fprintf(sw, "SHA256-Digest-Manifest: %s\n\n", base64.StdEncoding.EncodeToString(h2.Sum(nil)))
	sigfile = sw.Bytes()

	return
}

// Metafile is a file to pack into a JAR at .Name with contents .Body
type Metafile struct {
	Name string
	Body []byte
}

// IsNameValid checks whether a Metafile.Name is non-nil and begins
// with "META-INF/" functions taking Metafile args should validate
// names before reading or writing them to JARs
func (m *Metafile) IsNameValid() bool {
	return m != nil && strings.HasPrefix(m.Name, "META-INF/")
}

// repackJARWithMetafiles inserts metafiles in the input JAR file and returns a JAR ZIP archive
func repackJARWithMetafiles(input []byte, metafiles []Metafile) (output []byte, err error) {
	for _, f := range metafiles {
		if !f.IsNameValid() {
			err = errors.Errorf("Cannot pack metafile with invalid path %s", f.Name)
			return
		}
	}
	var (
		rc     io.ReadCloser
		fwhead *zip.FileHeader
		fw     io.Writer
		data   []byte
	)
	inputReader := bytes.NewReader(input)
	r, err := zip.NewReader(inputReader, int64(len(input)))
	if err != nil {
		return
	}
	// Create a buffer to write our archive to.
	buf := new(bytes.Buffer)

	// Create a new zip archive.
	w := zip.NewWriter(buf)

	// Iterate through the files in the archive,
	for _, f := range r.File {
		// skip signature files, we have new ones we'll add at the end
		if isJARSignatureFile(f.Name) {
			continue
		}
		rc, err = f.Open()
		if err != nil {
			return
		}
		fwhead := &zip.FileHeader{
			Name:   f.Name,
			Method: zip.Deflate,
		}
		// insert the file into the archive
		fw, err = w.CreateHeader(fwhead)
		if err != nil {
			return
		}
		data, err = ioutil.ReadAll(rc)
		if err != nil {
			return
		}
		_, err = fw.Write(data)
		if err != nil {
			return
		}
		rc.Close()
	}
	// insert the signature files. Those will be compressed
	// so we don't have to worry about their alignment
	for _, meta := range metafiles {
		fwhead = &zip.FileHeader{
			Name:   meta.Name,
			Method: zip.Deflate,
		}
		fw, err = w.CreateHeader(fwhead)
		if err != nil {
			return
		}
		_, err = fw.Write(meta.Body)
		if err != nil {
			return
		}
	}
	// Make sure to check the error on Close.
	err = w.Close()
	if err != nil {
		return
	}

	output = buf.Bytes()
	return
}

// repackJAR inserts the manifest, signature file and pkcs7 signature in the input JAR file,
// and return a JAR ZIP archive
func repackJAR(input, manifest, sigfile, signature []byte) (output []byte, err error) {
	var metas = []Metafile{
		{pkcs7ManifestPath, manifest},
		{pkcs7SignatureFilePath, sigfile},
		{pkcs7SigPath, signature},
	}
	return repackJARWithMetafiles(input, metas)
}

// The JAR format defines a number of signature files stored under the META-INF directory
// META-INF/MANIFEST.MF
// META-INF/*.SF
// META-INF/*.DSA
// META-INF/*.RSA
// META-INF/SIG-*
// and their lowercase variants
// https://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Signed_JAR_File
func isJARSignatureFile(name string) bool {
	if strings.HasPrefix(name, "META-INF/") {
		name = strings.TrimPrefix(name, "META-INF/")
		if name == "MANIFEST.MF" || name == "manifest.mf" ||
			strings.HasSuffix(name, ".SF") || strings.HasSuffix(name, ".sf") ||
			strings.HasSuffix(name, ".RSA") || strings.HasSuffix(name, ".rsa") ||
			strings.HasSuffix(name, ".DSA") || strings.HasSuffix(name, ".dsa") ||
			strings.HasPrefix(name, "SIG-") || strings.HasPrefix(name, "sig-") {
			return true
		}
	}
	return false
}

// readFileFromZIP reads a given filename out of a ZIP and returns it or an error
func readFileFromZIP(signedXPI []byte, filename string) ([]byte, error) {
	zipReader := bytes.NewReader(signedXPI)
	r, err := zip.NewReader(zipReader, int64(len(signedXPI)))
	if err != nil {
		return nil, errors.Wrap(err, "Error reading ZIP")
	}

	for _, f := range r.File {
		if f.Name == filename {
			rc, err := f.Open()
			defer rc.Close()
			if err != nil {
				return nil, errors.Wrapf(err, "Error opening file %s in ZIP", filename)
			}
			data, err := ioutil.ReadAll(rc)
			if err != nil {
				return nil, errors.Wrapf(err, "Error reading file %s in ZIP", filename)
			}
			return data, nil
		}
	}
	return nil, errors.Errorf("failed to find %s in ZIP", filename)
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package policy

import (
	"github.com/golang/protobuf/proto"
	"github.com/hyperledger/fabric/common/cauthdsl"
	"github.com/hyperledger/fabric/common/policies"
	"github.com/hyperledger/fabric/msp"
	"github.com/hyperledger/fabric/protos/common"
	"github.com/hyperledger/fabric/protos/peer"
	"github.com/hyperledger/fabric/protoutil"
	"github.com/pkg/errors"
)

// Provider provides the backing implementation of a policy
type SignaturePolicyProvider interface {
	// NewPolicy creates a new policy based on the policy bytes
	NewPolicy(signaturePolicy *common.SignaturePolicyEnvelope) (policies.Policy, error)
}

// ChannelPolicyReference is used to determine if a set of signature is valid and complies with a policy
type ChannelPolicyReferenceProvider interface {
	// NewPolicy creates a new policy based on the policy bytes
	NewPolicy(channelConfigPolicyReference string) (policies.Policy, error)
}

//go:generate mockery -dir ../../common/policies/ -name Policy -case underscore -output mocks/
//go:generate mockery -dir ../../common/policies/ -name ChannelPolicyManagerGetter -case underscore -output mocks/
//go:generate mockery -dir ../../common/policies/ -name Manager -case underscore -output mocks/
//go:generate mockery -dir . -name ChannelPolicyReferenceProvider -case underscore -output mocks/
//go:generate mockery -dir . -name SignaturePolicyProvider -case underscore -output mocks/

type ApplicationPolicyEvaluator struct {
	signaturePolicyProvider        SignaturePolicyProvider
	channelPolicyReferenceProvider ChannelPolicyReferenceProvider
}

type ChannelPolicyReferenceProviderImpl struct {
	policies.Manager
}

func (c *ChannelPolicyReferenceProviderImpl) NewPolicy(channelConfigPolicyReference string) (policies.Policy, error) {
	p, ok := c.GetPolicy(channelConfigPolicyReference)
	if !ok {
		return nil, errors.Errorf("failed to retrieve policy for reference %s", channelConfigPolicyReference)
	}

	return p, nil
}

// dynamicPolicyManager implements a policy manager that
// always acts on the latest config for this channel
type dynamicPolicyManager struct {
	channelPolicyManagerGetter policies.ChannelPolicyManagerGetter
	channelID                  string
}

func (d *dynamicPolicyManager) GetPolicy(id string) (policies.Policy, bool) {
	mgr, ok := d.channelPolicyManagerGetter.Manager(d.channelID)
	if !ok {
		// this will never happen - if we are here we
		// managed to retrieve the policy manager for
		// this channel once, and so by the way the
		// channel config is managed, we cannot fail.
		panic("programming error")
	}

	return mgr.GetPolicy(id)
}

func (d *dynamicPolicyManager) Manager(path []string) (policies.Manager, bool) {
	// we don't use this function
	panic("programming error")
}

// New returns an evaluator for application policies
func New(deserializer msp.IdentityDeserializer, channel string, channelPolicyManagerGetter policies.ChannelPolicyManagerGetter) (*ApplicationPolicyEvaluator, error) {
	_, ok := channelPolicyManagerGetter.Manager(channel)
	if !ok {
		return nil, errors.Errorf("failed to retrieve policy manager for channel %s", channel)
	}

	return &ApplicationPolicyEvaluator{
		signaturePolicyProvider: &cauthdsl.ProviderFromStruct{Deserializer: deserializer},
		channelPolicyReferenceProvider: &ChannelPolicyReferenceProviderImpl{Manager: &dynamicPolicyManager{
			channelID:                  channel,
			channelPolicyManagerGetter: channelPolicyManagerGetter,
		}},
	}, nil
}

func (a *ApplicationPolicyEvaluator) evaluateSignaturePolicy(signaturePolicy *common.SignaturePolicyEnvelope, signatureSet []*protoutil.SignedData) error {
	p, err := a.signaturePolicyProvider.NewPolicy(signaturePolicy)
	if err != nil {
		return errors.WithMessage(err, "could not create evaluator for signature policy")
	}

	return p.Evaluate(signatureSet)
}

func (a *ApplicationPolicyEvaluator) evaluateChannelConfigPolicyReference(channelConfigPolicyReference string, signatureSet []*protoutil.SignedData) error {
	p, err := a.channelPolicyReferenceProvider.NewPolicy(channelConfigPolicyReference)
	if err != nil {
		return errors.WithMessage(err, "could not create evaluator for channel reference policy")
	}

	return p.Evaluate(signatureSet)
}

func (a *ApplicationPolicyEvaluator) Evaluate(policyBytes []byte, signatureSet []*protoutil.SignedData) error {
	p := &peer.ApplicationPolicy{}
	err := proto.Unmarshal(policyBytes, p)
	if err != nil {
		return errors.Wrap(err, "failed to unmarshal ApplicationPolicy bytes")
	}

	switch policy := p.Type.(type) {
	case *peer.ApplicationPolicy_SignaturePolicy:
		return a.evaluateSignaturePolicy(policy.SignaturePolicy, signatureSet)
	case *peer.ApplicationPolicy_ChannelConfigPolicyReference:
		return a.evaluateChannelConfigPolicyReference(policy.ChannelConfigPolicyReference, signatureSet)
	default:
		return errors.Errorf("unsupported policy type %T", policy)
	}
}
<commit_msg>[FAB-14718] Remove unnecessary function from interface<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package policy

import (
	"github.com/golang/protobuf/proto"
	"github.com/hyperledger/fabric/common/cauthdsl"
	"github.com/hyperledger/fabric/common/policies"
	"github.com/hyperledger/fabric/msp"
	"github.com/hyperledger/fabric/protos/common"
	"github.com/hyperledger/fabric/protos/peer"
	"github.com/hyperledger/fabric/protoutil"
	"github.com/pkg/errors"
)

// Provider provides the backing implementation of a policy
type SignaturePolicyProvider interface {
	// NewPolicy creates a new policy based on the policy bytes
	NewPolicy(signaturePolicy *common.SignaturePolicyEnvelope) (policies.Policy, error)
}

// ChannelPolicyReference is used to determine if a set of signature is valid and complies with a policy
type ChannelPolicyReferenceProvider interface {
	// NewPolicy creates a new policy based on the policy bytes
	NewPolicy(channelConfigPolicyReference string) (policies.Policy, error)
}

//go:generate mockery -dir ../../common/policies/ -name Policy -case underscore -output mocks/
//go:generate mockery -dir ../../common/policies/ -name ChannelPolicyManagerGetter -case underscore -output mocks/
//go:generate mockery -dir ../../common/policies/ -name Manager -case underscore -output mocks/
//go:generate mockery -dir . -name ChannelPolicyReferenceProvider -case underscore -output mocks/
//go:generate mockery -dir . -name SignaturePolicyProvider -case underscore -output mocks/

type ApplicationPolicyEvaluator struct {
	signaturePolicyProvider        SignaturePolicyProvider
	channelPolicyReferenceProvider ChannelPolicyReferenceProvider
}

// Manager defines functions to interface with the policy manager of a channel
type Manager interface {
	// GetPolicy returns a policy and true if it was the policy requested, or false if it is the default policy
	GetPolicy(id string) (policies.Policy, bool)
}

type ChannelPolicyReferenceProviderImpl struct {
	Manager
}

func (c *ChannelPolicyReferenceProviderImpl) NewPolicy(channelConfigPolicyReference string) (policies.Policy, error) {
	p, ok := c.GetPolicy(channelConfigPolicyReference)
	if !ok {
		return nil, errors.Errorf("failed to retrieve policy for reference %s", channelConfigPolicyReference)
	}

	return p, nil
}

// dynamicPolicyManager implements a policy manager that
// always acts on the latest config for this channel
type dynamicPolicyManager struct {
	channelPolicyManagerGetter policies.ChannelPolicyManagerGetter
	channelID                  string
}

func (d *dynamicPolicyManager) GetPolicy(id string) (policies.Policy, bool) {
	mgr, ok := d.channelPolicyManagerGetter.Manager(d.channelID)
	if !ok {
		// this will never happen - if we are here we
		// managed to retrieve the policy manager for
		// this channel once, and so by the way the
		// channel config is managed, we cannot fail.
		panic("programming error")
	}

	return mgr.GetPolicy(id)
}

// New returns an evaluator for application policies
func New(deserializer msp.IdentityDeserializer, channel string, channelPolicyManagerGetter policies.ChannelPolicyManagerGetter) (*ApplicationPolicyEvaluator, error) {
	_, ok := channelPolicyManagerGetter.Manager(channel)
	if !ok {
		return nil, errors.Errorf("failed to retrieve policy manager for channel %s", channel)
	}

	return &ApplicationPolicyEvaluator{
		signaturePolicyProvider: &cauthdsl.ProviderFromStruct{Deserializer: deserializer},
		channelPolicyReferenceProvider: &ChannelPolicyReferenceProviderImpl{Manager: &dynamicPolicyManager{
			channelID:                  channel,
			channelPolicyManagerGetter: channelPolicyManagerGetter,
		}},
	}, nil
}

func (a *ApplicationPolicyEvaluator) evaluateSignaturePolicy(signaturePolicy *common.SignaturePolicyEnvelope, signatureSet []*protoutil.SignedData) error {
	p, err := a.signaturePolicyProvider.NewPolicy(signaturePolicy)
	if err != nil {
		return errors.WithMessage(err, "could not create evaluator for signature policy")
	}

	return p.Evaluate(signatureSet)
}

func (a *ApplicationPolicyEvaluator) evaluateChannelConfigPolicyReference(channelConfigPolicyReference string, signatureSet []*protoutil.SignedData) error {
	p, err := a.channelPolicyReferenceProvider.NewPolicy(channelConfigPolicyReference)
	if err != nil {
		return errors.WithMessage(err, "could not create evaluator for channel reference policy")
	}

	return p.Evaluate(signatureSet)
}

func (a *ApplicationPolicyEvaluator) Evaluate(policyBytes []byte, signatureSet []*protoutil.SignedData) error {
	p := &peer.ApplicationPolicy{}
	err := proto.Unmarshal(policyBytes, p)
	if err != nil {
		return errors.Wrap(err, "failed to unmarshal ApplicationPolicy bytes")
	}

	switch policy := p.Type.(type) {
	case *peer.ApplicationPolicy_SignaturePolicy:
		return a.evaluateSignaturePolicy(policy.SignaturePolicy, signatureSet)
	case *peer.ApplicationPolicy_ChannelConfigPolicyReference:
		return a.evaluateChannelConfigPolicyReference(policy.ChannelConfigPolicyReference, signatureSet)
	default:
		return errors.Errorf("unsupported policy type %T", policy)
	}
}
<|endoftext|>


<commit_before>package logging

import "io"
import "os"
import "fmt"
import "strings"
import "time"
import "bytes"
import "strconv"
import "net/http"
import "io/ioutil"
import "runtime"
import "runtime/debug"
import l "log"
import rx "regexp"

// Log levels
type LogLevel int16

const (
	Silent LogLevel = iota
	Fatal
	Error
	Warn
	Info
	Timing
	Debug
	Trace
)

// Logger interface
type Logger interface {
	// Warnings, logged by default.
	Warnf(format string, v ...interface{})
	// Errors, logged by default.
	Errorf(format string, v ...interface{})
	// Fatal errors. Will not terminate execution.
	Fatalf(format string, v ...interface{})
	// Informational messages.
	Infof(format string, v ...interface{})
	// Get stack trace
	StackTrace() string
	// Timing utility
	Timer(format string, v ...interface{}) Ender
	// Debugging messages
	Debugf(format string, v ...interface{})
	// Program execution
	Tracef(format string, v ...interface{})
	// Call and print the stringer if debugging enabled
	LazyDebug(fn func() string)
	// Call and print the stringer if tracing enabled
	LazyTrace(fn func() string)
}

// Timer interface
type Ender interface {
	// Stop and log timing
	End()
}

//
// Implementation
//

type overrideLoc struct {
	matcher *rx.Regexp
	level   LogLevel
}

// Messages administrator should eventually see.
func (t LogLevel) String() string {
	switch t {
	case Silent:
		return "Silent"
	case Fatal:
		return "Fatal"
	case Error:
		return "Error"
	case Warn:
		return "Warn"
	case Info:
		return "Info"
	case Timing:
		return "Timing"
	case Debug:
		return "Debug"
	case Trace:
		return "Trace"
	default:
		return "Info"
	}
}

func Level(s string) LogLevel {
	switch strings.ToUpper(s) {
	case "SILENT":
		return Silent
	case "FATAL":
		return Fatal
	case "ERROR":
		return Error
	case "WARN":
		return Warn
	case "INFO":
		return Info
	case "TIMING":
		return Timing
	case "DEBUG":
		return Debug
	case "TRACE":
		return Trace
	default:
		return Info
	}
}

type destination struct {
	baselevel LogLevel
	target    *l.Logger
	overrides *[]overrideLoc
}

type stopClock struct {
	comment string
	skip    int
	start   time.Time
	log     *destination
}

func (log *destination) Warnf(format string, v ...interface{}) {
	log.printf(Warn, 1, format, v...)
}

// Errors that caused problems in execution logic.
func (log *destination) Errorf(format string, v ...interface{}) {
	log.printf(Error, 1, format, v...)
}

// Fatal messages are to be logged prior to exiting due to errors.
func (log *destination) Fatalf(format string, v ...interface{}) {
	log.printf(Fatal, 1, format, v...)
}

// Info messages are those that are logged but not expected to be read.
func (log *destination) Infof(format string, v ...interface{}) {
	log.printf(Info, 1, format, v...)
}

// Function timing. Use as:
//    defer Time("Waiting for backfill").End()
//     ... function to be timed
// or
//    timer := Timer("For vbucket %d mutation %d", vbid, seq)
//     ... lines to be timed
//    timer.End()
//
func (log *destination) Timer(format string, v ...interface{}) Ender {
	return log.timer(1, format, v...)
}

// Debug messages to help analyze problem. Default off.
func (log *destination) Debugf(format string, v ...interface{}) {
	log.printf(Debug, 1, format, v...)
}

// Execution trace showing the program flow. Default off.
func (log *destination) Tracef(format string, v ...interface{}) {
	log.printf(Trace, 1, format, v...)
}

// Set the base log level
func (log *destination) SetLogLevel(to LogLevel) {
	log.baselevel = to
}

// Get stack trace
func (log *destination) StackTrace() string {
	return log.getStackTrace(2, debug.Stack())
}

// Get profiling info
func Profile(port string, endpoints ...string) func() string {
	if strings.HasPrefix(port, ":") {
		port = port[1:]
	}
	return func() string {
		var buf bytes.Buffer
		for _, endpoint := range endpoints {
			addr := fmt.Sprintf("http://localhost:%s/debug/pprof/%s?debug=1", port, endpoint)
			resp, err := http.Get(addr)
			if err != nil {
				continue
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				continue
			}
			buf.Write(body)
		}
		return buf.String()
	}
}

// Dump profiling info periodically
func PeriodicProfile(port string, endpoints ...string) {
	profiler := Profile(port, endpoints...)
	tick := time.NewTicker(60 * time.Second)
	go func() {
		for {
			select {
			case <-tick.C:
				LazyDebug(profiler)
			}
		}
	}()
}

// Run function only if output will be logged at debug level
func (log *destination) LazyDebug(fn func() string) {
	if log.isEnabled(Debug, 1) {
		log.printf(Debug, 1, "%s", fn())
	}
}

// Run function only if output will be logged at trace level
func (log *destination) LazyTrace(fn func() string) {
	if log.isEnabled(Trace, 1) {
		log.printf(Trace, 1, "%s", fn())
	}
}

// Add logging override. Format: filename[:line]=Level[,...]
func (log *destination) AddOverride(line string) {
	// infrequent, so clone to avoid locks
	added := make([]overrideLoc, 0, 16)
	if log.overrides != nil {
		added = append(added, *log.overrides...)
	}
	specs := strings.Split(line, ",")
	for _, spec := range specs {
		kv := strings.Split(spec, "=")
		if len(kv) != 2 {
			continue
		}
		exp, slvl := kv[0], kv[1]
		exp = strings.Replace(exp, `*`, `.*`, -1)
		if !strings.Contains(exp, ":") {
			exp += `:[\d]+`
		}
		exp = exp + "$"
		level := Level(slvl)
		re, err := rx.Compile(exp)
		if err == nil {
			entry := overrideLoc{matcher: re, level: level}
			added = append(added, entry)
		}
	}
	log.overrides = &added
}

// Clear all overrides
func (log *destination) ClearOverrides() {
	log.overrides = nil
}

// Stop the running timer and print timing
func (watch *stopClock) End() {
	elapsed := time.Since(watch.start).Nanoseconds()
	watch.log.printf(Timing, watch.skip, "%.1f μs - %s", float64(elapsed)/1000, watch.comment)
}

// Check if enabled
func (log *destination) isEnabled(at LogLevel, skip int) bool {
	// normal production case
	if log.overrides == nil {
		return log.baselevel >= at
	}

	// unusual case, perhaps troubleshooting
	_, file, line, _ := runtime.Caller(skip + 1)
	loc := file + ":" + strconv.Itoa(line)
	for _, spec := range *log.overrides {
		if spec.matcher.MatchString(loc) {
			return spec.level >= at
		}
	}

	return log.baselevel >= at
}

func (log *destination) printf(at LogLevel, skip int, format string, v ...interface{}) {
	if log.isEnabled(at, skip+1) {
		log.target.Printf("["+at.String()+"] "+format, v...)
	}
}

func (log *destination) getStackTrace(skip int, stack []byte) string {
	var buf bytes.Buffer
	lines := strings.Split(string(stack), "\n")
	for _, call := range lines[skip*2:] {
		buf.WriteString(fmt.Sprintf("%s\n", call))
	}
	return buf.String()
}

func (log *destination) timer(skip int, format string, v ...interface{}) Ender {
	if !log.isEnabled(Timing, skip) {
		return emptyclock
	}
	comment := fmt.Sprintf(format, v...)
	return &stopClock{comment: comment, skip: skip, start: time.Now(), log: log}
}

// No op clock
var emptyclock = &emptyClock{}

type emptyClock struct{}

func (_ *emptyClock) End() {
}

// The default logger
var SystemLogger destination

func init() {
	dest := l.New(os.Stdout, "", l.Lmicroseconds)
	SystemLogger = destination{baselevel: Info, target: dest, overrides: nil}
}

// SetLogWriter sets a new default destination
func SetLogWriter(w io.Writer) {
	dest := l.New(w, "", l.Lmicroseconds)
	SystemLogger = destination{baselevel: Info, target: dest, overrides: nil}
}

//
// A set of convenience methods to log to default logger
// See correspond methods on destination for details
//
func Warnf(format string, v ...interface{}) {
	SystemLogger.printf(Warn, 1, format, v...)
}

// Errorf to log message and warning messages will be logged.
func Errorf(format string, v ...interface{}) {
	SystemLogger.printf(Error, 1, format, v...)
}

// Fatalf to log message and warning messages will be logged.
func Fatalf(format string, v ...interface{}) {
	SystemLogger.printf(Fatal, 1, format, v...)
}

// Infof to log message at info level.
func Infof(format string, v ...interface{}) {
	SystemLogger.printf(Info, 1, format, v...)
}

// Debugf to log message at info level.
func Debugf(format string, v ...interface{}) {
	SystemLogger.printf(Debug, 1, format, v...)
}

// Tracef to log message at info level.
func Tracef(format string, v ...interface{}) {
	SystemLogger.printf(Trace, 1, format, v...)
}

// StackTrace prints current stack at specified log level
func StackTrace() string {
	return SystemLogger.getStackTrace(2, debug.Stack())
}

// Timing utility function
func Timer(format string, v ...interface{}) Ender {
	return SystemLogger.timer(2, format, v...)
}

// Set the base log level
func SetLogLevel(to LogLevel) {
	SystemLogger.SetLogLevel(to)
}

// Add logging override. Format: filename[:line]=Level[,...]
func AddOverride(line string) {
	SystemLogger.AddOverride(line)
}

// Clear all overrides
func ClearOverrides() {
	SystemLogger.ClearOverrides()
}

// Run function only if output will be logged at debug level
func LazyDebug(fn func() string) {
	if SystemLogger.isEnabled(Debug, 1) {
		SystemLogger.printf(Debug, 1, "%s", fn())
	}
}

// Run function only if output will be logged at trace level
func LazyTrace(fn func() string) {
	if SystemLogger.isEnabled(Trace, 1) {
		SystemLogger.printf(Trace, 1, "%s", fn())
	}
}
<commit_msg>MB-100: Log goroutines only every 5 minutes<commit_after>package logging

import "io"
import "os"
import "fmt"
import "strings"
import "time"
import "bytes"
import "strconv"
import "net/http"
import "io/ioutil"
import "runtime"
import "runtime/debug"
import l "log"
import rx "regexp"

// Log levels
type LogLevel int16

const (
	Silent LogLevel = iota
	Fatal
	Error
	Warn
	Info
	Timing
	Debug
	Trace
)

// Logger interface
type Logger interface {
	// Warnings, logged by default.
	Warnf(format string, v ...interface{})
	// Errors, logged by default.
	Errorf(format string, v ...interface{})
	// Fatal errors. Will not terminate execution.
	Fatalf(format string, v ...interface{})
	// Informational messages.
	Infof(format string, v ...interface{})
	// Get stack trace
	StackTrace() string
	// Timing utility
	Timer(format string, v ...interface{}) Ender
	// Debugging messages
	Debugf(format string, v ...interface{})
	// Program execution
	Tracef(format string, v ...interface{})
	// Call and print the stringer if debugging enabled
	LazyDebug(fn func() string)
	// Call and print the stringer if tracing enabled
	LazyTrace(fn func() string)
}

// Timer interface
type Ender interface {
	// Stop and log timing
	End()
}

//
// Implementation
//

type overrideLoc struct {
	matcher *rx.Regexp
	level   LogLevel
}

// Messages administrator should eventually see.
func (t LogLevel) String() string {
	switch t {
	case Silent:
		return "Silent"
	case Fatal:
		return "Fatal"
	case Error:
		return "Error"
	case Warn:
		return "Warn"
	case Info:
		return "Info"
	case Timing:
		return "Timing"
	case Debug:
		return "Debug"
	case Trace:
		return "Trace"
	default:
		return "Info"
	}
}

func Level(s string) LogLevel {
	switch strings.ToUpper(s) {
	case "SILENT":
		return Silent
	case "FATAL":
		return Fatal
	case "ERROR":
		return Error
	case "WARN":
		return Warn
	case "INFO":
		return Info
	case "TIMING":
		return Timing
	case "DEBUG":
		return Debug
	case "TRACE":
		return Trace
	default:
		return Info
	}
}

type destination struct {
	baselevel LogLevel
	target    *l.Logger
	overrides *[]overrideLoc
}

type stopClock struct {
	comment string
	skip    int
	start   time.Time
	log     *destination
}

func (log *destination) Warnf(format string, v ...interface{}) {
	log.printf(Warn, 1, format, v...)
}

// Errors that caused problems in execution logic.
func (log *destination) Errorf(format string, v ...interface{}) {
	log.printf(Error, 1, format, v...)
}

// Fatal messages are to be logged prior to exiting due to errors.
func (log *destination) Fatalf(format string, v ...interface{}) {
	log.printf(Fatal, 1, format, v...)
}

// Info messages are those that are logged but not expected to be read.
func (log *destination) Infof(format string, v ...interface{}) {
	log.printf(Info, 1, format, v...)
}

// Function timing. Use as:
//    defer Time("Waiting for backfill").End()
//     ... function to be timed
// or
//    timer := Timer("For vbucket %d mutation %d", vbid, seq)
//     ... lines to be timed
//    timer.End()
//
func (log *destination) Timer(format string, v ...interface{}) Ender {
	return log.timer(1, format, v...)
}

// Debug messages to help analyze problem. Default off.
func (log *destination) Debugf(format string, v ...interface{}) {
	log.printf(Debug, 1, format, v...)
}

// Execution trace showing the program flow. Default off.
func (log *destination) Tracef(format string, v ...interface{}) {
	log.printf(Trace, 1, format, v...)
}

// Set the base log level
func (log *destination) SetLogLevel(to LogLevel) {
	log.baselevel = to
}

// Get stack trace
func (log *destination) StackTrace() string {
	return log.getStackTrace(2, debug.Stack())
}

// Get profiling info
func Profile(port string, endpoints ...string) func() string {
	if strings.HasPrefix(port, ":") {
		port = port[1:]
	}
	return func() string {
		var buf bytes.Buffer
		for _, endpoint := range endpoints {
			addr := fmt.Sprintf("http://localhost:%s/debug/pprof/%s?debug=1", port, endpoint)
			resp, err := http.Get(addr)
			if err != nil {
				continue
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				continue
			}
			buf.Write(body)
		}
		return buf.String()
	}
}

// Dump profiling info periodically
func PeriodicProfile(port string, endpoints ...string) {
	profiler := Profile(port, endpoints...)
	tick := time.NewTicker(5 * time.Minute)
	go func() {
		for {
			select {
			case <-tick.C:
				LazyDebug(profiler)
			}
		}
	}()
}

// Run function only if output will be logged at debug level
func (log *destination) LazyDebug(fn func() string) {
	if log.isEnabled(Debug, 1) {
		log.printf(Debug, 1, "%s", fn())
	}
}

// Run function only if output will be logged at trace level
func (log *destination) LazyTrace(fn func() string) {
	if log.isEnabled(Trace, 1) {
		log.printf(Trace, 1, "%s", fn())
	}
}

// Add logging override. Format: filename[:line]=Level[,...]
func (log *destination) AddOverride(line string) {
	// infrequent, so clone to avoid locks
	added := make([]overrideLoc, 0, 16)
	if log.overrides != nil {
		added = append(added, *log.overrides...)
	}
	specs := strings.Split(line, ",")
	for _, spec := range specs {
		kv := strings.Split(spec, "=")
		if len(kv) != 2 {
			continue
		}
		exp, slvl := kv[0], kv[1]
		exp = strings.Replace(exp, `*`, `.*`, -1)
		if !strings.Contains(exp, ":") {
			exp += `:[\d]+`
		}
		exp = exp + "$"
		level := Level(slvl)
		re, err := rx.Compile(exp)
		if err == nil {
			entry := overrideLoc{matcher: re, level: level}
			added = append(added, entry)
		}
	}
	log.overrides = &added
}

// Clear all overrides
func (log *destination) ClearOverrides() {
	log.overrides = nil
}

// Stop the running timer and print timing
func (watch *stopClock) End() {
	elapsed := time.Since(watch.start).Nanoseconds()
	watch.log.printf(Timing, watch.skip, "%.1f μs - %s", float64(elapsed)/1000, watch.comment)
}

// Check if enabled
func (log *destination) isEnabled(at LogLevel, skip int) bool {
	// normal production case
	if log.overrides == nil {
		return log.baselevel >= at
	}

	// unusual case, perhaps troubleshooting
	_, file, line, _ := runtime.Caller(skip + 1)
	loc := file + ":" + strconv.Itoa(line)
	for _, spec := range *log.overrides {
		if spec.matcher.MatchString(loc) {
			return spec.level >= at
		}
	}

	return log.baselevel >= at
}

func (log *destination) printf(at LogLevel, skip int, format string, v ...interface{}) {
	if log.isEnabled(at, skip+1) {
		log.target.Printf("["+at.String()+"] "+format, v...)
	}
}

func (log *destination) getStackTrace(skip int, stack []byte) string {
	var buf bytes.Buffer
	lines := strings.Split(string(stack), "\n")
	for _, call := range lines[skip*2:] {
		buf.WriteString(fmt.Sprintf("%s\n", call))
	}
	return buf.String()
}

func (log *destination) timer(skip int, format string, v ...interface{}) Ender {
	if !log.isEnabled(Timing, skip) {
		return emptyclock
	}
	comment := fmt.Sprintf(format, v...)
	return &stopClock{comment: comment, skip: skip, start: time.Now(), log: log}
}

// No op clock
var emptyclock = &emptyClock{}

type emptyClock struct{}

func (_ *emptyClock) End() {
}

// The default logger
var SystemLogger destination

func init() {
	dest := l.New(os.Stdout, "", l.Lmicroseconds)
	SystemLogger = destination{baselevel: Info, target: dest, overrides: nil}
}

// SetLogWriter sets a new default destination
func SetLogWriter(w io.Writer) {
	dest := l.New(w, "", l.Lmicroseconds)
	SystemLogger = destination{baselevel: Info, target: dest, overrides: nil}
}

//
// A set of convenience methods to log to default logger
// See correspond methods on destination for details
//
func Warnf(format string, v ...interface{}) {
	SystemLogger.printf(Warn, 1, format, v...)
}

// Errorf to log message and warning messages will be logged.
func Errorf(format string, v ...interface{}) {
	SystemLogger.printf(Error, 1, format, v...)
}

// Fatalf to log message and warning messages will be logged.
func Fatalf(format string, v ...interface{}) {
	SystemLogger.printf(Fatal, 1, format, v...)
}

// Infof to log message at info level.
func Infof(format string, v ...interface{}) {
	SystemLogger.printf(Info, 1, format, v...)
}

// Debugf to log message at info level.
func Debugf(format string, v ...interface{}) {
	SystemLogger.printf(Debug, 1, format, v...)
}

// Tracef to log message at info level.
func Tracef(format string, v ...interface{}) {
	SystemLogger.printf(Trace, 1, format, v...)
}

// StackTrace prints current stack at specified log level
func StackTrace() string {
	return SystemLogger.getStackTrace(2, debug.Stack())
}

// Timing utility function
func Timer(format string, v ...interface{}) Ender {
	return SystemLogger.timer(2, format, v...)
}

// Set the base log level
func SetLogLevel(to LogLevel) {
	SystemLogger.SetLogLevel(to)
}

// Add logging override. Format: filename[:line]=Level[,...]
func AddOverride(line string) {
	SystemLogger.AddOverride(line)
}

// Clear all overrides
func ClearOverrides() {
	SystemLogger.ClearOverrides()
}

// Run function only if output will be logged at debug level
func LazyDebug(fn func() string) {
	if SystemLogger.isEnabled(Debug, 1) {
		SystemLogger.printf(Debug, 1, "%s", fn())
	}
}

// Run function only if output will be logged at trace level
func LazyTrace(fn func() string) {
	if SystemLogger.isEnabled(Trace, 1) {
		SystemLogger.printf(Trace, 1, "%s", fn())
	}
}
<|endoftext|>


<commit_before>package opentsdb

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"net"
	"net/http"
	"net/textproto"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/influxdb/influxdb/cluster"
	"github.com/influxdb/influxdb/meta"
	"github.com/influxdb/influxdb/tsdb"
)

const leaderWaitTimeout = 30 * time.Second

// Service manages the listener and handler for an HTTP endpoint.
type Service struct {
	ln     net.Listener  // main listener
	httpln *chanListener // http channel-based listener

	wg  sync.WaitGroup
	err chan error

	BindAddress      string
	Database         string
	RetentionPolicy  string
	ConsistencyLevel cluster.ConsistencyLevel

	PointsWriter interface {
		WritePoints(p *cluster.WritePointsRequest) error
	}
	MetaStore interface {
		WaitForLeader(d time.Duration) error
		CreateDatabaseIfNotExists(name string) (*meta.DatabaseInfo, error)
	}

	Logger *log.Logger
}

// NewService returns a new instance of Service.
func NewService(c Config) (*Service, error) {
	consistencyLevel, err := cluster.ParseConsistencyLevel(c.ConsistencyLevel)
	if err != nil {
		return nil, err
	}

	s := &Service{
		err:              make(chan error),
		BindAddress:      c.BindAddress,
		Database:         c.Database,
		RetentionPolicy:  c.RetentionPolicy,
		ConsistencyLevel: consistencyLevel,
		Logger:           log.New(os.Stderr, "[opentsdb] ", log.LstdFlags),
	}
	return s, nil
}

// Open starts the service
func (s *Service) Open() error {
	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.Logger.Printf("failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.Database); err != nil {
		s.Logger.Printf("failed to ensure target database %s exists: %s", s.Database, err.Error())
		return err
	}
	s.Logger.Printf("ensured target database %s exists", s.Database)

	// Open listener.
	ln, err := net.Listen("tcp", s.BindAddress)
	if err != nil {
		return err
	}
	s.ln = ln
	s.httpln = newChanListener(ln.Addr())

	s.Logger.Println("listening on:", ln.Addr().String())

	// Begin listening for connections.
	s.wg.Add(2)
	go s.serveHTTP()
	go s.serve()

	return nil
}

// Close closes the underlying listener.
func (s *Service) Close() error {
	if s.ln != nil {
		return s.ln.Close()
	}

	s.wg.Wait()
	return nil
}

// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) { s.Logger = l }

// Err returns a channel for fatal errors that occur on the listener.
func (s *Service) Err() <-chan error { return s.err }

// Addr returns the listener's address. Returns nil if listener is closed.
func (s *Service) Addr() net.Addr {
	if s.ln == nil {
		return nil
	}
	return s.ln.Addr()
}

// serve serves the handler from the listener.
func (s *Service) serve() {
	defer s.wg.Done()

	for {
		// Wait for next connection.
		conn, err := s.ln.Accept()
		if opErr, ok := err.(*net.OpError); ok && !opErr.Temporary() {
			s.Logger.Println("openTSDB TCP listener closed")
			return
		} else if err != nil {
			s.Logger.Println("error accepting openTSDB: ", err.Error())
			continue
		}

		// Handle connection in separate goroutine.
		go s.handleConn(conn)
	}
}

// handleConn processes conn. This is run in a separate goroutine.
func (s *Service) handleConn(conn net.Conn) {
	// Read header into buffer to check if it's HTTP.
	var buf bytes.Buffer
	r := bufio.NewReader(io.TeeReader(conn, &buf))

	// Attempt to parse connection as HTTP.
	_, err := http.ReadRequest(r)

	// Rebuild connection from buffer and remaining connection data.
	bufr := bufio.NewReader(io.MultiReader(&buf, conn))
	conn = &readerConn{Conn: conn, r: bufr}

	// If no HTTP parsing error occurred then process as HTTP.
	if err == nil {
		s.httpln.ch <- conn
		return
	}

	// Otherwise handle in telnet format.
	s.wg.Add(1)
	s.handleTelnetConn(conn)
}

// handleTelnetConn accepts OpenTSDB's telnet protocol.
// Each telnet command consists of a line of the form:
//   put sys.cpu.user 1356998400 42.5 host=webserver01 cpu=0
func (s *Service) handleTelnetConn(conn net.Conn) {
	defer conn.Close()
	defer s.wg.Done()

	// Wrap connection in a text protocol reader.
	r := textproto.NewReader(bufio.NewReader(conn))
	for {
		line, err := r.ReadLine()
		if err != nil {
			s.Logger.Println("error reading from openTSDB connection", err.Error())
			return
		}

		inputStrs := strings.Fields(line)

		if len(inputStrs) == 1 && inputStrs[0] == "version" {
			conn.Write([]byte("InfluxDB TSDB proxy"))
			continue
		}

		if len(inputStrs) < 4 || inputStrs[0] != "put" {
			s.Logger.Println("TSDBServer: malformed line, skipping: ", line)
			continue
		}

		measurement := inputStrs[1]
		tsStr := inputStrs[2]
		valueStr := inputStrs[3]
		tagStrs := inputStrs[4:]

		var t time.Time
		ts, err := strconv.ParseInt(tsStr, 10, 64)
		if err != nil {
			s.Logger.Println("TSDBServer: malformed time, skipping: ", tsStr)
		}

		switch len(tsStr) {
		case 10:
			t = time.Unix(ts, 0)
			break
		case 13:
			t = time.Unix(ts/1000, (ts%1000)*1000)
			break
		default:
			s.Logger.Println("TSDBServer: time must be 10 or 13 chars, skipping: ", tsStr)
			continue
		}

		tags := make(map[string]string)
		for t := range tagStrs {
			parts := strings.SplitN(tagStrs[t], "=", 2)
			if len(parts) != 2 {
				s.Logger.Println("TSDBServer: malformed tag data", tagStrs[t])
				continue
			}
			k := parts[0]

			tags[k] = parts[1]
		}

		fields := make(map[string]interface{})
		fields["value"], err = strconv.ParseFloat(valueStr, 64)
		if err != nil {
			s.Logger.Println("TSDBServer: could not parse value as float: ", valueStr)
			continue
		}

		p := tsdb.NewPoint(measurement, tags, fields, t)
		if err := s.PointsWriter.WritePoints(&cluster.WritePointsRequest{
			Database:         s.Database,
			RetentionPolicy:  s.RetentionPolicy,
			ConsistencyLevel: s.ConsistencyLevel,
			Points:           []tsdb.Point{p},
		}); err != nil {
			s.Logger.Println("TSDB cannot write data: ", err)
			continue
		}
	}
}

// serveHTTP handles connections in HTTP format.
func (s *Service) serveHTTP() {
	srv := &http.Server{Handler: &Handler{
		Database:         s.Database,
		RetentionPolicy:  s.RetentionPolicy,
		ConsistencyLevel: s.ConsistencyLevel,
		PointsWriter:     s.PointsWriter,
		Logger:           s.Logger,
	}}
	srv.Serve(s.httpln)
}
<commit_msg>catch opentsdb malformed tags if they are missing keys or values<commit_after>package opentsdb

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"net"
	"net/http"
	"net/textproto"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/influxdb/influxdb/cluster"
	"github.com/influxdb/influxdb/meta"
	"github.com/influxdb/influxdb/tsdb"
)

const leaderWaitTimeout = 30 * time.Second

// Service manages the listener and handler for an HTTP endpoint.
type Service struct {
	ln     net.Listener  // main listener
	httpln *chanListener // http channel-based listener

	wg  sync.WaitGroup
	err chan error

	BindAddress      string
	Database         string
	RetentionPolicy  string
	ConsistencyLevel cluster.ConsistencyLevel

	PointsWriter interface {
		WritePoints(p *cluster.WritePointsRequest) error
	}
	MetaStore interface {
		WaitForLeader(d time.Duration) error
		CreateDatabaseIfNotExists(name string) (*meta.DatabaseInfo, error)
	}

	Logger *log.Logger
}

// NewService returns a new instance of Service.
func NewService(c Config) (*Service, error) {
	consistencyLevel, err := cluster.ParseConsistencyLevel(c.ConsistencyLevel)
	if err != nil {
		return nil, err
	}

	s := &Service{
		err:              make(chan error),
		BindAddress:      c.BindAddress,
		Database:         c.Database,
		RetentionPolicy:  c.RetentionPolicy,
		ConsistencyLevel: consistencyLevel,
		Logger:           log.New(os.Stderr, "[opentsdb] ", log.LstdFlags),
	}
	return s, nil
}

// Open starts the service
func (s *Service) Open() error {
	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.Logger.Printf("failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.Database); err != nil {
		s.Logger.Printf("failed to ensure target database %s exists: %s", s.Database, err.Error())
		return err
	}
	s.Logger.Printf("ensured target database %s exists", s.Database)

	// Open listener.
	ln, err := net.Listen("tcp", s.BindAddress)
	if err != nil {
		return err
	}
	s.ln = ln
	s.httpln = newChanListener(ln.Addr())

	s.Logger.Println("listening on:", ln.Addr().String())

	// Begin listening for connections.
	s.wg.Add(2)
	go s.serveHTTP()
	go s.serve()

	return nil
}

// Close closes the underlying listener.
func (s *Service) Close() error {
	if s.ln != nil {
		return s.ln.Close()
	}

	s.wg.Wait()
	return nil
}

// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) { s.Logger = l }

// Err returns a channel for fatal errors that occur on the listener.
func (s *Service) Err() <-chan error { return s.err }

// Addr returns the listener's address. Returns nil if listener is closed.
func (s *Service) Addr() net.Addr {
	if s.ln == nil {
		return nil
	}
	return s.ln.Addr()
}

// serve serves the handler from the listener.
func (s *Service) serve() {
	defer s.wg.Done()

	for {
		// Wait for next connection.
		conn, err := s.ln.Accept()
		if opErr, ok := err.(*net.OpError); ok && !opErr.Temporary() {
			s.Logger.Println("openTSDB TCP listener closed")
			return
		} else if err != nil {
			s.Logger.Println("error accepting openTSDB: ", err.Error())
			continue
		}

		// Handle connection in separate goroutine.
		go s.handleConn(conn)
	}
}

// handleConn processes conn. This is run in a separate goroutine.
func (s *Service) handleConn(conn net.Conn) {
	// Read header into buffer to check if it's HTTP.
	var buf bytes.Buffer
	r := bufio.NewReader(io.TeeReader(conn, &buf))

	// Attempt to parse connection as HTTP.
	_, err := http.ReadRequest(r)

	// Rebuild connection from buffer and remaining connection data.
	bufr := bufio.NewReader(io.MultiReader(&buf, conn))
	conn = &readerConn{Conn: conn, r: bufr}

	// If no HTTP parsing error occurred then process as HTTP.
	if err == nil {
		s.httpln.ch <- conn
		return
	}

	// Otherwise handle in telnet format.
	s.wg.Add(1)
	s.handleTelnetConn(conn)
}

// handleTelnetConn accepts OpenTSDB's telnet protocol.
// Each telnet command consists of a line of the form:
//   put sys.cpu.user 1356998400 42.5 host=webserver01 cpu=0
func (s *Service) handleTelnetConn(conn net.Conn) {
	defer conn.Close()
	defer s.wg.Done()

	// Wrap connection in a text protocol reader.
	r := textproto.NewReader(bufio.NewReader(conn))
	for {
		line, err := r.ReadLine()
		if err != nil {
			s.Logger.Println("error reading from openTSDB connection", err.Error())
			return
		}

		inputStrs := strings.Fields(line)

		if len(inputStrs) == 1 && inputStrs[0] == "version" {
			conn.Write([]byte("InfluxDB TSDB proxy"))
			continue
		}

		if len(inputStrs) < 4 || inputStrs[0] != "put" {
			s.Logger.Println("TSDBServer: malformed line, skipping: ", line)
			continue
		}

		measurement := inputStrs[1]
		tsStr := inputStrs[2]
		valueStr := inputStrs[3]
		tagStrs := inputStrs[4:]

		var t time.Time
		ts, err := strconv.ParseInt(tsStr, 10, 64)
		if err != nil {
			s.Logger.Println("TSDBServer: malformed time, skipping: ", tsStr)
		}

		switch len(tsStr) {
		case 10:
			t = time.Unix(ts, 0)
			break
		case 13:
			t = time.Unix(ts/1000, (ts%1000)*1000)
			break
		default:
			s.Logger.Println("TSDBServer: time must be 10 or 13 chars, skipping: ", tsStr)
			continue
		}

		tags := make(map[string]string)
		for t := range tagStrs {
			parts := strings.SplitN(tagStrs[t], "=", 2)
			if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 {
				s.Logger.Println("TSDBServer: malformed tag data", tagStrs[t])
				continue
			}
			k := parts[0]

			tags[k] = parts[1]
		}

		fields := make(map[string]interface{})
		fields["value"], err = strconv.ParseFloat(valueStr, 64)
		if err != nil {
			s.Logger.Println("TSDBServer: could not parse value as float: ", valueStr)
			continue
		}

		p := tsdb.NewPoint(measurement, tags, fields, t)
		if err := s.PointsWriter.WritePoints(&cluster.WritePointsRequest{
			Database:         s.Database,
			RetentionPolicy:  s.RetentionPolicy,
			ConsistencyLevel: s.ConsistencyLevel,
			Points:           []tsdb.Point{p},
		}); err != nil {
			s.Logger.Println("TSDB cannot write data: ", err)
			continue
		}
	}
}

// serveHTTP handles connections in HTTP format.
func (s *Service) serveHTTP() {
	srv := &http.Server{Handler: &Handler{
		Database:         s.Database,
		RetentionPolicy:  s.RetentionPolicy,
		ConsistencyLevel: s.ConsistencyLevel,
		PointsWriter:     s.PointsWriter,
		Logger:           s.Logger,
	}}
	srv.Serve(s.httpln)
}
<|endoftext|>


<commit_before>package termios

import (
	"flag"
	"os"
	"runtime"
	"syscall"
	"testing"
)

var dev = flag.String("device", "/dev/tty", "device to use")

func TestTcgetattr(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
}

func TestTcsetattr(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
	for _, opt := range []uintptr{TCSANOW, TCSADRAIN, TCSAFLUSH} {
		if err := Tcsetattr(f.Fd(), opt, &termios); err != nil {
			t.Fatal(err)
		}
	}
}

func TestTcsendbreak(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	if err := Tcsendbreak(f.Fd(), 0); err != nil {
		t.Fatal(err)
	}
}

func TestTcdrain(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	if err := Tcdrain(f.Fd()); err != nil {
		t.Fatal(err)
	}
}

func TestTiocmget(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var status int
	if err := Tiocmget(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocmset(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var status int
	if err := Tiocmget(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
	if err := Tiocmset(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocmbis(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	status := 0
	if err := Tiocmbis(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocmbic(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	status := 0
	if err := Tiocmbic(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocinq(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var inq int
	if err := Tiocinq(f.Fd(), &inq); err != nil {
		t.Fatal(err)
	}
	if inq != 0 {
		t.Fatal("Expected 0 bytes, got %v", inq)
	}
}

func TestTiocoutq(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var inq int
	if err := Tiocoutq(f.Fd(), &inq); err != nil {
		t.Fatal(err)
	}
	if inq != 0 {
		t.Fatal("Expected 0 bytes, got %v", inq)
	}
}

func TestCfgetispeed(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
	if baud := Cfgetispeed(&termios); baud == 0 {
		t.Fatalf("Cfgetispeed: expected > 0, got %v", baud)
	}
}

func TestCfgetospeed(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
	if baud := Cfgetospeed(&termios); baud == 0 {
		t.Fatalf("Cfgetospeed: expected > 0, got %v", baud)
	}
}

func opendev(t *testing.T) *os.File {
	_, pts, err := Pty()
	if err != nil {
		t.Fatal(err)
	}
	return pts
}

func checktty(t *testing.T, err error) {

	// some ioctls fail against char devices if they do not
	// support a particular feature
	if (runtime.GOOS == "darwin" && err == syscall.ENOTTY) || (runtime.GOOS == "linux" && err == syscall.EINVAL) {
		t.Skip(err)
	}
}
<commit_msg>skip two more tests on linux<commit_after>package termios

import (
	"flag"
	"os"
	"runtime"
	"syscall"
	"testing"
)

var dev = flag.String("device", "/dev/tty", "device to use")

func TestTcgetattr(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
}

func TestTcsetattr(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
	for _, opt := range []uintptr{TCSANOW, TCSADRAIN, TCSAFLUSH} {
		if err := Tcsetattr(f.Fd(), opt, &termios); err != nil {
			t.Fatal(err)
		}
	}
}

func TestTcsendbreak(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	if err := Tcsendbreak(f.Fd(), 0); err != nil {
		t.Fatal(err)
	}
}

func TestTcdrain(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	if err := Tcdrain(f.Fd()); err != nil {
		t.Fatal(err)
	}
}

func TestTiocmget(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var status int
	if err := Tiocmget(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocmset(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var status int
	if err := Tiocmget(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
	if err := Tiocmset(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocmbis(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	status := 0
	if err := Tiocmbis(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocmbic(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	status := 0
	if err := Tiocmbic(f.Fd(), &status); err != nil {
		checktty(t, err)
		t.Fatal(err)
	}
}

func TestTiocinq(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var inq int
	if err := Tiocinq(f.Fd(), &inq); err != nil {
		t.Fatal(err)
	}
	if inq != 0 {
		t.Fatal("Expected 0 bytes, got %v", inq)
	}
}

func TestTiocoutq(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var inq int
	if err := Tiocoutq(f.Fd(), &inq); err != nil {
		t.Fatal(err)
	}
	if inq != 0 {
		t.Fatal("Expected 0 bytes, got %v", inq)
	}
}

func TestCfgetispeed(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
	if baud := Cfgetispeed(&termios); baud == 0 && runtime.GOOS != "linux" {
		t.Fatalf("Cfgetispeed: expected > 0, got %v", baud)
	}
}

func TestCfgetospeed(t *testing.T) {
	f := opendev(t)
	defer f.Close()

	var termios syscall.Termios
	if err := Tcgetattr(f.Fd(), &termios); err != nil {
		t.Fatal(err)
	}
	if baud := Cfgetospeed(&termios); baud == 0 && runtime.GOOS != "linux" {
		t.Fatalf("Cfgetospeed: expected > 0, got %v", baud)
	}
}

func opendev(t *testing.T) *os.File {
	_, pts, err := Pty()
	if err != nil {
		t.Fatal(err)
	}
	return pts
}

func checktty(t *testing.T, err error) {

	// some ioctls fail against char devices if they do not
	// support a particular feature
	if (runtime.GOOS == "darwin" && err == syscall.ENOTTY) || (runtime.GOOS == "linux" && err == syscall.EINVAL) {
		t.Skip(err)
	}
}
<|endoftext|>


<commit_before>package termite

import (
	"bytes"
	"os"
	"fmt"
	"crypto"
	"hash"
	"path/filepath"
	"log"
	"io"
	"io/ioutil"
	"sync"
)

// Content based addressing cache.
//
// TODO - a successful GetAttr() will often be followed by a read.  we
// should have a small LRU cache for the content so we can serve the
// contents from memory.
type ContentCache struct {
	dir string

	sync.Mutex
	md5PathMap map[string]string
}

func NewContentCache(d string) *ContentCache {
	if fi, _ := os.Lstat(d); fi == nil {
		err := os.MkdirAll(d, 0700)
		if err != nil {
			panic(err)
		}
	}

	return &ContentCache{
		dir: d,
		md5PathMap: make(map[string]string),
	}
}

func HashPath(dir string, md5 []byte) string {
	s := fmt.Sprintf("%x", md5)
	prefix := s[:2]
	name := s[2:]
	dst := filepath.Join(dir, prefix, name)
	prefixDir, _ := filepath.Split(dst)
	if err := os.MkdirAll(prefixDir, 0700); err != nil {
		log.Fatal("MkdirAll error:", err)
	}
	return dst
}

func (me *ContentCache) HasHash(hash []byte) bool {
	p := HashPath(me.dir, hash)
	_, err := os.Lstat(p)
	return err == nil
}

func (me *ContentCache) Path(hash []byte) string {
	return HashPath(me.dir, hash)
}

func (me *ContentCache) NewHashWriter() *HashWriter {
	return NewHashWriter(me.dir, crypto.MD5)
}

type HashWriter struct {
	hasher hash.Hash
	dest   *os.File
	hash   []byte
}

func NewHashWriter(dir string, hashfunc crypto.Hash) *HashWriter {
	me := &HashWriter{}
	tmp, err := ioutil.TempFile(dir, ".md5temp")
	if err != nil {
		panic(err)
		log.Fatal(err)
	}

	me.dest = tmp
	me.hasher = hashfunc.New()
	return me
}

func (me *HashWriter) Write(p []byte) (n int, err os.Error) {
	n, err = me.dest.Write(p)
	me.hasher.Write(p[:n])
	return n, err
}

func (me *HashWriter) Close() os.Error {
	err := me.dest.Close()

	if err != nil {
		return err
	}
	src := me.dest.Name()
	dir, _ := filepath.Split(src)
	sum := me.hasher.Sum()
	sumpath := HashPath(dir, sum)

	log.Printf("saving hash %x\n", sum)
	err = os.Rename(src, sumpath)
	if err != nil {
		if fi, _ := os.Lstat(sumpath); fi == nil {
			log.Println("already have", sumpath)
			os.Remove(src)
		}
	}
	return err
}

const _BUFSIZE = 32 * 1024

func (me *ContentCache) DestructiveSavePath(path string) (md5 []byte, content []byte) {
	f, err := os.Open(path)
	if err != nil {
		return nil, nil
	}
	h := crypto.MD5.New()
	content, err = SavingCopy(h, f, _BUFSIZE)
	if err != nil {
		log.Fatal("DestructiveSavePath:", err)
	}

	s := h.Sum()
	p := me.Path(s)
	err = os.Rename(path, p)
	if err != nil {
		if fi, _ := os.Lstat(p); fi != nil {
			os.Remove(p)
			return s, content
		}
		log.Fatal("DestructiveSavePath:", err)
	}
	return s, content
}

func (me *ContentCache) SavePath(path string) (md5 []byte, content []byte) {
	f, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	return me.SaveStream(f)
}

func (me *ContentCache) Save(content []byte) (md5 []byte) {
	buf := bytes.NewBuffer(content)
	md5, _ = me.SaveStream(buf)
	return md5
}

func (me *ContentCache) SaveStream(input io.Reader) (md5 []byte, content []byte) {
	dup := me.NewHashWriter()
	content, err := SavingCopy(dup, input, _BUFSIZE)
	if err != nil {
		log.Fatal(err)
	}
	err = dup.Close()
	if err != nil {
		log.Fatal(err)
	}
	return dup.hasher.Sum(), content
}
<commit_msg>Add SaveImmutablePath() to ContentCache.<commit_after>package termite

import (
	"bytes"
	"os"
	"fmt"
	"crypto"
	"hash"
	"path/filepath"
	"log"
	"io"
	"io/ioutil"
	"sync"
)

// Content based addressing cache.
//
// TODO - a successful GetAttr() will often be followed by a read.  we
// should have a small LRU cache for the content so we can serve the
// contents from memory.
type ContentCache struct {
	dir string

	hashPathMapMutex sync.Mutex
	hashPathMap map[string]string
}

func NewContentCache(d string) *ContentCache {
	if fi, _ := os.Lstat(d); fi == nil {
		err := os.MkdirAll(d, 0700)
		if err != nil {
			panic(err)
		}
	}

	return &ContentCache{
		dir: d,
		hashPathMap: make(map[string]string),
	}
}

func HashPath(dir string, md5 []byte) string {
	s := fmt.Sprintf("%x", md5)
	prefix := s[:2]
	name := s[2:]
	dst := filepath.Join(dir, prefix, name)
	prefixDir, _ := filepath.Split(dst)
	if err := os.MkdirAll(prefixDir, 0700); err != nil {
		log.Fatal("MkdirAll error:", err)
	}
	return dst
}

func (me *ContentCache) HasHash(hash []byte) bool {
	p := HashPath(me.dir, hash)
	_, err := os.Lstat(p)
	return err == nil
}

func (me *ContentCache) Path(hash []byte) string {
	return HashPath(me.dir, hash)
}

func (me *ContentCache) NewHashWriter() *HashWriter {
	return NewHashWriter(me.dir, crypto.MD5)
}

type HashWriter struct {
	hasher hash.Hash
	dest   *os.File
	hash   []byte
}

func NewHashWriter(dir string, hashfunc crypto.Hash) *HashWriter {
	me := &HashWriter{}
	tmp, err := ioutil.TempFile(dir, ".md5temp")
	if err != nil {
		panic(err)
		log.Fatal(err)
	}

	me.dest = tmp
	me.hasher = hashfunc.New()
	return me
}

func (me *HashWriter) Write(p []byte) (n int, err os.Error) {
	n, err = me.dest.Write(p)
	me.hasher.Write(p[:n])
	return n, err
}

func (me *HashWriter) Close() os.Error {
	err := me.dest.Close()

	if err != nil {
		return err
	}
	src := me.dest.Name()
	dir, _ := filepath.Split(src)
	sum := me.hasher.Sum()
	sumpath := HashPath(dir, sum)

	log.Printf("saving hash %x\n", sum)
	err = os.Rename(src, sumpath)
	if err != nil {
		if fi, _ := os.Lstat(sumpath); fi == nil {
			log.Println("already have", sumpath)
			os.Remove(src)
		}
	}
	return err
}

const _BUFSIZE = 32 * 1024

func (me *ContentCache) DestructiveSavePath(path string) (md5 []byte, content []byte) {
	f, err := os.Open(path)
	if err != nil {
		return nil, nil
	}
	h := crypto.MD5.New()
	content, err = SavingCopy(h, f, _BUFSIZE)
	if err != nil {
		log.Fatal("DestructiveSavePath:", err)
	}

	s := h.Sum()
	p := me.Path(s)
	err = os.Rename(path, p)
	if err != nil {
		if fi, _ := os.Lstat(p); fi != nil {
			os.Remove(p)
			return s, content
		}
		log.Fatal("DestructiveSavePath:", err)
	}
	return s, content
}

func (me *ContentCache) SavePath(path string) (md5 []byte, content []byte) {
	f, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	return me.SaveStream(f)
}

func (me *ContentCache) SaveImmutablePath(path string) (md5 []byte, content []byte) {
	hasher := crypto.MD5.New()

	f, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	content, err = SavingCopy(hasher, f, 32*1024)
	if err != nil && err != os.EOF {
		log.Fatal(err)
	}

	md5 = hasher.Sum()
	me.hashPathMapMutex.Lock()
	defer me.hashPathMapMutex.Unlock()
	me.hashPathMap[string(md5)] = path
	return md5, content
}

func (me *ContentCache) Save(content []byte) (md5 []byte) {
	buf := bytes.NewBuffer(content)
	md5, _ = me.SaveStream(buf)
	return md5
}

func (me *ContentCache) SaveStream(input io.Reader) (md5 []byte, content []byte) {
	dup := me.NewHashWriter()
	content, err := SavingCopy(dup, input, _BUFSIZE)
	if err != nil {
		log.Fatal(err)
	}
	err = dup.Close()
	if err != nil {
		log.Fatal(err)
	}
	return dup.hasher.Sum(), content
}
<|endoftext|>