<commit_before>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"archive/tar"
	"crypto/sha512"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"syscall"

	specaci "github.com/appc/spec/aci"
	"github.com/appc/spec/pkg/tarheader"
	"github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/pkg/aci"
	"github.com/coreos/rkt/pkg/fileutil"
	"github.com/coreos/rkt/pkg/sys"
	"github.com/coreos/rkt/pkg/uid"
	"github.com/hashicorp/errwrap"
)

const (
	hashfilename     = "hash"
	renderedfilename = "rendered"
	imagefilename    = "image"
)

// TreeStore represents a store of rendered ACIs
type TreeStore struct {
	path string
}

// Write renders the ACI with the provided key in the treestore. id references
// that specific tree store rendered image.
// Write, to avoid having a rendered ACI with old stale files, requires that
// the destination directory doesn't exist (usually Remove should be called
// before Write)
func (ts *TreeStore) Write(id string, key string, s *Store) (string, error) {
	treepath := ts.GetPath(id)
	fi, _ := os.Stat(treepath)
	if fi != nil {
		return "", fmt.Errorf("path %s already exists", treepath)
	}
	imageID, err := types.NewHash(key)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot convert key to imageID"), err)
	}
	if err := os.MkdirAll(treepath, 0755); err != nil {
		return "", errwrap.Wrap(fmt.Errorf("cannot create treestore directory %s", treepath), err)
	}
	err = aci.RenderACIWithImageID(*imageID, treepath, s, uid.NewBlankUidRange())
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot render aci"), err)
	}
	hash, err := ts.Hash(id)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot calculate tree hash"), err)
	}
	err = ioutil.WriteFile(filepath.Join(treepath, hashfilename), []byte(hash), 0644)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot write hash file"), err)
	}
	// before creating the "rendered" flag file we need to ensure that all data is fsynced
	dfd, err := syscall.Open(treepath, syscall.O_RDONLY, 0)
	if err != nil {
		return "", err
	}
	defer syscall.Close(dfd)
	if err := sys.Syncfs(dfd); err != nil {
		return "", errwrap.Wrap(errors.New("failed to sync data"), err)
	}
	// Create rendered file
	f, err := os.Create(filepath.Join(treepath, renderedfilename))
	if err != nil {
		return "", errwrap.Wrap(errors.New("failed to write rendered file"), err)
	}
	f.Close()

	// Write the hash of the image that will use this tree store
	err = ioutil.WriteFile(filepath.Join(treepath, imagefilename), []byte(key), 0644)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot write image file"), err)
	}

	if err := syscall.Fsync(dfd); err != nil {
		return "", errwrap.Wrap(errors.New("failed to sync tree store directory"), err)
	}

	treeSize, err := ts.Size(id)
	if err != nil {
		return "", err
	}

	if err := s.UpdateTreeStoreSize(key, treeSize); err != nil {
		return "", err
	}

	return string(hash), nil
}

// Remove cleans the directory for the provided id
func (ts *TreeStore) Remove(id string, s *Store) error {
	treepath := ts.GetPath(id)
	// If tree path doesn't exist we're done
	_, err := os.Stat(treepath)
	if err != nil && os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		return errwrap.Wrap(errors.New("failed to open tree store directory"), err)
	}

	renderedFilePath := filepath.Join(treepath, renderedfilename)
	// The "rendered" flag file should be the firstly removed file. So if
	// the removal ends with some error leaving some stale files IsRendered()
	// will return false.
	_, err = os.Stat(renderedFilePath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !os.IsNotExist(err) {
		err := os.Remove(renderedFilePath)
		// Ensure that the treepath directory is fsynced after removing the
		// "rendered" flag file
		f, err := os.Open(treepath)
		if err != nil {
			return errwrap.Wrap(errors.New("failed to open tree store directory"), err)
		}
		defer f.Close()
		err = f.Sync()
		if err != nil {
			return errwrap.Wrap(errors.New("failed to sync tree store directory"), err)
		}
	}

	key, err := ts.GetImageHash(id)
	if err != nil {
		return err
	}

	if err := os.RemoveAll(treepath); err != nil {
		return err
	}

	return s.UpdateTreeStoreSize(key, 0)
}

// IsRendered checks if the tree store with the provided id is fully rendered
func (ts *TreeStore) IsRendered(id string) (bool, error) {
	// if the "rendered" flag file exists, assume that the store is already
	// fully rendered.
	treepath := ts.GetPath(id)
	_, err := os.Stat(filepath.Join(treepath, renderedfilename))
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return true, nil
}

// GetPath returns the absolute path of the treestore for the provided id.
// It doesn't ensure that the path exists and is fully rendered. This should
// be done calling IsRendered()
func (ts *TreeStore) GetPath(id string) string {
	return filepath.Join(ts.path, id)
}

// GetRootFS returns the absolute path of the rootfs for the provided id.
// It doesn't ensure that the rootfs exists and is fully rendered. This should
// be done calling IsRendered()
func (ts *TreeStore) GetRootFS(id string) string {
	return filepath.Join(ts.GetPath(id), "rootfs")
}

// Hash calculates an hash of the rendered ACI. It uses the same functions
// used to create a tar but instead of writing the full archive is just
// computes the sha512 sum of the file infos and contents.
func (ts *TreeStore) Hash(id string) (string, error) {
	treepath := ts.GetPath(id)

	hash := sha512.New()
	iw := NewHashWriter(hash)
	err := filepath.Walk(treepath, buildWalker(treepath, iw))
	if err != nil {
		return "", errwrap.Wrap(errors.New("error walking rootfs"), err)
	}

	hashstring := hashToKey(hash)

	return hashstring, nil
}

// Check calculates the actual rendered ACI's hash and verifies that it matches
// the saved value. Returns the calculated hash.
func (ts *TreeStore) Check(id string) (string, error) {
	treepath := ts.GetPath(id)
	hash, err := ioutil.ReadFile(filepath.Join(treepath, hashfilename))
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot read hash file"), err)
	}
	curhash, err := ts.Hash(id)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot calculate tree hash"), err)
	}
	if curhash != string(hash) {
		return "", fmt.Errorf("wrong tree hash: %s, expected: %s", curhash, hash)
	}
	return curhash, nil
}

// Size returns the size of the rootfs for the provided id. It is a relatively
// expensive operation, it goes through all the files and adds up their size.
func (ts *TreeStore) Size(id string) (int64, error) {
	sz, err := fileutil.DirSize(ts.GetPath(id))
	if err != nil {
		return -1, errwrap.Wrap(errors.New("error calculating size"), err)
	}
	return sz, nil
}

// GetImageHash returns the hash of the image that uses the tree store
// identified by id.
func (ts *TreeStore) GetImageHash(id string) (string, error) {
	treepath := ts.GetPath(id)

	imgHash, err := ioutil.ReadFile(filepath.Join(treepath, imagefilename))
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot read image file"), err)
	}

	return string(imgHash), nil
}

type xattr struct {
	Name  string
	Value string
}

// Like tar Header but, to keep json output reproducible:
// * Xattrs as a slice
// * Skip Uname and Gname
// TODO. Should ModTime/AccessTime/ChangeTime be saved? For validation its
// probably enough to hash the file contents and the other infos and avoid
// problems due to them changing.
// TODO(sgotti) Is it possible that json output will change between go
// versions? Use another or our own Marshaller?
type fileInfo struct {
	Name     string // name of header file entry
	Mode     int64  // permission and mode bits
	Uid      int    // user id of owner
	Gid      int    // group id of owner
	Size     int64  // length in bytes
	Typeflag byte   // type of header entry
	Linkname string // target name of link
	Devmajor int64  // major number of character or block device
	Devminor int64  // minor number of character or block device
	Xattrs   []xattr
}

func FileInfoFromHeader(hdr *tar.Header) *fileInfo {
	fi := &fileInfo{
		Name:     hdr.Name,
		Mode:     hdr.Mode,
		Uid:      hdr.Uid,
		Gid:      hdr.Gid,
		Size:     hdr.Size,
		Typeflag: hdr.Typeflag,
		Linkname: hdr.Linkname,
		Devmajor: hdr.Devmajor,
		Devminor: hdr.Devminor,
	}
	keys := make([]string, len(hdr.Xattrs))
	for k := range hdr.Xattrs {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	xattrs := make([]xattr, 0)
	for _, k := range keys {
		xattrs = append(xattrs, xattr{Name: k, Value: hdr.Xattrs[k]})
	}
	fi.Xattrs = xattrs
	return fi
}

// TODO(sgotti) this func is copied from appcs/spec/aci/build.go but also
// removes the hash, rendered and image files. Find a way to reuse it.
func buildWalker(root string, aw specaci.ArchiveWriter) filepath.WalkFunc {
	// cache of inode -> filepath, used to leverage hard links in the archive
	inos := map[uint64]string{}
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		relpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if relpath == "." {
			return nil
		}
		if relpath == specaci.ManifestFile ||
			relpath == hashfilename ||
			relpath == renderedfilename ||
			relpath == imagefilename {
			// ignore; this will be written by the archive writer
			// TODO(jonboulle): does this make sense? maybe just remove from archivewriter?
			return nil
		}

		link := ""
		var r io.Reader
		switch info.Mode() & os.ModeType {
		case os.ModeSocket:
			return nil
		case os.ModeNamedPipe:
		case os.ModeCharDevice:
		case os.ModeDevice:
		case os.ModeDir:
		case os.ModeSymlink:
			target, err := os.Readlink(path)
			if err != nil {
				return err
			}
			link = target
		default:
			file, err := os.Open(path)
			if err != nil {
				return err
			}
			defer file.Close()
			r = file
		}

		hdr, err := tar.FileInfoHeader(info, link)
		if err != nil {
			panic(err)
		}
		// Because os.FileInfo's Name method returns only the base
		// name of the file it describes, it may be necessary to
		// modify the Name field of the returned header to provide the
		// full path name of the file.
		hdr.Name = relpath
		tarheader.Populate(hdr, info, inos)
		// If the file is a hard link to a file we've already seen, we
		// don't need the contents
		if hdr.Typeflag == tar.TypeLink {
			hdr.Size = 0
			r = nil
		}

		if err := aw.AddFile(hdr, r); err != nil {
			return err
		}
		return nil
	}
}

type imageHashWriter struct {
	io.Writer
}

func NewHashWriter(w io.Writer) specaci.ArchiveWriter {
	return &imageHashWriter{w}
}

func (aw *imageHashWriter) AddFile(hdr *tar.Header, r io.Reader) error {
	// Write the json encoding of the FileInfo struct
	hdrj, err := json.Marshal(FileInfoFromHeader(hdr))
	if err != nil {
		return err
	}
	_, err = aw.Writer.Write(hdrj)
	if err != nil {
		return err
	}

	if r != nil {
		// Write the file data
		_, err := io.Copy(aw.Writer, r)
		if err != nil {
			return err
		}
	}

	return nil
}

func (aw *imageHashWriter) Close() error {
	return nil
}
<commit_msg>store: force removal of the tree store even if it's in a bad state<commit_after>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"archive/tar"
	"crypto/sha512"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"syscall"

	specaci "github.com/appc/spec/aci"
	"github.com/appc/spec/pkg/tarheader"
	"github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/pkg/aci"
	"github.com/coreos/rkt/pkg/fileutil"
	"github.com/coreos/rkt/pkg/sys"
	"github.com/coreos/rkt/pkg/uid"
	"github.com/hashicorp/errwrap"
)

const (
	hashfilename     = "hash"
	renderedfilename = "rendered"
	imagefilename    = "image"
)

// TreeStore represents a store of rendered ACIs
type TreeStore struct {
	path string
}

// Write renders the ACI with the provided key in the treestore. id references
// that specific tree store rendered image.
// Write, to avoid having a rendered ACI with old stale files, requires that
// the destination directory doesn't exist (usually Remove should be called
// before Write)
func (ts *TreeStore) Write(id string, key string, s *Store) (string, error) {
	treepath := ts.GetPath(id)
	fi, _ := os.Stat(treepath)
	if fi != nil {
		return "", fmt.Errorf("path %s already exists", treepath)
	}
	imageID, err := types.NewHash(key)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot convert key to imageID"), err)
	}
	if err := os.MkdirAll(treepath, 0755); err != nil {
		return "", errwrap.Wrap(fmt.Errorf("cannot create treestore directory %s", treepath), err)
	}
	err = aci.RenderACIWithImageID(*imageID, treepath, s, uid.NewBlankUidRange())
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot render aci"), err)
	}
	hash, err := ts.Hash(id)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot calculate tree hash"), err)
	}
	err = ioutil.WriteFile(filepath.Join(treepath, hashfilename), []byte(hash), 0644)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot write hash file"), err)
	}
	// before creating the "rendered" flag file we need to ensure that all data is fsynced
	dfd, err := syscall.Open(treepath, syscall.O_RDONLY, 0)
	if err != nil {
		return "", err
	}
	defer syscall.Close(dfd)
	if err := sys.Syncfs(dfd); err != nil {
		return "", errwrap.Wrap(errors.New("failed to sync data"), err)
	}
	// Create rendered file
	f, err := os.Create(filepath.Join(treepath, renderedfilename))
	if err != nil {
		return "", errwrap.Wrap(errors.New("failed to write rendered file"), err)
	}
	f.Close()

	// Write the hash of the image that will use this tree store
	err = ioutil.WriteFile(filepath.Join(treepath, imagefilename), []byte(key), 0644)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot write image file"), err)
	}

	if err := syscall.Fsync(dfd); err != nil {
		return "", errwrap.Wrap(errors.New("failed to sync tree store directory"), err)
	}

	treeSize, err := ts.Size(id)
	if err != nil {
		return "", err
	}

	if err := s.UpdateTreeStoreSize(key, treeSize); err != nil {
		return "", err
	}

	return string(hash), nil
}

// Remove cleans the directory for the provided id
func (ts *TreeStore) Remove(id string, s *Store) error {
	treepath := ts.GetPath(id)
	// If tree path doesn't exist we're done
	_, err := os.Stat(treepath)
	if err != nil && os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		return errwrap.Wrap(errors.New("failed to open tree store directory"), err)
	}

	renderedFilePath := filepath.Join(treepath, renderedfilename)
	// The "rendered" flag file should be the firstly removed file. So if
	// the removal ends with some error leaving some stale files IsRendered()
	// will return false.
	_, err = os.Stat(renderedFilePath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !os.IsNotExist(err) {
		err := os.Remove(renderedFilePath)
		// Ensure that the treepath directory is fsynced after removing the
		// "rendered" flag file
		f, err := os.Open(treepath)
		if err != nil {
			return errwrap.Wrap(errors.New("failed to open tree store directory"), err)
		}
		defer f.Close()
		err = f.Sync()
		if err != nil {
			return errwrap.Wrap(errors.New("failed to sync tree store directory"), err)
		}
	}

	key, err := ts.GetImageHash(id)
	if err != nil {
		fmt.Fprintf(os.Stderr, "store: warning: tree store in a bad state, forcing removal\n")
	}

	if err := os.RemoveAll(treepath); err != nil {
		return err
	}

	if key != "" {
		return s.UpdateTreeStoreSize(key, 0)
	}

	return nil
}

// IsRendered checks if the tree store with the provided id is fully rendered
func (ts *TreeStore) IsRendered(id string) (bool, error) {
	// if the "rendered" flag file exists, assume that the store is already
	// fully rendered.
	treepath := ts.GetPath(id)
	_, err := os.Stat(filepath.Join(treepath, renderedfilename))
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return true, nil
}

// GetPath returns the absolute path of the treestore for the provided id.
// It doesn't ensure that the path exists and is fully rendered. This should
// be done calling IsRendered()
func (ts *TreeStore) GetPath(id string) string {
	return filepath.Join(ts.path, id)
}

// GetRootFS returns the absolute path of the rootfs for the provided id.
// It doesn't ensure that the rootfs exists and is fully rendered. This should
// be done calling IsRendered()
func (ts *TreeStore) GetRootFS(id string) string {
	return filepath.Join(ts.GetPath(id), "rootfs")
}

// Hash calculates an hash of the rendered ACI. It uses the same functions
// used to create a tar but instead of writing the full archive is just
// computes the sha512 sum of the file infos and contents.
func (ts *TreeStore) Hash(id string) (string, error) {
	treepath := ts.GetPath(id)

	hash := sha512.New()
	iw := NewHashWriter(hash)
	err := filepath.Walk(treepath, buildWalker(treepath, iw))
	if err != nil {
		return "", errwrap.Wrap(errors.New("error walking rootfs"), err)
	}

	hashstring := hashToKey(hash)

	return hashstring, nil
}

// Check calculates the actual rendered ACI's hash and verifies that it matches
// the saved value. Returns the calculated hash.
func (ts *TreeStore) Check(id string) (string, error) {
	treepath := ts.GetPath(id)
	hash, err := ioutil.ReadFile(filepath.Join(treepath, hashfilename))
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot read hash file"), err)
	}
	curhash, err := ts.Hash(id)
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot calculate tree hash"), err)
	}
	if curhash != string(hash) {
		return "", fmt.Errorf("wrong tree hash: %s, expected: %s", curhash, hash)
	}
	return curhash, nil
}

// Size returns the size of the rootfs for the provided id. It is a relatively
// expensive operation, it goes through all the files and adds up their size.
func (ts *TreeStore) Size(id string) (int64, error) {
	sz, err := fileutil.DirSize(ts.GetPath(id))
	if err != nil {
		return -1, errwrap.Wrap(errors.New("error calculating size"), err)
	}
	return sz, nil
}

// GetImageHash returns the hash of the image that uses the tree store
// identified by id.
func (ts *TreeStore) GetImageHash(id string) (string, error) {
	treepath := ts.GetPath(id)

	imgHash, err := ioutil.ReadFile(filepath.Join(treepath, imagefilename))
	if err != nil {
		return "", errwrap.Wrap(errors.New("cannot read image file"), err)
	}

	return string(imgHash), nil
}

type xattr struct {
	Name  string
	Value string
}

// Like tar Header but, to keep json output reproducible:
// * Xattrs as a slice
// * Skip Uname and Gname
// TODO. Should ModTime/AccessTime/ChangeTime be saved? For validation its
// probably enough to hash the file contents and the other infos and avoid
// problems due to them changing.
// TODO(sgotti) Is it possible that json output will change between go
// versions? Use another or our own Marshaller?
type fileInfo struct {
	Name     string // name of header file entry
	Mode     int64  // permission and mode bits
	Uid      int    // user id of owner
	Gid      int    // group id of owner
	Size     int64  // length in bytes
	Typeflag byte   // type of header entry
	Linkname string // target name of link
	Devmajor int64  // major number of character or block device
	Devminor int64  // minor number of character or block device
	Xattrs   []xattr
}

func FileInfoFromHeader(hdr *tar.Header) *fileInfo {
	fi := &fileInfo{
		Name:     hdr.Name,
		Mode:     hdr.Mode,
		Uid:      hdr.Uid,
		Gid:      hdr.Gid,
		Size:     hdr.Size,
		Typeflag: hdr.Typeflag,
		Linkname: hdr.Linkname,
		Devmajor: hdr.Devmajor,
		Devminor: hdr.Devminor,
	}
	keys := make([]string, len(hdr.Xattrs))
	for k := range hdr.Xattrs {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	xattrs := make([]xattr, 0)
	for _, k := range keys {
		xattrs = append(xattrs, xattr{Name: k, Value: hdr.Xattrs[k]})
	}
	fi.Xattrs = xattrs
	return fi
}

// TODO(sgotti) this func is copied from appcs/spec/aci/build.go but also
// removes the hash, rendered and image files. Find a way to reuse it.
func buildWalker(root string, aw specaci.ArchiveWriter) filepath.WalkFunc {
	// cache of inode -> filepath, used to leverage hard links in the archive
	inos := map[uint64]string{}
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		relpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if relpath == "." {
			return nil
		}
		if relpath == specaci.ManifestFile ||
			relpath == hashfilename ||
			relpath == renderedfilename ||
			relpath == imagefilename {
			// ignore; this will be written by the archive writer
			// TODO(jonboulle): does this make sense? maybe just remove from archivewriter?
			return nil
		}

		link := ""
		var r io.Reader
		switch info.Mode() & os.ModeType {
		case os.ModeSocket:
			return nil
		case os.ModeNamedPipe:
		case os.ModeCharDevice:
		case os.ModeDevice:
		case os.ModeDir:
		case os.ModeSymlink:
			target, err := os.Readlink(path)
			if err != nil {
				return err
			}
			link = target
		default:
			file, err := os.Open(path)
			if err != nil {
				return err
			}
			defer file.Close()
			r = file
		}

		hdr, err := tar.FileInfoHeader(info, link)
		if err != nil {
			panic(err)
		}
		// Because os.FileInfo's Name method returns only the base
		// name of the file it describes, it may be necessary to
		// modify the Name field of the returned header to provide the
		// full path name of the file.
		hdr.Name = relpath
		tarheader.Populate(hdr, info, inos)
		// If the file is a hard link to a file we've already seen, we
		// don't need the contents
		if hdr.Typeflag == tar.TypeLink {
			hdr.Size = 0
			r = nil
		}

		if err := aw.AddFile(hdr, r); err != nil {
			return err
		}
		return nil
	}
}

type imageHashWriter struct {
	io.Writer
}

func NewHashWriter(w io.Writer) specaci.ArchiveWriter {
	return &imageHashWriter{w}
}

func (aw *imageHashWriter) AddFile(hdr *tar.Header, r io.Reader) error {
	// Write the json encoding of the FileInfo struct
	hdrj, err := json.Marshal(FileInfoFromHeader(hdr))
	if err != nil {
		return err
	}
	_, err = aw.Writer.Write(hdrj)
	if err != nil {
		return err
	}

	if r != nil {
		// Write the file data
		_, err := io.Copy(aw.Writer, r)
		if err != nil {
			return err
		}
	}

	return nil
}

func (aw *imageHashWriter) Close() error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Zack Guo <gizak@icloud.com>. All rights reserved.
// Use of this source code is governed by a MIT license that can
// be found in the LICENSE file.

package termui

// Block is a base struct for all other upper level widgets,
// consider it as css: display:block.
// Normally you do not need to create it manually.
type Block struct {
	X             int
	Y             int
	Border        labeledBorder
	IsDisplay     bool
	HasBorder     bool
	BgColor       Attribute
	Width         int
	Height        int
	innerWidth    int
	innerHeight   int
	innerX        int
	innerY        int
	PaddingTop    int
	PaddingBottom int
	PaddingLeft   int
	PaddingRight  int
}

// NewBlock returns a *Block which inherits styles from current theme.
func NewBlock() *Block {
	d := Block{}
	d.IsDisplay = true
	d.HasBorder = theme.HasBorder
	d.Border.BgColor = theme.BorderBg
	d.Border.FgColor = theme.BorderFg
	d.Border.LabelBgColor = theme.BorderLabelTextBg
	d.Border.LabelFgColor = theme.BorderLabelTextFg
	d.BgColor = theme.BlockBg
	d.Width = 2
	d.Height = 2
	return &d
}

// compute box model
func (d *Block) align() {
	d.innerWidth = d.Width - d.PaddingLeft - d.PaddingRight
	d.innerHeight = d.Height - d.PaddingTop - d.PaddingBottom
	d.innerX = d.X + d.PaddingLeft
	d.innerY = d.Y + d.PaddingTop

	if d.HasBorder {
		d.innerHeight -= 2
		d.innerWidth -= 2
		d.Border.X = d.X
		d.Border.Y = d.Y
		d.Border.Width = d.Width
		d.Border.Height = d.Height
		d.innerX++
		d.innerY++
	}
}

// Buffer implements Bufferer interface.
// Draw background and border (if any).
func (d *Block) Buffer() []Point {
	d.align()

	ps := []Point{}
	if !d.IsDisplay {
		return ps
	}

	if d.HasBorder {
		ps = d.Border.Buffer()
	}

	for i := 0; i < d.innerWidth; i++ {
		for j := 0; j < d.innerHeight; j++ {
			p := Point{}
			p.X = d.X + 1 + i
			p.Y = d.Y + 1 + j
			p.Ch = ' '
			p.Bg = d.BgColor
			ps = append(ps, p)
		}
	}
	return ps
}

// GetHeight implements GridBufferer.
// It returns current height of the block.
func (d Block) GetHeight() int {
	return d.Height
}

// SetX implements GridBufferer interface, which sets block's x position.
func (d *Block) SetX(x int) {
	d.X = x
}

// SetY implements GridBufferer interface, it sets y position for block.
func (d *Block) SetY(y int) {
	d.Y = y
}

// SetWidth implements GridBuffer interface, it sets block's width.
func (d *Block) SetWidth(w int) {
	d.Width = w
}
<commit_msg>Fix https://github.com/gizak/termui/issues/14<commit_after>// Copyright 2015 Zack Guo <gizak@icloud.com>. All rights reserved.
// Use of this source code is governed by a MIT license that can
// be found in the LICENSE file.

package termui

// Block is a base struct for all other upper level widgets,
// consider it as css: display:block.
// Normally you do not need to create it manually.
type Block struct {
	X             int
	Y             int
	Border        labeledBorder
	IsDisplay     bool
	HasBorder     bool
	BgColor       Attribute
	Width         int
	Height        int
	innerWidth    int
	innerHeight   int
	innerX        int
	innerY        int
	PaddingTop    int
	PaddingBottom int
	PaddingLeft   int
	PaddingRight  int
}

// NewBlock returns a *Block which inherits styles from current theme.
func NewBlock() *Block {
	d := Block{}
	d.IsDisplay = true
	d.HasBorder = theme.HasBorder
	d.Border.BgColor = theme.BorderBg
	d.Border.FgColor = theme.BorderFg
	d.Border.LabelBgColor = theme.BorderLabelTextBg
	d.Border.LabelFgColor = theme.BorderLabelTextFg
	d.BgColor = theme.BlockBg
	d.Width = 2
	d.Height = 2
	return &d
}

// compute box model
func (d *Block) align() {
	d.innerWidth = d.Width - d.PaddingLeft - d.PaddingRight
	d.innerHeight = d.Height - d.PaddingTop - d.PaddingBottom
	d.innerX = d.X + d.PaddingLeft
	d.innerY = d.Y + d.PaddingTop

	if d.HasBorder {
		d.innerHeight -= 2
		d.innerWidth -= 2
		d.Border.X = d.X
		d.Border.Y = d.Y
		d.Border.Width = d.Width
		d.Border.Height = d.Height
		d.innerX++
		d.innerY++
	}

	if d.innerHeight < 0 {
		d.innerHeight = 0
	}
	if d.innerWidth < 0 {
		d.innerWidth = 0
	}

}

// Buffer implements Bufferer interface.
// Draw background and border (if any).
func (d *Block) Buffer() []Point {
	d.align()

	ps := []Point{}
	if !d.IsDisplay {
		return ps
	}

	if d.HasBorder {
		ps = d.Border.Buffer()
	}

	for i := 0; i < d.innerWidth; i++ {
		for j := 0; j < d.innerHeight; j++ {
			p := Point{}
			p.X = d.X + 1 + i
			p.Y = d.Y + 1 + j
			p.Ch = ' '
			p.Bg = d.BgColor
			ps = append(ps, p)
		}
	}
	return ps
}

// GetHeight implements GridBufferer.
// It returns current height of the block.
func (d Block) GetHeight() int {
	return d.Height
}

// SetX implements GridBufferer interface, which sets block's x position.
func (d *Block) SetX(x int) {
	d.X = x
}

// SetY implements GridBufferer interface, it sets y position for block.
func (d *Block) SetY(y int) {
	d.Y = y
}

// SetWidth implements GridBuffer interface, it sets block's width.
func (d *Block) SetWidth(w int) {
	d.Width = w
}
<|endoftext|>


<commit_before>// Copyright (c) 2013, Danilo Cabello.
// Use of this source code is governed by LICENSE file.

package bloom

import (
	"crypto/sha256"
	"math"
	"math/big"
)

type Filter struct {
	Capacity      int
	ErrorRate     float64
	bitTotal      uint64
	bitArray      []byte
	hashFunctions int
	chunkSize     int
}

func New(capacity int, errorRate float64) Filter {
	bitTotal := uint64(math.Pow(2, math.Ceil(math.Log2(-float64(capacity)*math.Log(errorRate)/math.Pow(math.Log(2), 2)))))
	bitArray := make([]byte, bitTotal>>3)
	hashFunctions := int(math.Floor(math.Log2(1.0 / errorRate)))
	chunkSize := int(math.Ceil(math.Log2(float64(bitTotal)) / 8))

	return Filter{
		capacity,
		errorRate,
		bitTotal,
		bitArray,
		hashFunctions,
		chunkSize,
	}
}

func (f *Filter) Add(key []byte) {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		f.bitArray[byteIndex] |= bitMask
	}

	// fmt.Printf("\n")
}

func (f *Filter) Contains(key []byte) bool {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		if f.bitArray[byteIndex]&bitMask == 0 {
			return false
		}
	}

	return true
}

func (f *Filter) generateHashForKey(key []byte) []byte {
	var completeHash []byte
	var hash = sha256.New()

	hash.Write(key)
	completeHash = append(completeHash, hash.Sum(nil)...)

	for f.chunkSize*f.hashFunctions > len(completeHash) {
		hash.Reset()
		hash.Write(completeHash)
		completeHash = append(completeHash, hash.Sum(nil)...)
	}

	return completeHash
}

func (f *Filter) findBitArrayIndexAndMask(hash []byte, index int) (bitArrayIndex uint64, mask byte) {
	var subHashInt big.Int

	subHashBytes := hash[index*f.chunkSize : (index+1)*f.chunkSize]
	subHashInt.SetBytes(subHashBytes)
	position := subHashInt.Uint64()

	position %= f.bitTotal

	bitArrayIndex = uint64(position >> 3)
	mask = 1 << uint(position%8)
	return
}

func (f *Filter) GetSizeInBytes() uint64 {
	return f.bitTotal >> 3
}
<commit_msg>Returns pointer instead of a copy of the filter<commit_after>// Copyright (c) 2013, Danilo Cabello.
// Use of this source code is governed by LICENSE file.

package bloom

import (
	"crypto/sha256"
	"math"
	"math/big"
)

type Filter struct {
	Capacity      int
	ErrorRate     float64
	bitTotal      uint64
	bitArray      []byte
	hashFunctions int
	chunkSize     int
}

func New(capacity int, errorRate float64) *Filter {
	bitTotal := uint64(math.Pow(2, math.Ceil(math.Log2(-float64(capacity)*math.Log(errorRate)/math.Pow(math.Log(2), 2)))))
	bitArray := make([]byte, bitTotal>>3)
	hashFunctions := int(math.Floor(math.Log2(1.0 / errorRate)))
	chunkSize := int(math.Ceil(math.Log2(float64(bitTotal)) / 8))

	return &Filter{
		capacity,
		errorRate,
		bitTotal,
		bitArray,
		hashFunctions,
		chunkSize,
	}
}

func (f *Filter) Add(key []byte) {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		f.bitArray[byteIndex] |= bitMask
	}
}

func (f *Filter) Contains(key []byte) bool {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		if f.bitArray[byteIndex]&bitMask == 0 {
			return false
		}
	}

	return true
}

func (f *Filter) generateHashForKey(key []byte) []byte {
	var completeHash []byte
	var hash = sha256.New()

	hash.Write(key)
	completeHash = append(completeHash, hash.Sum(nil)...)

	for f.chunkSize*f.hashFunctions > len(completeHash) {
		hash.Reset()
		hash.Write(completeHash)
		completeHash = append(completeHash, hash.Sum(nil)...)
	}

	return completeHash
}

func (f *Filter) findBitArrayIndexAndMask(hash []byte, index int) (bitArrayIndex uint64, mask byte) {
	var subHashInt big.Int

	subHashBytes := hash[index*f.chunkSize : (index+1)*f.chunkSize]
	subHashInt.SetBytes(subHashBytes)
	position := subHashInt.Uint64()

	position %= f.bitTotal

	bitArrayIndex = uint64(position >> 3)
	mask = 1 << uint(position%8)
	return
}

func (f *Filter) GetSizeInBytes() uint64 {
	return f.bitTotal >> 3
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/camptocamp/conplicity/engines"
	"github.com/camptocamp/conplicity/handler"
	"github.com/camptocamp/conplicity/providers"
	"github.com/camptocamp/conplicity/util"
	"github.com/camptocamp/conplicity/volume"
)

var version = "undefined"

func main() {
	var err error
	var exitCode int

	c, err := handler.NewConplicity(version)
	util.CheckErr(err, "Failed to setup Conplicity handler: %v", "fatal")

	log.Infof("Conplity v%s starting backup...", version)

	vols, err := c.GetVolumes()
	util.CheckErr(err, "Failed to get Docker volumes: %v", "fatal")

	err = c.MetricsHandler.GetMetrics()
	util.CheckErr(err, "Failed to get exiting Prometheus metrics: %v", "fatal")

	for _, vol := range vols {
		c.LogTime(vol, "backupStartTime")
		err = backupVolume(c, vol)
		c.LogTime(vol, "backupEndTime")
		if err != nil {
			log.Errorf("Failed to backup volume %s: %v", vol.Name, err)
			exitCode = 1
			continue
		}
	}

	err = c.MetricsHandler.Push()
	if err != nil {
		log.Errorf("Failed to post data to Prometheus Pushgateway: %v", err)
		exitCode = 2
	}

	log.Infof("End backup...")
	os.Exit(exitCode)
}

func backupVolume(c *handler.Conplicity, vol *volume.Volume) (err error) {
	p := providers.GetProvider(c, vol)
	log.WithFields(log.Fields{
		"volume":   vol.Name,
		"provider": p.GetName(),
	}).Info("Found data provider")
	err = providers.PrepareBackup(p)
	if err != nil {
		err = fmt.Errorf("failed to prepare backup: %v", err)
		return
	}

	e := engines.GetEngine(c, vol)
	log.WithFields(log.Fields{
		"volume": vol.Name,
		"engine": e.GetName(),
	}).Info("Found backup engine")

	err = e.Backup()
	if err != nil {
		err = fmt.Errorf("failed to backup volume: %v", err)
		return
	}
	return
}
<commit_msg>Fix typo<commit_after>package main

import (
	"fmt"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/camptocamp/conplicity/engines"
	"github.com/camptocamp/conplicity/handler"
	"github.com/camptocamp/conplicity/providers"
	"github.com/camptocamp/conplicity/util"
	"github.com/camptocamp/conplicity/volume"
)

var version = "undefined"

func main() {
	var err error
	var exitCode int

	c, err := handler.NewConplicity(version)
	util.CheckErr(err, "Failed to setup Conplicity handler: %v", "fatal")

	log.Infof("Conplicity v%s starting backup...", version)

	vols, err := c.GetVolumes()
	util.CheckErr(err, "Failed to get Docker volumes: %v", "fatal")

	err = c.MetricsHandler.GetMetrics()
	util.CheckErr(err, "Failed to get exiting Prometheus metrics: %v", "fatal")

	for _, vol := range vols {
		c.LogTime(vol, "backupStartTime")
		err = backupVolume(c, vol)
		c.LogTime(vol, "backupEndTime")
		if err != nil {
			log.Errorf("Failed to backup volume %s: %v", vol.Name, err)
			exitCode = 1
			continue
		}
	}

	err = c.MetricsHandler.Push()
	if err != nil {
		log.Errorf("Failed to post data to Prometheus Pushgateway: %v", err)
		exitCode = 2
	}

	log.Infof("End backup...")
	os.Exit(exitCode)
}

func backupVolume(c *handler.Conplicity, vol *volume.Volume) (err error) {
	p := providers.GetProvider(c, vol)
	log.WithFields(log.Fields{
		"volume":   vol.Name,
		"provider": p.GetName(),
	}).Info("Found data provider")
	err = providers.PrepareBackup(p)
	if err != nil {
		err = fmt.Errorf("failed to prepare backup: %v", err)
		return
	}

	e := engines.GetEngine(c, vol)
	log.WithFields(log.Fields{
		"volume": vol.Name,
		"engine": e.GetName(),
	}).Info("Found backup engine")

	err = e.Backup()
	if err != nil {
		err = fmt.Errorf("failed to backup volume: %v", err)
		return
	}
	return
}
<|endoftext|>


<commit_before>/*
Package bloom provides data structures and methods for creating Bloom filters.

A Bloom filter is a representation of a set of _n_ items, where the main
requirement is to make membership queries; _i.e._, whether an item is a
member of a set.

A Bloom filter has two parameters: _m_, a maximum size (typically a reasonably large
multiple of the cardinality of the set to represent) and _k_, the number of hashing
functions on elements of the set. (The actual hashing functions are important, too,
but this is not a parameter for this implementation). A Bloom filter is backed by
a BitSet; a key is represented in the filter by setting the bits at each value of the
hashing functions (modulo _m_). Set membership is done by _testing_ whether the
bits at each value of the hashing functions (again, modulo _m_) are set. If so,
the item is in the set. If the item is actually in the set, a Bloom filter will
never fail (the true positive rate is 1.0); but it is susceptible to false
positives. The art is to choose _k_ and _m_ correctly.

In this implementation, the hashing functions used is a local version of FNV,
a non-cryptographic hashing function, seeded with the index number of
the kth hashing function.

This implementation accepts keys for setting as testing as []byte. Thus, to
add a string item, "Love":

    uint n = 1000
    filter := bloom.New(20*n, 5) // load of 20, 5 keys
    filter.Add([]byte("Love"))

Similarly, to test if "Love" is in bloom:

    if filter.Test([]byte("Love"))

For numeric data, I recommend that you look into the binary/encoding library. But,
for example, to add a uint32 to the filter:

    i := uint32(100)
    n1 := make([]byte,4)
    binary.BigEndian.PutUint32(n1,i)
    f.Add(n1)

Finally, there is a method to estimate the false positive rate of a particular
Bloom filter for a set of size _n_:

    if filter.EstimateFalsePositiveRate(1000) > 0.001

Given the particular hashing scheme, it's best to be empirical about this. Note
that estimating the FP rate will clear the Bloom filter.
*/
package bloom

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"math"

	"github.com/spaolacci/murmur3"
	"github.com/willf/bitset"
)

// A BloomFilter is a representation of a set of _n_ items, where the main
// requirement is to make membership queries; _i.e._, whether an item is a
// member of a set.
type BloomFilter struct {
	m uint
	k uint
	b *bitset.BitSet
}

// New creates a new Bloom filter with _m_ bits and _k_ hashing functions
func New(m uint, k uint) *BloomFilter {
	return &BloomFilter{m, k, bitset.New(m)}
}

// baseHashes returns the four hash values of data that are used to create k
// hashes
func baseHashes(data []byte) [4]uint64 {
	a1 := []byte{1} // to grab another bit of data
	hasher := murmur3.New128()
	hasher.Write(data) // #nosec
	v1, v2 := hasher.Sum128()
	hasher.Write(a1) // #nosec
	v3, v4 := hasher.Sum128()
	return [4]uint64{
		v1, v2, v3, v4,
	}
}

// location returns the ith hashed location using the four base hash values
func (f *BloomFilter) location(h [4]uint64, i uint) (location uint) {
	ii := uint64(i)
	location = uint((h[ii%2] + ii*h[2+(((ii+(ii%2))%4)/2)]) % uint64(f.m))
	return
}

// EstimateParameters estimates requirements for m and k.
// Based on https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go
// used with permission.
func EstimateParameters(n uint, p float64) (m uint, k uint) {
	m = uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2)))
	k = uint(math.Ceil(math.Log(2) * float64(m) / float64(n)))
	return
}

// NewWithEstimates creates a new Bloom filter for about n items with fp
// false positive rate
func NewWithEstimates(n uint, fp float64) *BloomFilter {
	m, k := EstimateParameters(n, fp)
	return New(m, k)
}

// Cap returns the capacity, _m_, of a Bloom filter
func (f *BloomFilter) Cap() uint {
	return f.m
}

// K returns the number of hash functions used in the BloomFilter
func (f *BloomFilter) K() uint {
	return f.k
}

// Add data to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) Add(data []byte) *BloomFilter {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		f.b.Set(f.location(h, i))
	}
	return f
}

// Merge the data from two Bloom Filters.
func (f *BloomFilter) Merge(g *BloomFilter) error {
	// Make sure the m's and k's are the same, otherwise merging has no real use.
	if f.m != g.m {
		return fmt.Errorf("m's don't match: %d != %d", f.m, g.m)
	}

	if f.k != g.k {
		return fmt.Errorf("k's don't match: %d != %d", f.m, g.m)
	}

	f.b.InPlaceUnion(g.b)
	return nil
}

// Copy creates a copy of a Bloom filter.
func (f *BloomFilter) Copy() *BloomFilter {
	fc := New(f.m, f.k)
	fc.Merge(f) // #nosec
	return fc
}

// AddString to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) AddString(data string) *BloomFilter {
	return f.Add([]byte(data))
}

// Test returns true if the data is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) Test(data []byte) bool {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		if !f.b.Test(f.location(h, i)) {
			return false
		}
	}
	return true
}

// TestString returns true if the string is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) TestString(data string) bool {
	return f.Test([]byte(data))
}

// TestAndAdd is the equivalent to calling Test(data) then Add(data).
// Returns the result of Test.
func (f *BloomFilter) TestAndAdd(data []byte) bool {
	present := true
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		l := f.location(h, i)
		if !f.b.Test(l) {
			present = false
		}
		f.b.Set(l)
	}
	return present
}

// TestAndAddString is the equivalent to calling Test(string) then Add(string).
// Returns the result of Test.
func (f *BloomFilter) TestAndAddString(data string) bool {
	return f.TestAndAdd([]byte(data))
}

// ClearAll clears all the data in a Bloom filter, removing all keys
func (f *BloomFilter) ClearAll() *BloomFilter {
	f.b.ClearAll()
	return f
}

// EstimateFalsePositiveRate returns, for a BloomFilter with a estimate of m bits
// and k hash functions, what the false positive rate will be
// while storing n entries; runs 100,000 tests. This is an empirical
// test using integers as keys. As a side-effect, it clears the BloomFilter.
func (f *BloomFilter) EstimateFalsePositiveRate(n uint) (fpRate float64) {
	rounds := uint32(100000)
	f.ClearAll()
	n1 := make([]byte, 4)
	for i := uint32(0); i < uint32(n); i++ {
		binary.BigEndian.PutUint32(n1, i)
		f.Add(n1)
	}
	fp := 0
	// test for number of rounds
	for i := uint32(0); i < rounds; i++ {
		binary.BigEndian.PutUint32(n1, i+uint32(n)+1)
		if f.Test(n1) {
			//fmt.Printf("%v failed.\n", i+uint32(n)+1)
			fp++
		}
	}
	fpRate = float64(fp) / (float64(rounds))
	f.ClearAll()
	return
}

// bloomFilterJSON is an unexported type for marshaling/unmarshaling BloomFilter struct.
type bloomFilterJSON struct {
	M uint           `json:"m"`
	K uint           `json:"k"`
	B *bitset.BitSet `json:"b"`
}

// MarshalJSON implements json.Marshaler interface.
func (f *BloomFilter) MarshalJSON() ([]byte, error) {
	return json.Marshal(bloomFilterJSON{f.m, f.k, f.b})
}

// UnmarshalJSON implements json.Unmarshaler interface.
func (f *BloomFilter) UnmarshalJSON(data []byte) error {
	var j bloomFilterJSON
	err := json.Unmarshal(data, &j)
	if err != nil {
		return err
	}
	f.m = j.M
	f.k = j.K
	f.b = j.B
	return nil
}

// WriteTo writes a binary representation of the BloomFilter to an i/o stream.
// It returns the number of bytes written.
func (f *BloomFilter) WriteTo(stream io.Writer) (int64, error) {
	err := binary.Write(stream, binary.BigEndian, uint64(f.m))
	if err != nil {
		return 0, err
	}
	err = binary.Write(stream, binary.BigEndian, uint64(f.k))
	if err != nil {
		return 0, err
	}
	numBytes, err := f.b.WriteTo(stream)
	return numBytes + int64(2*binary.Size(uint64(0))), err
}

// ReadFrom reads a binary representation of the BloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (f *BloomFilter) ReadFrom(stream io.Reader) (int64, error) {
	var m, k uint64
	err := binary.Read(stream, binary.BigEndian, &m)
	if err != nil {
		return 0, err
	}
	err = binary.Read(stream, binary.BigEndian, &k)
	if err != nil {
		return 0, err
	}
	b := &bitset.BitSet{}
	numBytes, err := b.ReadFrom(stream)
	if err != nil {
		return 0, err
	}
	f.m = uint(m)
	f.k = uint(k)
	f.b = b
	return numBytes + int64(2*binary.Size(uint64(0))), nil
}

// GobEncode implements gob.GobEncoder interface.
func (f *BloomFilter) GobEncode() ([]byte, error) {
	var buf bytes.Buffer
	_, err := f.WriteTo(&buf)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

// GobDecode implements gob.GobDecoder interface.
func (f *BloomFilter) GobDecode(data []byte) error {
	buf := bytes.NewBuffer(data)
	_, err := f.ReadFrom(buf)

	return err
}

// Equal tests for the equality of two Bloom filters
func (f *BloomFilter) Equal(g *BloomFilter) bool {
	return f.m == g.m && f.k == g.k && f.b.Equal(g.b)
}
<commit_msg>acknowledge that theoretical errors might occur, and, if they do, we will ignore them<commit_after>/*
Package bloom provides data structures and methods for creating Bloom filters.

A Bloom filter is a representation of a set of _n_ items, where the main
requirement is to make membership queries; _i.e._, whether an item is a
member of a set.

A Bloom filter has two parameters: _m_, a maximum size (typically a reasonably large
multiple of the cardinality of the set to represent) and _k_, the number of hashing
functions on elements of the set. (The actual hashing functions are important, too,
but this is not a parameter for this implementation). A Bloom filter is backed by
a BitSet; a key is represented in the filter by setting the bits at each value of the
hashing functions (modulo _m_). Set membership is done by _testing_ whether the
bits at each value of the hashing functions (again, modulo _m_) are set. If so,
the item is in the set. If the item is actually in the set, a Bloom filter will
never fail (the true positive rate is 1.0); but it is susceptible to false
positives. The art is to choose _k_ and _m_ correctly.

In this implementation, the hashing functions used is a local version of FNV,
a non-cryptographic hashing function, seeded with the index number of
the kth hashing function.

This implementation accepts keys for setting as testing as []byte. Thus, to
add a string item, "Love":

    uint n = 1000
    filter := bloom.New(20*n, 5) // load of 20, 5 keys
    filter.Add([]byte("Love"))

Similarly, to test if "Love" is in bloom:

    if filter.Test([]byte("Love"))

For numeric data, I recommend that you look into the binary/encoding library. But,
for example, to add a uint32 to the filter:

    i := uint32(100)
    n1 := make([]byte,4)
    binary.BigEndian.PutUint32(n1,i)
    f.Add(n1)

Finally, there is a method to estimate the false positive rate of a particular
Bloom filter for a set of size _n_:

    if filter.EstimateFalsePositiveRate(1000) > 0.001

Given the particular hashing scheme, it's best to be empirical about this. Note
that estimating the FP rate will clear the Bloom filter.
*/
package bloom

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"math"

	"github.com/spaolacci/murmur3"
	"github.com/willf/bitset"
)

// A BloomFilter is a representation of a set of _n_ items, where the main
// requirement is to make membership queries; _i.e._, whether an item is a
// member of a set.
type BloomFilter struct {
	m uint
	k uint
	b *bitset.BitSet
}

// New creates a new Bloom filter with _m_ bits and _k_ hashing functions
func New(m uint, k uint) *BloomFilter {
	return &BloomFilter{m, k, bitset.New(m)}
}

// baseHashes returns the four hash values of data that are used to create k
// hashes
func baseHashes(data []byte) [4]uint64 {
	a1 := []byte{1} // to grab another bit of data
	hasher := murmur3.New128()
	_, _ = hasher.Write(data)
	v1, v2 := hasher.Sum128()
	_, _ = hasher.Write(a1)
	v3, v4 := hasher.Sum128()
	return [4]uint64{
		v1, v2, v3, v4,
	}
}

// location returns the ith hashed location using the four base hash values
func (f *BloomFilter) location(h [4]uint64, i uint) (location uint) {
	ii := uint64(i)
	location = uint((h[ii%2] + ii*h[2+(((ii+(ii%2))%4)/2)]) % uint64(f.m))
	return
}

// EstimateParameters estimates requirements for m and k.
// Based on https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go
// used with permission.
func EstimateParameters(n uint, p float64) (m uint, k uint) {
	m = uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2)))
	k = uint(math.Ceil(math.Log(2) * float64(m) / float64(n)))
	return
}

// NewWithEstimates creates a new Bloom filter for about n items with fp
// false positive rate
func NewWithEstimates(n uint, fp float64) *BloomFilter {
	m, k := EstimateParameters(n, fp)
	return New(m, k)
}

// Cap returns the capacity, _m_, of a Bloom filter
func (f *BloomFilter) Cap() uint {
	return f.m
}

// K returns the number of hash functions used in the BloomFilter
func (f *BloomFilter) K() uint {
	return f.k
}

// Add data to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) Add(data []byte) *BloomFilter {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		f.b.Set(f.location(h, i))
	}
	return f
}

// Merge the data from two Bloom Filters.
func (f *BloomFilter) Merge(g *BloomFilter) error {
	// Make sure the m's and k's are the same, otherwise merging has no real use.
	if f.m != g.m {
		return fmt.Errorf("m's don't match: %d != %d", f.m, g.m)
	}

	if f.k != g.k {
		return fmt.Errorf("k's don't match: %d != %d", f.m, g.m)
	}

	f.b.InPlaceUnion(g.b)
	return nil
}

// Copy creates a copy of a Bloom filter.
func (f *BloomFilter) Copy() *BloomFilter {
	fc := New(f.m, f.k)
	_ = fc.Merge(f)
	return fc
}

// AddString to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) AddString(data string) *BloomFilter {
	return f.Add([]byte(data))
}

// Test returns true if the data is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) Test(data []byte) bool {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		if !f.b.Test(f.location(h, i)) {
			return false
		}
	}
	return true
}

// TestString returns true if the string is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) TestString(data string) bool {
	return f.Test([]byte(data))
}

// TestAndAdd is the equivalent to calling Test(data) then Add(data).
// Returns the result of Test.
func (f *BloomFilter) TestAndAdd(data []byte) bool {
	present := true
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		l := f.location(h, i)
		if !f.b.Test(l) {
			present = false
		}
		f.b.Set(l)
	}
	return present
}

// TestAndAddString is the equivalent to calling Test(string) then Add(string).
// Returns the result of Test.
func (f *BloomFilter) TestAndAddString(data string) bool {
	return f.TestAndAdd([]byte(data))
}

// ClearAll clears all the data in a Bloom filter, removing all keys
func (f *BloomFilter) ClearAll() *BloomFilter {
	f.b.ClearAll()
	return f
}

// EstimateFalsePositiveRate returns, for a BloomFilter with a estimate of m bits
// and k hash functions, what the false positive rate will be
// while storing n entries; runs 100,000 tests. This is an empirical
// test using integers as keys. As a side-effect, it clears the BloomFilter.
func (f *BloomFilter) EstimateFalsePositiveRate(n uint) (fpRate float64) {
	rounds := uint32(100000)
	f.ClearAll()
	n1 := make([]byte, 4)
	for i := uint32(0); i < uint32(n); i++ {
		binary.BigEndian.PutUint32(n1, i)
		f.Add(n1)
	}
	fp := 0
	// test for number of rounds
	for i := uint32(0); i < rounds; i++ {
		binary.BigEndian.PutUint32(n1, i+uint32(n)+1)
		if f.Test(n1) {
			//fmt.Printf("%v failed.\n", i+uint32(n)+1)
			fp++
		}
	}
	fpRate = float64(fp) / (float64(rounds))
	f.ClearAll()
	return
}

// bloomFilterJSON is an unexported type for marshaling/unmarshaling BloomFilter struct.
type bloomFilterJSON struct {
	M uint           `json:"m"`
	K uint           `json:"k"`
	B *bitset.BitSet `json:"b"`
}

// MarshalJSON implements json.Marshaler interface.
func (f *BloomFilter) MarshalJSON() ([]byte, error) {
	return json.Marshal(bloomFilterJSON{f.m, f.k, f.b})
}

// UnmarshalJSON implements json.Unmarshaler interface.
func (f *BloomFilter) UnmarshalJSON(data []byte) error {
	var j bloomFilterJSON
	err := json.Unmarshal(data, &j)
	if err != nil {
		return err
	}
	f.m = j.M
	f.k = j.K
	f.b = j.B
	return nil
}

// WriteTo writes a binary representation of the BloomFilter to an i/o stream.
// It returns the number of bytes written.
func (f *BloomFilter) WriteTo(stream io.Writer) (int64, error) {
	err := binary.Write(stream, binary.BigEndian, uint64(f.m))
	if err != nil {
		return 0, err
	}
	err = binary.Write(stream, binary.BigEndian, uint64(f.k))
	if err != nil {
		return 0, err
	}
	numBytes, err := f.b.WriteTo(stream)
	return numBytes + int64(2*binary.Size(uint64(0))), err
}

// ReadFrom reads a binary representation of the BloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (f *BloomFilter) ReadFrom(stream io.Reader) (int64, error) {
	var m, k uint64
	err := binary.Read(stream, binary.BigEndian, &m)
	if err != nil {
		return 0, err
	}
	err = binary.Read(stream, binary.BigEndian, &k)
	if err != nil {
		return 0, err
	}
	b := &bitset.BitSet{}
	numBytes, err := b.ReadFrom(stream)
	if err != nil {
		return 0, err
	}
	f.m = uint(m)
	f.k = uint(k)
	f.b = b
	return numBytes + int64(2*binary.Size(uint64(0))), nil
}

// GobEncode implements gob.GobEncoder interface.
func (f *BloomFilter) GobEncode() ([]byte, error) {
	var buf bytes.Buffer
	_, err := f.WriteTo(&buf)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

// GobDecode implements gob.GobDecoder interface.
func (f *BloomFilter) GobDecode(data []byte) error {
	buf := bytes.NewBuffer(data)
	_, err := f.ReadFrom(buf)

	return err
}

// Equal tests for the equality of two Bloom filters
func (f *BloomFilter) Equal(g *BloomFilter) bool {
	return f.m == g.m && f.k == g.k && f.b.Equal(g.b)
}
<|endoftext|>


<commit_before>// Copyright 2015 The draw2d Authors. All rights reserved.
// created: 26/06/2015 by Stani Michiels
// TODO: dashed line

package draw2dpdf

import (
	"bytes"
	"image"
	"image/color"
	"image/png"
	"log"
	"math"
	"os"
	"strconv"

	"code.google.com/p/freetype-go/freetype/truetype"

	"github.com/jung-kurt/gofpdf"
	"github.com/llgcode/draw2d"
)

const (
	// DPI of a pdf document is fixed at 72.
	DPI  = 72
	c255 = 255.0 / 65535.0
)

var (
	caps = map[draw2d.Cap]string{
		draw2d.RoundCap:  "round",
		draw2d.ButtCap:   "butt",
		draw2d.SquareCap: "square"}
	joins = map[draw2d.Join]string{
		draw2d.RoundJoin: "round",
		draw2d.BevelJoin: "bevel",
		draw2d.MiterJoin: "miter",
	}
	imageCount uint32
	white      color.Color = color.RGBA{255, 255, 255, 255}
)

// NewPdf creates a new pdf document with the draw2d fontfolder, adds
// a page and set fill color to white.
func NewPdf(orientationStr, unitStr, sizeStr string) *gofpdf.Fpdf {
	pdf := gofpdf.New(orientationStr, unitStr, sizeStr, draw2d.GetFontFolder())
	// to be compatible with draw2d
	pdf.SetMargins(0, 0, 0)
	pdf.SetDrawColor(0, 0, 0)
	pdf.SetFillColor(255, 255, 255)
	pdf.SetLineCapStyle("round")
	pdf.SetLineJoinStyle("round")
	pdf.SetLineWidth(1)
	pdf.AddPage()
	return pdf
}

// rgb converts a color (used by draw2d) into 3 int (used by gofpdf)
func rgb(c color.Color) (int, int, int) {
	r, g, b, _ := c.RGBA()
	return int(float64(r) * c255), int(float64(g) * c255), int(float64(b) * c255)
}

// clearRect draws a white rectangle
func clearRect(gc *GraphicContext, x1, y1, x2, y2 float64) {
	// save state
	f := gc.Current.FillColor
	x, y := gc.pdf.GetXY()
	// cover page with white rectangle
	gc.SetFillColor(white)
	draw2d.Rect(gc, x1, y1, x2, y2)
	gc.Fill()
	// restore state
	gc.SetFillColor(f)
	gc.pdf.MoveTo(x, y)
}

// GraphicContext implements the draw2d.GraphicContext interface
// It provides draw2d with a pdf backend (based on gofpdf)
type GraphicContext struct {
	*draw2d.StackGraphicContext
	pdf *gofpdf.Fpdf
	DPI int
}

// NewGraphicContext creates a new pdf GraphicContext
func NewGraphicContext(pdf *gofpdf.Fpdf) *GraphicContext {
	gc := &GraphicContext{draw2d.NewStackGraphicContext(), pdf, DPI}
	gc.SetDPI(DPI)
	return gc
}

// DrawImage draws an image as PNG
// TODO: add type (tp) as parameter to argument list?
func (gc *GraphicContext) DrawImage(image image.Image) {
	name := strconv.Itoa(int(imageCount))
	imageCount++
	tp := "PNG" // "JPG", "JPEG", "PNG" and "GIF"
	b := &bytes.Buffer{}
	png.Encode(b, image)
	gc.pdf.RegisterImageReader(name, tp, b)
	bounds := image.Bounds()
	x0, y0 := float64(bounds.Min.X), float64(bounds.Min.Y)
	w, h := float64(bounds.Dx()), float64(bounds.Dy())
	gc.pdf.Image(name, x0, y0, w, h, false, tp, 0, "")
}

// Clear draws a white rectangle over the whole page
func (gc *GraphicContext) Clear() {
	width, height := gc.pdf.GetPageSize()
	clearRect(gc, 0, 0, width, height)
}

// ClearRect draws a white rectangle over the specified area.
// Samples: line.
func (gc *GraphicContext) ClearRect(x1, y1, x2, y2 int) {
	clearRect(gc, float64(x1), float64(y1), float64(x2), float64(y2))
}

// recalc recalculates scale and bounds values from the font size, screen
// resolution and font metrics, and invalidates the glyph cache.
func (gc *GraphicContext) recalc() {
	// TODO: resolve properly the font size for pdf and bitmap
	gc.Current.Scale = 3 * float64(gc.DPI) / 72
}

// SetDPI sets the DPI which influences the font size.
func (gc *GraphicContext) SetDPI(dpi int) {
	gc.DPI = dpi
	gc.recalc()
}

// GetDPI returns the DPI which influences the font size.
// (Note that gofpdf uses a fixed dpi of 72:
// https://godoc.org/code.google.com/p/gofpdf#Fpdf.PointConvert)
func (gc *GraphicContext) GetDPI() int {
	return gc.DPI
}

// GetStringBounds returns the approximate pixel bounds of the string s at x, y.
// The left edge of the em square of the first character of s
// and the baseline intersect at 0, 0 in the returned coordinates.
// Therefore the top and left coordinates may well be negative.
func (gc *GraphicContext) GetStringBounds(s string) (left, top, right, bottom float64) {
	_, h := gc.pdf.GetFontSize()
	d := gc.pdf.GetFontDesc("", "")
	if d.Ascent == 0 {
		// not defined (standard font?), use average of 81%
		top = 0.81 * h
	} else {
		top = -float64(d.Ascent) * h / float64(d.Ascent-d.Descent)
	}
	bottom = top + h
	margin := gc.pdf.GetCellMargin()
	return -margin, top, -margin + gc.pdf.GetStringWidth(s), bottom
}

// CreateStringPath creates a path from the string s at x, y, and returns the string width.
func (gc *GraphicContext) CreateStringPath(text string, x, y float64) (cursor float64) {
	//fpdf uses the top left corner
	left, top, right, bottom := gc.GetStringBounds(text)
	w := right - left
	h := bottom - top
	// gc.pdf.SetXY(x, y-h) do not use this as y-h might be negative
	gc.pdf.MoveTo(x+left, y+top)
	gc.pdf.CellFormat(w, h, text, "", 0, "BL", false, 0, "")
	return w
}

// FillString draws a string at 0, 0
func (gc *GraphicContext) FillString(text string) (cursor float64) {
	return gc.FillStringAt(text, 0, 0)
}

// FillStringAt draws a string at x, y
func (gc *GraphicContext) FillStringAt(text string, x, y float64) (cursor float64) {
	return gc.CreateStringPath(text, x, y)
}

// StrokeString draws a string at 0, 0 (stroking is unsupported,
// string will be filled)
func (gc *GraphicContext) StrokeString(text string) (cursor float64) {
	return gc.StrokeStringAt(text, 0, 0)
}

// StrokeStringAt draws a string at x, y (stroking is unsupported,
// string will be filled)
func (gc *GraphicContext) StrokeStringAt(text string, x, y float64) (cursor float64) {
	return gc.CreateStringPath(text, x, y)
}

// Stroke strokes the paths with the color specified by SetStrokeColor
func (gc *GraphicContext) Stroke(paths ...*draw2d.PathStorage) {
	gc.draw("D", paths...)
}

// Fill fills the paths with the color specified by SetFillColor
func (gc *GraphicContext) Fill(paths ...*draw2d.PathStorage) {
	style := "F"
	if !gc.Current.FillRule.UseNonZeroWinding() {
		style += "*"
	}
	gc.draw(style, paths...)
}

// FillStroke first fills the paths and than strokes them
func (gc *GraphicContext) FillStroke(paths ...*draw2d.PathStorage) {
	style := "FD"
	if !gc.Current.FillRule.UseNonZeroWinding() {
		style += "*"
	}
	gc.draw(style, paths...)
}

var logger = log.New(os.Stdout, "", log.Lshortfile)

// draw fills and/or strokes paths
func (gc *GraphicContext) draw(style string, paths ...*draw2d.PathStorage) {
	paths = append(paths, gc.Current.Path)
	pathConverter := NewPathConverter(gc.pdf)
	pathConverter.Convert(paths...)
	gc.pdf.DrawPath(style)
	gc.Current.Path = draw2d.NewPathStorage()
}

// overwrite StackGraphicContext methods

// SetStrokeColor sets the stroke color
func (gc *GraphicContext) SetStrokeColor(c color.Color) {
	gc.StackGraphicContext.SetStrokeColor(c)
	gc.pdf.SetDrawColor(rgb(c))
}

// SetFillColor sets the fill and text color
func (gc *GraphicContext) SetFillColor(c color.Color) {
	gc.StackGraphicContext.SetFillColor(c)
	gc.pdf.SetFillColor(rgb(c))
	gc.pdf.SetTextColor(rgb(c))
}

// SetFont is unsupported by the pdf graphic context, use SetFontData
// instead.
func (gc *GraphicContext) SetFont(font *truetype.Font) {
	// TODO: what to do with this api conflict between draw2d and gofpdf?!
}

// SetFontData sets the current font used to draw text. Always use
// this method, as SetFont is unsupported by the pdf graphic context.
// It is mandatory to call this method at least once before printing
// text or the resulting document will not be valid.
// It is necessary to generate a font definition file first with the
// makefont utility. It is not necessary to call this function for the
// core PDF fonts (courier, helvetica, times, zapfdingbats).
// go get github.com/jung-kurt/gofpdf/makefont
// http://godoc.org/github.com/jung-kurt/gofpdf#Fpdf.AddFont
func (gc *GraphicContext) SetFontData(fontData draw2d.FontData) {
	// TODO: call Makefont embed if json file does not exist yet
	gc.StackGraphicContext.SetFontData(fontData)
	var style string
	if fontData.Style&draw2d.FontStyleBold != 0 {
		style += "B"
	}
	if fontData.Style&draw2d.FontStyleItalic != 0 {
		style += "I"
	}
	fn := draw2d.FontFileName(fontData)
	fn = fn[:len(fn)-4]
	size, _ := gc.pdf.GetFontSize()
	gc.pdf.AddFont(fontData.Name, style, fn+".json")
	gc.pdf.SetFont(fontData.Name, style, size)
}

// SetFontSize sets the font size in points (as in ``a 12 point font'').
// TODO: resolve this with ImgGraphicContext (now done with gc.Current.Scale)
func (gc *GraphicContext) SetFontSize(fontSize float64) {
	gc.StackGraphicContext.SetFontSize(fontSize)
	gc.recalc()
	gc.pdf.SetFontSize(fontSize * gc.Current.Scale)
}

// SetLineDash sets the line dash pattern
func (gc *GraphicContext) SetLineDash(Dash []float64, DashOffset float64) {
	gc.StackGraphicContext.SetLineDash(Dash, DashOffset)
	gc.pdf.SetDashPattern(Dash, DashOffset)
}

// SetLineWidth sets the line width
func (gc *GraphicContext) SetLineWidth(LineWidth float64) {
	gc.StackGraphicContext.SetLineWidth(LineWidth)
	gc.pdf.SetLineWidth(LineWidth)
}

// SetLineCap sets the line cap (round, but or square)
func (gc *GraphicContext) SetLineCap(Cap draw2d.Cap) {
	gc.StackGraphicContext.SetLineCap(Cap)
	gc.pdf.SetLineCapStyle(caps[Cap])
}

// SetLineJoin sets the line cap (round, bevel or miter)
func (gc *GraphicContext) SetLineJoin(Join draw2d.Join) {
	gc.StackGraphicContext.SetLineJoin(Join)
	gc.pdf.SetLineJoinStyle(joins[Join])
}

// Transformations

// Scale generally scales the following text, drawings and images.
// sx and sy are the scaling factors for width and height.
// This must be placed between gc.Save() and gc.Restore(), otherwise
// the pdf is invalid.
func (gc *GraphicContext) Scale(sx, sy float64) {
	gc.StackGraphicContext.Scale(sx, sy)
	gc.pdf.TransformScale(sx*100, sy*100, 0, 0)
}

// Rotate rotates the following text, drawings and images.
// Angle is specified in radians and measured clockwise from the
// 3 o'clock position.
// This must be placed between gc.Save() and gc.Restore(), otherwise
// the pdf is invalid.
func (gc *GraphicContext) Rotate(angle float64) {
	gc.StackGraphicContext.Rotate(angle)
	gc.pdf.TransformRotate(-angle*180/math.Pi, 0, 0)
}

// Translate moves the following text, drawings and images
// horizontally and vertically by the amounts specified by tx and ty.
// This must be placed between gc.Save() and gc.Restore(), otherwise
// the pdf is invalid.
func (gc *GraphicContext) Translate(tx, ty float64) {
	gc.StackGraphicContext.Translate(tx, ty)
	gc.pdf.TransformTranslate(tx, ty)
}

// Save saves the current context stack
// (transformation, font, color,...).
func (gc *GraphicContext) Save() {
	gc.StackGraphicContext.Save()
	gc.pdf.TransformBegin()
}

// Restore restores the current context stack
// (transformation, color,...). Restoring the font is not supported.
func (gc *GraphicContext) Restore() {
	gc.pdf.TransformEnd()
	gc.StackGraphicContext.Restore()
	c := gc.Current
	gc.SetFontSize(c.FontSize)
	// gc.SetFontData(c.FontData) unsupported, causes bug (do not enable)
	gc.SetLineWidth(c.LineWidth)
	gc.SetStrokeColor(c.StrokeColor)
	gc.SetFillColor(c.FillColor)
	gc.SetFillRule(c.FillRule)
	// gc.SetLineDash(c.Dash, c.DashOffset) // TODO
	gc.SetLineCap(c.Cap)
	gc.SetLineJoin(c.Join)
	// c.Path unsupported
	// c.Font unsupported
}
<commit_msg>fix horizontal alignment of text<commit_after>// Copyright 2015 The draw2d Authors. All rights reserved.
// created: 26/06/2015 by Stani Michiels
// TODO: dashed line

package draw2dpdf

import (
	"bytes"
	"image"
	"image/color"
	"image/png"
	"log"
	"math"
	"os"
	"strconv"

	"code.google.com/p/freetype-go/freetype/truetype"

	"github.com/jung-kurt/gofpdf"
	"github.com/llgcode/draw2d"
)

const (
	// DPI of a pdf document is fixed at 72.
	DPI  = 72
	c255 = 255.0 / 65535.0
)

var (
	caps = map[draw2d.Cap]string{
		draw2d.RoundCap:  "round",
		draw2d.ButtCap:   "butt",
		draw2d.SquareCap: "square"}
	joins = map[draw2d.Join]string{
		draw2d.RoundJoin: "round",
		draw2d.BevelJoin: "bevel",
		draw2d.MiterJoin: "miter",
	}
	imageCount uint32
	white      color.Color = color.RGBA{255, 255, 255, 255}
)

// NewPdf creates a new pdf document with the draw2d fontfolder, adds
// a page and set fill color to white.
func NewPdf(orientationStr, unitStr, sizeStr string) *gofpdf.Fpdf {
	pdf := gofpdf.New(orientationStr, unitStr, sizeStr, draw2d.GetFontFolder())
	// to be compatible with draw2d
	pdf.SetMargins(0, 0, 0)
	pdf.SetDrawColor(0, 0, 0)
	pdf.SetFillColor(255, 255, 255)
	pdf.SetLineCapStyle("round")
	pdf.SetLineJoinStyle("round")
	pdf.SetLineWidth(1)
	pdf.AddPage()
	return pdf
}

// rgb converts a color (used by draw2d) into 3 int (used by gofpdf)
func rgb(c color.Color) (int, int, int) {
	r, g, b, _ := c.RGBA()
	return int(float64(r) * c255), int(float64(g) * c255), int(float64(b) * c255)
}

// clearRect draws a white rectangle
func clearRect(gc *GraphicContext, x1, y1, x2, y2 float64) {
	// save state
	f := gc.Current.FillColor
	x, y := gc.pdf.GetXY()
	// cover page with white rectangle
	gc.SetFillColor(white)
	draw2d.Rect(gc, x1, y1, x2, y2)
	gc.Fill()
	// restore state
	gc.SetFillColor(f)
	gc.pdf.MoveTo(x, y)
}

// GraphicContext implements the draw2d.GraphicContext interface
// It provides draw2d with a pdf backend (based on gofpdf)
type GraphicContext struct {
	*draw2d.StackGraphicContext
	pdf *gofpdf.Fpdf
	DPI int
}

// NewGraphicContext creates a new pdf GraphicContext
func NewGraphicContext(pdf *gofpdf.Fpdf) *GraphicContext {
	gc := &GraphicContext{draw2d.NewStackGraphicContext(), pdf, DPI}
	gc.SetDPI(DPI)
	return gc
}

// DrawImage draws an image as PNG
// TODO: add type (tp) as parameter to argument list?
func (gc *GraphicContext) DrawImage(image image.Image) {
	name := strconv.Itoa(int(imageCount))
	imageCount++
	tp := "PNG" // "JPG", "JPEG", "PNG" and "GIF"
	b := &bytes.Buffer{}
	png.Encode(b, image)
	gc.pdf.RegisterImageReader(name, tp, b)
	bounds := image.Bounds()
	x0, y0 := float64(bounds.Min.X), float64(bounds.Min.Y)
	w, h := float64(bounds.Dx()), float64(bounds.Dy())
	gc.pdf.Image(name, x0, y0, w, h, false, tp, 0, "")
}

// Clear draws a white rectangle over the whole page
func (gc *GraphicContext) Clear() {
	width, height := gc.pdf.GetPageSize()
	clearRect(gc, 0, 0, width, height)
}

// ClearRect draws a white rectangle over the specified area.
// Samples: line.
func (gc *GraphicContext) ClearRect(x1, y1, x2, y2 int) {
	clearRect(gc, float64(x1), float64(y1), float64(x2), float64(y2))
}

// recalc recalculates scale and bounds values from the font size, screen
// resolution and font metrics, and invalidates the glyph cache.
func (gc *GraphicContext) recalc() {
	// TODO: resolve properly the font size for pdf and bitmap
	gc.Current.Scale = 3 * float64(gc.DPI) / 72
}

// SetDPI sets the DPI which influences the font size.
func (gc *GraphicContext) SetDPI(dpi int) {
	gc.DPI = dpi
	gc.recalc()
}

// GetDPI returns the DPI which influences the font size.
// (Note that gofpdf uses a fixed dpi of 72:
// https://godoc.org/code.google.com/p/gofpdf#Fpdf.PointConvert)
func (gc *GraphicContext) GetDPI() int {
	return gc.DPI
}

// GetStringBounds returns the approximate pixel bounds of the string s at x, y.
// The left edge of the em square of the first character of s
// and the baseline intersect at 0, 0 in the returned coordinates.
// Therefore the top and left coordinates may well be negative.
func (gc *GraphicContext) GetStringBounds(s string) (left, top, right, bottom float64) {
	_, h := gc.pdf.GetFontSize()
	d := gc.pdf.GetFontDesc("", "")
	if d.Ascent == 0 {
		// not defined (standard font?), use average of 81%
		top = 0.81 * h
	} else {
		top = -float64(d.Ascent) * h / float64(d.Ascent-d.Descent)
	}
	return 0, top, gc.pdf.GetStringWidth(s), top + h
}

// CreateStringPath creates a path from the string s at x, y, and returns the string width.
func (gc *GraphicContext) CreateStringPath(text string, x, y float64) (cursor float64) {
	//fpdf uses the top left corner
	left, top, right, bottom := gc.GetStringBounds(text)
	w := right - left
	h := bottom - top
	// gc.pdf.SetXY(x, y-h) do not use this as y-h might be negative
	margin := gc.pdf.GetCellMargin()
	gc.pdf.MoveTo(x-left-margin, y+top)
	gc.pdf.CellFormat(w, h, text, "", 0, "BL", false, 0, "")
	return w
}

// FillString draws a string at 0, 0
func (gc *GraphicContext) FillString(text string) (cursor float64) {
	return gc.FillStringAt(text, 0, 0)
}

// FillStringAt draws a string at x, y
func (gc *GraphicContext) FillStringAt(text string, x, y float64) (cursor float64) {
	return gc.CreateStringPath(text, x, y)
}

// StrokeString draws a string at 0, 0 (stroking is unsupported,
// string will be filled)
func (gc *GraphicContext) StrokeString(text string) (cursor float64) {
	return gc.StrokeStringAt(text, 0, 0)
}

// StrokeStringAt draws a string at x, y (stroking is unsupported,
// string will be filled)
func (gc *GraphicContext) StrokeStringAt(text string, x, y float64) (cursor float64) {
	return gc.CreateStringPath(text, x, y)
}

// Stroke strokes the paths with the color specified by SetStrokeColor
func (gc *GraphicContext) Stroke(paths ...*draw2d.PathStorage) {
	gc.draw("D", paths...)
}

// Fill fills the paths with the color specified by SetFillColor
func (gc *GraphicContext) Fill(paths ...*draw2d.PathStorage) {
	style := "F"
	if !gc.Current.FillRule.UseNonZeroWinding() {
		style += "*"
	}
	gc.draw(style, paths...)
}

// FillStroke first fills the paths and than strokes them
func (gc *GraphicContext) FillStroke(paths ...*draw2d.PathStorage) {
	style := "FD"
	if !gc.Current.FillRule.UseNonZeroWinding() {
		style += "*"
	}
	gc.draw(style, paths...)
}

var logger = log.New(os.Stdout, "", log.Lshortfile)

// draw fills and/or strokes paths
func (gc *GraphicContext) draw(style string, paths ...*draw2d.PathStorage) {
	paths = append(paths, gc.Current.Path)
	pathConverter := NewPathConverter(gc.pdf)
	pathConverter.Convert(paths...)
	gc.pdf.DrawPath(style)
	gc.Current.Path = draw2d.NewPathStorage()
}

// overwrite StackGraphicContext methods

// SetStrokeColor sets the stroke color
func (gc *GraphicContext) SetStrokeColor(c color.Color) {
	gc.StackGraphicContext.SetStrokeColor(c)
	gc.pdf.SetDrawColor(rgb(c))
}

// SetFillColor sets the fill and text color
func (gc *GraphicContext) SetFillColor(c color.Color) {
	gc.StackGraphicContext.SetFillColor(c)
	gc.pdf.SetFillColor(rgb(c))
	gc.pdf.SetTextColor(rgb(c))
}

// SetFont is unsupported by the pdf graphic context, use SetFontData
// instead.
func (gc *GraphicContext) SetFont(font *truetype.Font) {
	// TODO: what to do with this api conflict between draw2d and gofpdf?!
}

// SetFontData sets the current font used to draw text. Always use
// this method, as SetFont is unsupported by the pdf graphic context.
// It is mandatory to call this method at least once before printing
// text or the resulting document will not be valid.
// It is necessary to generate a font definition file first with the
// makefont utility. It is not necessary to call this function for the
// core PDF fonts (courier, helvetica, times, zapfdingbats).
// go get github.com/jung-kurt/gofpdf/makefont
// http://godoc.org/github.com/jung-kurt/gofpdf#Fpdf.AddFont
func (gc *GraphicContext) SetFontData(fontData draw2d.FontData) {
	// TODO: call Makefont embed if json file does not exist yet
	gc.StackGraphicContext.SetFontData(fontData)
	var style string
	if fontData.Style&draw2d.FontStyleBold != 0 {
		style += "B"
	}
	if fontData.Style&draw2d.FontStyleItalic != 0 {
		style += "I"
	}
	fn := draw2d.FontFileName(fontData)
	fn = fn[:len(fn)-4]
	size, _ := gc.pdf.GetFontSize()
	gc.pdf.AddFont(fontData.Name, style, fn+".json")
	gc.pdf.SetFont(fontData.Name, style, size)
}

// SetFontSize sets the font size in points (as in ``a 12 point font'').
// TODO: resolve this with ImgGraphicContext (now done with gc.Current.Scale)
func (gc *GraphicContext) SetFontSize(fontSize float64) {
	gc.StackGraphicContext.SetFontSize(fontSize)
	gc.recalc()
	gc.pdf.SetFontSize(fontSize * gc.Current.Scale)
}

// SetLineDash sets the line dash pattern
func (gc *GraphicContext) SetLineDash(Dash []float64, DashOffset float64) {
	gc.StackGraphicContext.SetLineDash(Dash, DashOffset)
	gc.pdf.SetDashPattern(Dash, DashOffset)
}

// SetLineWidth sets the line width
func (gc *GraphicContext) SetLineWidth(LineWidth float64) {
	gc.StackGraphicContext.SetLineWidth(LineWidth)
	gc.pdf.SetLineWidth(LineWidth)
}

// SetLineCap sets the line cap (round, but or square)
func (gc *GraphicContext) SetLineCap(Cap draw2d.Cap) {
	gc.StackGraphicContext.SetLineCap(Cap)
	gc.pdf.SetLineCapStyle(caps[Cap])
}

// SetLineJoin sets the line cap (round, bevel or miter)
func (gc *GraphicContext) SetLineJoin(Join draw2d.Join) {
	gc.StackGraphicContext.SetLineJoin(Join)
	gc.pdf.SetLineJoinStyle(joins[Join])
}

// Transformations

// Scale generally scales the following text, drawings and images.
// sx and sy are the scaling factors for width and height.
// This must be placed between gc.Save() and gc.Restore(), otherwise
// the pdf is invalid.
func (gc *GraphicContext) Scale(sx, sy float64) {
	gc.StackGraphicContext.Scale(sx, sy)
	gc.pdf.TransformScale(sx*100, sy*100, 0, 0)
}

// Rotate rotates the following text, drawings and images.
// Angle is specified in radians and measured clockwise from the
// 3 o'clock position.
// This must be placed between gc.Save() and gc.Restore(), otherwise
// the pdf is invalid.
func (gc *GraphicContext) Rotate(angle float64) {
	gc.StackGraphicContext.Rotate(angle)
	gc.pdf.TransformRotate(-angle*180/math.Pi, 0, 0)
}

// Translate moves the following text, drawings and images
// horizontally and vertically by the amounts specified by tx and ty.
// This must be placed between gc.Save() and gc.Restore(), otherwise
// the pdf is invalid.
func (gc *GraphicContext) Translate(tx, ty float64) {
	gc.StackGraphicContext.Translate(tx, ty)
	gc.pdf.TransformTranslate(tx, ty)
}

// Save saves the current context stack
// (transformation, font, color,...).
func (gc *GraphicContext) Save() {
	gc.StackGraphicContext.Save()
	gc.pdf.TransformBegin()
}

// Restore restores the current context stack
// (transformation, color,...). Restoring the font is not supported.
func (gc *GraphicContext) Restore() {
	gc.pdf.TransformEnd()
	gc.StackGraphicContext.Restore()
	c := gc.Current
	gc.SetFontSize(c.FontSize)
	// gc.SetFontData(c.FontData) unsupported, causes bug (do not enable)
	gc.SetLineWidth(c.LineWidth)
	gc.SetStrokeColor(c.StrokeColor)
	gc.SetFillColor(c.FillColor)
	gc.SetFillRule(c.FillRule)
	// gc.SetLineDash(c.Dash, c.DashOffset) // TODO
	gc.SetLineCap(c.Cap)
	gc.SetLineJoin(c.Join)
	// c.Path unsupported
	// c.Font unsupported
}
<|endoftext|>