<commit_before>// archive is package that helps create archives in a format that
// Atlas expects with its various upload endpoints.
package archive

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

//
type Archive struct {
	io.ReadCloser

	Size     int64
	Metadata map[string]string
}

// ArchiveOpts are the options for defining how the archive will be built.
type ArchiveOpts struct {
	// Exclude and Include are filters of files to include/exclude in
	// the archive when creating it from a directory. These filters should
	// be relative to the packaging directory and should be basic glob
	// patterns.
	Exclude []string
	Include []string

	// Extra is a mapping of extra files to include within the archive. The
	// key should be the path within the archive and the value should be
	// an absolute path to the file to put into the archive. These extra
	// files will override any other files in the archive.
	Extra map[string]string

	// VCS, if true, will detect and use a VCS system to determien what
	// files to include the archive.
	VCS bool
}

// IsSet says whether any options were set.
func (o *ArchiveOpts) IsSet() bool {
	return len(o.Exclude) > 0 || len(o.Include) > 0 || o.VCS
}

// Archive takes the given path and ArchiveOpts and archives it.
//
// The archive will be fully completed and put into a temporary file.
// This must be done to retrieve the content length of the archive which
// is need for almost all operations involving archives with Atlas. Because
// of this, sufficient disk space will be required to buffer the archive.
func CreateArchive(path string, opts *ArchiveOpts) (*Archive, error) {
	// io.ReadCloser, int64, error

	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	// Direct file paths cannot have archive options
	if !fi.IsDir() && opts.IsSet() {
		return nil, fmt.Errorf(
			"Options such as exclude, include, and VCS can't be set when " +
				"the path is a file.")
	}

	if fi.IsDir() {
		return archiveDir(path, opts)
	} else {
		return archiveFile(path, opts)
	}
}

func archiveFile(path string, opts *ArchiveOpts) (*Archive, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	if _, err := gzip.NewReader(f); err == nil {
		// Reset the read offset for future reading
		if _, err := f.Seek(0, 0); err != nil {
			f.Close()
			return nil, err
		}

		// Get the file info for the size
		fi, err := f.Stat()
		if err != nil {
			f.Close()
			return nil, err
		}

		// This is a gzip file, let it through.
		return &Archive{ReadCloser: f, Size: fi.Size()}, nil
	}

	// Close the file, no use for it anymore
	f.Close()

	// We have a single file that is not gzipped. Compress it.
	path, err = filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	// Act like we're compressing a directory, but only include this one
	// file.
	return archiveDir(filepath.Dir(path), &ArchiveOpts{
		Include: []string{filepath.Base(path)},
	})
}

func archiveDir(root string, opts *ArchiveOpts) (*Archive, error) {
	var vcsInclude []string
	if opts.VCS {
		var err error
		vcsInclude, err = vcsFiles(root)
		if err != nil {
			return nil, err
		}
	}

	// Create the temporary file that we'll send the archive data to.
	archiveF, err := ioutil.TempFile("", "atlas-archive")
	if err != nil {
		return nil, err
	}

	// Create the wrapper for the result which will automatically
	// remove the temporary file on close.
	archiveWrapper := &readCloseRemover{F: archiveF}

	// Buffer the writer so that we can push as much data to disk at
	// a time as possible. 4M should be good.
	bufW := bufio.NewWriterSize(archiveF, 4096*1024)

	// Gzip compress all the output data
	gzipW := gzip.NewWriter(bufW)

	// Tar the file contents
	tarW := tar.NewWriter(gzipW)

	// Build the function that'll do all the compression
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Get the relative path from the path since it contains the root
		// plus the path.
		subpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if subpath == "." {
			return nil
		}

		// If we have a list of VCS files, check that first
		skip := false
		if len(vcsInclude) > 0 {
			skip = true
			for _, f := range vcsInclude {
				if f == subpath {
					skip = false
					break
				}

				if info.IsDir() && strings.HasPrefix(f, subpath+"/") {
					skip = false
					break
				}
			}
		}

		// If include is present, we only include what is listed
		if len(opts.Include) > 0 {
			skip = true
			for _, include := range opts.Include {
				match, err := filepath.Match(include, subpath)
				if err != nil {
					return err
				}
				if match {
					skip = false
					break
				}
			}
		}

		// If exclude, it is one last gate to excluding files
		for _, exclude := range opts.Exclude {
			match, err := filepath.Match(exclude, subpath)
			if err != nil {
				return err
			}
			if match {
				skip = true
				break
			}
		}

		// If we have to skip this file, then skip it, properly skipping
		// children if we're a directory.
		if skip {
			if info.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"Failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = subpath
		if info.IsDir() {
			header.Name += "/"
		}

		// Write the header first to the archive.
		if err := tarW.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"Failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"Failed opening file '%s' to write compressed archive.", path)
		}
		defer f.Close()

		if _, err = io.Copy(tarW, f); err != nil {
			return fmt.Errorf(
				"Failed copying file to archive: %s", path)
		}

		return nil
	}

	// First, walk the path and do the normal files
	werr := filepath.Walk(root, walkFn)
	if werr == nil {
		// If that succeeded, handle the extra files
		werr = copyExtras(tarW, opts.Extra)
	}

	// Attempt to close all the things. If we get an error on the way
	// and we haven't had an error yet, then record that as the critical
	// error. But we still try to close everything.

	// Close the tar writer
	if err := tarW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Close the gzip writer
	if err := gzipW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Flush the buffer
	if err := bufW.Flush(); err != nil && werr == nil {
		werr = err
	}

	// If we had an error, then close the file (removing it) and
	// return the error.
	if werr != nil {
		archiveWrapper.Close()
		return nil, werr
	}

	// Seek to the beginning
	if _, err := archiveWrapper.F.Seek(0, 0); err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	// Get the file information so we can get the size
	fi, err := archiveWrapper.F.Stat()
	if err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	return &Archive{
		ReadCloser: archiveWrapper,
		Size:       fi.Size(),
	}, nil
}

func copyExtras(w *tar.Writer, extra map[string]string) error {
	for entry, path := range extra {
		info, err := os.Stat(path)
		if err != nil {
			return err
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"Failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = entry

		// Write the header first to the archive.
		if err := w.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"Failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"Failed opening file '%s' to write compressed archive.", path)
		}

		_, err = io.Copy(w, f)
		f.Close()
		if err != nil {
			return fmt.Errorf(
				"Failed copying file to archive: %s", path)
		}
	}

	return nil
}

// readCloseRemover is an io.ReadCloser implementation that will remove
// the file on Close(). We use this to clean up our temporary file for
// the archive.
type readCloseRemover struct {
	F *os.File
}

func (r *readCloseRemover) Read(p []byte) (int, error) {
	return r.F.Read(p)
}

func (r *readCloseRemover) Close() error {
	// First close the file
	err := r.F.Close()

	// Next make sure to remove it, or at least try, regardless of error
	// above.
	os.Remove(r.F.Name())

	return err
}
<commit_msg>Spell determine correctly<commit_after>// archive is package that helps create archives in a format that
// Atlas expects with its various upload endpoints.
package archive

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

//
type Archive struct {
	io.ReadCloser

	Size     int64
	Metadata map[string]string
}

// ArchiveOpts are the options for defining how the archive will be built.
type ArchiveOpts struct {
	// Exclude and Include are filters of files to include/exclude in
	// the archive when creating it from a directory. These filters should
	// be relative to the packaging directory and should be basic glob
	// patterns.
	Exclude []string
	Include []string

	// Extra is a mapping of extra files to include within the archive. The
	// key should be the path within the archive and the value should be
	// an absolute path to the file to put into the archive. These extra
	// files will override any other files in the archive.
	Extra map[string]string

	// VCS, if true, will detect and use a VCS system to determine what
	// files to include the archive.
	VCS bool
}

// IsSet says whether any options were set.
func (o *ArchiveOpts) IsSet() bool {
	return len(o.Exclude) > 0 || len(o.Include) > 0 || o.VCS
}

// Archive takes the given path and ArchiveOpts and archives it.
//
// The archive will be fully completed and put into a temporary file.
// This must be done to retrieve the content length of the archive which
// is need for almost all operations involving archives with Atlas. Because
// of this, sufficient disk space will be required to buffer the archive.
func CreateArchive(path string, opts *ArchiveOpts) (*Archive, error) {
	// io.ReadCloser, int64, error

	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	// Direct file paths cannot have archive options
	if !fi.IsDir() && opts.IsSet() {
		return nil, fmt.Errorf(
			"Options such as exclude, include, and VCS can't be set when " +
				"the path is a file.")
	}

	if fi.IsDir() {
		return archiveDir(path, opts)
	} else {
		return archiveFile(path, opts)
	}
}

func archiveFile(path string, opts *ArchiveOpts) (*Archive, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	if _, err := gzip.NewReader(f); err == nil {
		// Reset the read offset for future reading
		if _, err := f.Seek(0, 0); err != nil {
			f.Close()
			return nil, err
		}

		// Get the file info for the size
		fi, err := f.Stat()
		if err != nil {
			f.Close()
			return nil, err
		}

		// This is a gzip file, let it through.
		return &Archive{ReadCloser: f, Size: fi.Size()}, nil
	}

	// Close the file, no use for it anymore
	f.Close()

	// We have a single file that is not gzipped. Compress it.
	path, err = filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	// Act like we're compressing a directory, but only include this one
	// file.
	return archiveDir(filepath.Dir(path), &ArchiveOpts{
		Include: []string{filepath.Base(path)},
	})
}

func archiveDir(root string, opts *ArchiveOpts) (*Archive, error) {
	var vcsInclude []string
	if opts.VCS {
		var err error
		vcsInclude, err = vcsFiles(root)
		if err != nil {
			return nil, err
		}
	}

	// Create the temporary file that we'll send the archive data to.
	archiveF, err := ioutil.TempFile("", "atlas-archive")
	if err != nil {
		return nil, err
	}

	// Create the wrapper for the result which will automatically
	// remove the temporary file on close.
	archiveWrapper := &readCloseRemover{F: archiveF}

	// Buffer the writer so that we can push as much data to disk at
	// a time as possible. 4M should be good.
	bufW := bufio.NewWriterSize(archiveF, 4096*1024)

	// Gzip compress all the output data
	gzipW := gzip.NewWriter(bufW)

	// Tar the file contents
	tarW := tar.NewWriter(gzipW)

	// Build the function that'll do all the compression
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Get the relative path from the path since it contains the root
		// plus the path.
		subpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if subpath == "." {
			return nil
		}

		// If we have a list of VCS files, check that first
		skip := false
		if len(vcsInclude) > 0 {
			skip = true
			for _, f := range vcsInclude {
				if f == subpath {
					skip = false
					break
				}

				if info.IsDir() && strings.HasPrefix(f, subpath+"/") {
					skip = false
					break
				}
			}
		}

		// If include is present, we only include what is listed
		if len(opts.Include) > 0 {
			skip = true
			for _, include := range opts.Include {
				match, err := filepath.Match(include, subpath)
				if err != nil {
					return err
				}
				if match {
					skip = false
					break
				}
			}
		}

		// If exclude, it is one last gate to excluding files
		for _, exclude := range opts.Exclude {
			match, err := filepath.Match(exclude, subpath)
			if err != nil {
				return err
			}
			if match {
				skip = true
				break
			}
		}

		// If we have to skip this file, then skip it, properly skipping
		// children if we're a directory.
		if skip {
			if info.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"Failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = subpath
		if info.IsDir() {
			header.Name += "/"
		}

		// Write the header first to the archive.
		if err := tarW.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"Failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"Failed opening file '%s' to write compressed archive.", path)
		}
		defer f.Close()

		if _, err = io.Copy(tarW, f); err != nil {
			return fmt.Errorf(
				"Failed copying file to archive: %s", path)
		}

		return nil
	}

	// First, walk the path and do the normal files
	werr := filepath.Walk(root, walkFn)
	if werr == nil {
		// If that succeeded, handle the extra files
		werr = copyExtras(tarW, opts.Extra)
	}

	// Attempt to close all the things. If we get an error on the way
	// and we haven't had an error yet, then record that as the critical
	// error. But we still try to close everything.

	// Close the tar writer
	if err := tarW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Close the gzip writer
	if err := gzipW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Flush the buffer
	if err := bufW.Flush(); err != nil && werr == nil {
		werr = err
	}

	// If we had an error, then close the file (removing it) and
	// return the error.
	if werr != nil {
		archiveWrapper.Close()
		return nil, werr
	}

	// Seek to the beginning
	if _, err := archiveWrapper.F.Seek(0, 0); err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	// Get the file information so we can get the size
	fi, err := archiveWrapper.F.Stat()
	if err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	return &Archive{
		ReadCloser: archiveWrapper,
		Size:       fi.Size(),
	}, nil
}

func copyExtras(w *tar.Writer, extra map[string]string) error {
	for entry, path := range extra {
		info, err := os.Stat(path)
		if err != nil {
			return err
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"Failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = entry

		// Write the header first to the archive.
		if err := w.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"Failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"Failed opening file '%s' to write compressed archive.", path)
		}

		_, err = io.Copy(w, f)
		f.Close()
		if err != nil {
			return fmt.Errorf(
				"Failed copying file to archive: %s", path)
		}
	}

	return nil
}

// readCloseRemover is an io.ReadCloser implementation that will remove
// the file on Close(). We use this to clean up our temporary file for
// the archive.
type readCloseRemover struct {
	F *os.File
}

func (r *readCloseRemover) Read(p []byte) (int, error) {
	return r.F.Read(p)
}

func (r *readCloseRemover) Close() error {
	// First close the file
	err := r.F.Close()

	// Next make sure to remove it, or at least try, regardless of error
	// above.
	os.Remove(r.F.Name())

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"strconv"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"

	pbc "github.com/brotherlogic/cardserver/card"
	pbdi "github.com/brotherlogic/discovery/proto"
)

func getIP(servername string, ip string, port int) (string, int) {
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	registry := pbdi.NewDiscoveryServiceClient(conn)
	entry := pbdi.RegistryEntry{Name: servername}
	r, _ := registry.Discover(context.Background(), &entry)
	return r.Ip, int(r.Port)
}

func main() {
	c := InitFromFile("crontstore", "cron")
	dryRun := flag.Bool("dry_run", false, "Don't write anything.")
	quiet := flag.Bool("quiet", true, "Don't log owt.")
	flag.Parse()

	if *quiet {
		log.SetOutput(ioutil.Discard)
		grpclog.SetLogger(log.New(ioutil.Discard, "", -1))
	}

	cards := c.GetCards(c.last, time.Now())

	if *dryRun {
		log.Printf("Would write: %v", cards)
	} else {
		var host = flag.String("host", "192.168.86.34", "Hostname of server.")
		var port = flag.Int("port", 50055, "Port number of server")

		cServer, cPort := getIP("cardserver", *host, *port)
		conn, err := grpc.Dial(cServer+":"+strconv.Itoa(cPort), grpc.WithInsecure())
		if err != nil {
			log.Fatalf("Failure to dial cardserver (%v)", err)
		}
		defer conn.Close()
		client := pbc.NewCardServiceClient(conn)

		_, err = client.AddCards(context.Background(), &pbc.CardList{Cards: cards})
		if err != nil {
			log.Fatalf("Failure to add cards: %v", err)
		}
	}
}
<commit_msg>Hid discover fails<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"strconv"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"

	pbc "github.com/brotherlogic/cardserver/card"
	pbdi "github.com/brotherlogic/discovery/proto"
)

func getIP(servername string, ip string, port int) (string, int) {
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	registry := pbdi.NewDiscoveryServiceClient(conn)
	entry := pbdi.RegistryEntry{Name: servername}
	r, err := registry.Discover(context.Background(), &entry)

	if err != nil {
		return "", -1
	}

	return r.Ip, int(r.Port)
}

func main() {
	c := InitFromFile("crontstore", "cron")
	dryRun := flag.Bool("dry_run", false, "Don't write anything.")
	quiet := flag.Bool("quiet", true, "Don't log owt.")
	flag.Parse()

	if *quiet {
		log.SetOutput(ioutil.Discard)
		grpclog.SetLogger(log.New(ioutil.Discard, "", -1))
	}

	cards := c.GetCards(c.last, time.Now())

	if *dryRun {
		log.Printf("Would write: %v", cards)
	} else {
		var host = flag.String("host", "192.168.86.34", "Hostname of server.")
		var port = flag.Int("port", 50055, "Port number of server")

		cServer, cPort := getIP("cardserver", *host, *port)
		if cPort > 0 {
			conn, err := grpc.Dial(cServer+":"+strconv.Itoa(cPort), grpc.WithInsecure())
			if err != nil {
				log.Fatalf("Failure to dial cardserver (%v)", err)
			}
			defer conn.Close()
			client := pbc.NewCardServiceClient(conn)

			_, err = client.AddCards(context.Background(), &pbc.CardList{Cards: cards})
			if err != nil {
				log.Fatalf("Failure to add cards: %v", err)
			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>chore: re-enable tests in integration/sync_test.go (#7511)<commit_after><|endoftext|>


<commit_before><commit_msg>internal/lsp: fail test if command cannot be applied<commit_after><|endoftext|>


<commit_before>// Copyright 2015, 2016 Eris Industries (UK) Ltd.
// This file is part of Eris-RT

// Eris-RT is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Eris-RT is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Eris-RT.  If not, see <http://www.gnu.org/licenses/>.

package erismint

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"sync"

	tendermint_events "github.com/tendermint/go-events"
	rpcclient "github.com/tendermint/go-rpc/client"
	wire "github.com/tendermint/go-wire"
	ctypes "github.com/tendermint/tendermint/rpc/core/types"
	tmsp "github.com/tendermint/tmsp/types"

	log "github.com/eris-ltd/eris-logger"

	sm "github.com/eris-ltd/eris-db/manager/eris-mint/state"
	manager_types "github.com/eris-ltd/eris-db/manager/types"
	"github.com/eris-ltd/eris-db/txs"
)

//--------------------------------------------------------------------------------
// ErisMint holds the current state, runs transactions, computes hashes.
// Typically two connections are opened by the tendermint core:
// one for mempool, one for consensus.

type ErisMint struct {
	mtx sync.Mutex

	state      *sm.State
	cache      *sm.BlockCache
	checkCache *sm.BlockCache // for CheckTx (eg. so we get nonces right)

	evc  *tendermint_events.EventCache
	evsw *tendermint_events.EventSwitch

	// client to the tendermint core rpc
	client *rpcclient.ClientURI
	host   string // tendermint core endpoint

	nTxs int // count txs in a block
}

// NOTE [ben] Compiler check to ensure ErisMint successfully implements
// eris-db/manager/types.Application
var _ manager_types.Application = (*ErisMint)(nil)

// NOTE: [ben] also automatically implements tmsp.Application,
// undesired but unharmful
// var _ tmsp.Application = (*ErisMint)(nil)

func (app *ErisMint) GetState() *sm.State {
	app.mtx.Lock()
	defer app.mtx.Unlock()
	return app.state.Copy()
}

// TODO: this is used for call/callcode and to get nonces during mempool.
// the former should work on last committed state only and the later should
// be handled by the client, or a separate wallet-like nonce tracker thats not part of the app
func (app *ErisMint) GetCheckCache() *sm.BlockCache {
	return app.checkCache
}

func (app *ErisMint) SetHostAddress(host string) {
	app.host = host
	app.client = rpcclient.NewClientURI(host) //fmt.Sprintf("http://%s", host))
}

// Broadcast a tx to the tendermint core
// NOTE: this assumes we know the address of core
func (app *ErisMint) BroadcastTx(tx txs.Tx) error {
	buf := new(bytes.Buffer)
	var n int
	var err error
	wire.WriteBinary(struct{ txs.Tx }{tx}, buf, &n, &err)
	if err != nil {
		return err
	}

	params := map[string]interface{}{
		"tx": hex.EncodeToString(buf.Bytes()),
	}

	var result ctypes.TMResult
	_, err = app.client.Call("broadcast_tx_sync", params, &result)
	return err
}

func NewErisMint(s *sm.State, evsw *tendermint_events.EventSwitch) *ErisMint {
	return &ErisMint{
		state:      s,
		cache:      sm.NewBlockCache(s),
		checkCache: sm.NewBlockCache(s),
		evc:        tendermint_events.NewEventCache(evsw),
		evsw:       evsw,
	}
}

// Implements manager/types.Application
func (app *ErisMint) Info() (info string) {
	return "ErisDB"
}

// Implements manager/types.Application
func (app *ErisMint) SetOption(key string, value string) (log string) {
	return ""
}

// Implements manager/types.Application
func (app *ErisMint) AppendTx(txBytes []byte) (res tmsp.Result) {

	app.nTxs += 1

	// XXX: if we had tx ids we could cache the decoded txs on CheckTx
	var n int
	var err error
	tx := new(txs.Tx)
	buf := bytes.NewBuffer(txBytes)
	wire.ReadBinaryPtr(tx, buf, len(txBytes), &n, &err)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_EncodingError, fmt.Sprintf("Encoding error: %v", err))
	}

	err = sm.ExecTx(app.cache, *tx, true, app.evc)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_InternalError, fmt.Sprintf("Internal error: %v", err))
	}

	receipt := txs.GenerateReceipt(app.state.ChainID, *tx)
	receiptBytes := wire.BinaryBytes(receipt)
	return tmsp.NewResultOK(receiptBytes, "Success")
}

// Implements manager/types.Application
func (app *ErisMint) CheckTx(txBytes []byte) (res tmsp.Result) {
	var n int
	var err error
	tx := new(txs.Tx)
	buf := bytes.NewBuffer(txBytes)
	wire.ReadBinaryPtr(tx, buf, len(txBytes), &n, &err)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_EncodingError, fmt.Sprintf("Encoding error: %v", err))
	}

	// TODO: make errors tmsp aware
	err = sm.ExecTx(app.checkCache, *tx, false, nil)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_InternalError, fmt.Sprintf("Internal error: %v", err))
	}
	receipt := txs.GenerateReceipt(app.state.ChainID, *tx)
	receiptBytes := wire.BinaryBytes(receipt)
	return tmsp.NewResultOK(receiptBytes, "Success")
}

// Implements manager/types.Application
// Commit the state (called at end of block)
// NOTE: CheckTx/AppendTx must not run concurrently with Commit -
//  the mempool should run during AppendTxs, but lock for Commit and Update
func (app *ErisMint) Commit() (res tmsp.Result) {
	app.mtx.Lock() // the lock protects app.state
	defer app.mtx.Unlock()

	app.state.LastBlockHeight += 1
	log.WithFields(log.Fields{
		"blockheight": app.state.LastBlockHeight,
	}).Info("Commit block")

	// sync the AppendTx cache
	app.cache.Sync()

	// if there were any txs in the block,
	// reset the check cache to the new height
	if app.nTxs > 0 {
		log.WithFields(log.Fields{
			"txs": app.nTxs,
		}).Info("Reset checkCache")
		app.checkCache = sm.NewBlockCache(app.state)
	}
	app.nTxs = 0

	// save state to disk
	app.state.Save()

	// flush events to listeners (XXX: note issue with blocking)
	app.evc.Flush()

	return tmsp.NewResultOK(app.state.Hash(), "Success")
}

func (app *ErisMint) Query(query []byte) (res tmsp.Result) {
	return tmsp.NewResultOK(nil, "Success")
}
<commit_msg>Don't special case empty blocks when refreshing the checkCache on commit<commit_after>// Copyright 2015, 2016 Eris Industries (UK) Ltd.
// This file is part of Eris-RT

// Eris-RT is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Eris-RT is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Eris-RT.  If not, see <http://www.gnu.org/licenses/>.

package erismint

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"sync"

	tendermint_events "github.com/tendermint/go-events"
	rpcclient "github.com/tendermint/go-rpc/client"
	wire "github.com/tendermint/go-wire"
	ctypes "github.com/tendermint/tendermint/rpc/core/types"
	tmsp "github.com/tendermint/tmsp/types"

	log "github.com/eris-ltd/eris-logger"

	sm "github.com/eris-ltd/eris-db/manager/eris-mint/state"
	manager_types "github.com/eris-ltd/eris-db/manager/types"
	"github.com/eris-ltd/eris-db/txs"
)

//--------------------------------------------------------------------------------
// ErisMint holds the current state, runs transactions, computes hashes.
// Typically two connections are opened by the tendermint core:
// one for mempool, one for consensus.

type ErisMint struct {
	mtx sync.Mutex

	state      *sm.State
	cache      *sm.BlockCache
	checkCache *sm.BlockCache // for CheckTx (eg. so we get nonces right)

	evc  *tendermint_events.EventCache
	evsw *tendermint_events.EventSwitch

	// client to the tendermint core rpc
	client *rpcclient.ClientURI
	host   string // tendermint core endpoint

	nTxs int // count txs in a block
}

// NOTE [ben] Compiler check to ensure ErisMint successfully implements
// eris-db/manager/types.Application
var _ manager_types.Application = (*ErisMint)(nil)

// NOTE: [ben] also automatically implements tmsp.Application,
// undesired but unharmful
// var _ tmsp.Application = (*ErisMint)(nil)

func (app *ErisMint) GetState() *sm.State {
	app.mtx.Lock()
	defer app.mtx.Unlock()
	return app.state.Copy()
}

// TODO: this is used for call/callcode and to get nonces during mempool.
// the former should work on last committed state only and the later should
// be handled by the client, or a separate wallet-like nonce tracker thats not part of the app
func (app *ErisMint) GetCheckCache() *sm.BlockCache {
	return app.checkCache
}

func (app *ErisMint) SetHostAddress(host string) {
	app.host = host
	app.client = rpcclient.NewClientURI(host) //fmt.Sprintf("http://%s", host))
}

// Broadcast a tx to the tendermint core
// NOTE: this assumes we know the address of core
func (app *ErisMint) BroadcastTx(tx txs.Tx) error {
	buf := new(bytes.Buffer)
	var n int
	var err error
	wire.WriteBinary(struct{ txs.Tx }{tx}, buf, &n, &err)
	if err != nil {
		return err
	}

	params := map[string]interface{}{
		"tx": hex.EncodeToString(buf.Bytes()),
	}

	var result ctypes.TMResult
	_, err = app.client.Call("broadcast_tx_sync", params, &result)
	return err
}

func NewErisMint(s *sm.State, evsw *tendermint_events.EventSwitch) *ErisMint {
	return &ErisMint{
		state:      s,
		cache:      sm.NewBlockCache(s),
		checkCache: sm.NewBlockCache(s),
		evc:        tendermint_events.NewEventCache(evsw),
		evsw:       evsw,
	}
}

// Implements manager/types.Application
func (app *ErisMint) Info() (info string) {
	return "ErisDB"
}

// Implements manager/types.Application
func (app *ErisMint) SetOption(key string, value string) (log string) {
	return ""
}

// Implements manager/types.Application
func (app *ErisMint) AppendTx(txBytes []byte) (res tmsp.Result) {

	app.nTxs += 1

	// XXX: if we had tx ids we could cache the decoded txs on CheckTx
	var n int
	var err error
	tx := new(txs.Tx)
	buf := bytes.NewBuffer(txBytes)
	wire.ReadBinaryPtr(tx, buf, len(txBytes), &n, &err)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_EncodingError, fmt.Sprintf("Encoding error: %v", err))
	}

	err = sm.ExecTx(app.cache, *tx, true, app.evc)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_InternalError, fmt.Sprintf("Internal error: %v", err))
	}

	receipt := txs.GenerateReceipt(app.state.ChainID, *tx)
	receiptBytes := wire.BinaryBytes(receipt)
	return tmsp.NewResultOK(receiptBytes, "Success")
}

// Implements manager/types.Application
func (app *ErisMint) CheckTx(txBytes []byte) (res tmsp.Result) {
	var n int
	var err error
	tx := new(txs.Tx)
	buf := bytes.NewBuffer(txBytes)
	wire.ReadBinaryPtr(tx, buf, len(txBytes), &n, &err)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_EncodingError, fmt.Sprintf("Encoding error: %v", err))
	}

	// TODO: make errors tmsp aware
	err = sm.ExecTx(app.checkCache, *tx, false, nil)
	if err != nil {
		return tmsp.NewError(tmsp.CodeType_InternalError, fmt.Sprintf("Internal error: %v", err))
	}
	receipt := txs.GenerateReceipt(app.state.ChainID, *tx)
	receiptBytes := wire.BinaryBytes(receipt)
	return tmsp.NewResultOK(receiptBytes, "Success")
}

// Implements manager/types.Application
// Commit the state (called at end of block)
// NOTE: CheckTx/AppendTx must not run concurrently with Commit -
//  the mempool should run during AppendTxs, but lock for Commit and Update
func (app *ErisMint) Commit() (res tmsp.Result) {
	app.mtx.Lock() // the lock protects app.state
	defer app.mtx.Unlock()

	app.state.LastBlockHeight += 1
	log.WithFields(log.Fields{
		"blockheight": app.state.LastBlockHeight,
	}).Info("Commit block")

	// sync the AppendTx cache
	app.cache.Sync()

	// Refresh the checkCache with the latest commited state
	log.WithFields(log.Fields{
		"txs": app.nTxs,
	}).Info("Reset checkCache")
	app.checkCache = sm.NewBlockCache(app.state)
	app.nTxs = 0

	// save state to disk
	app.state.Save()

	// flush events to listeners (XXX: note issue with blocking)
	app.evc.Flush()

	return tmsp.NewResultOK(app.state.Hash(), "Success")
}

func (app *ErisMint) Query(query []byte) (res tmsp.Result) {
	return tmsp.NewResultOK(nil, "Success")
}
<|endoftext|>


<commit_before>package transform

import "errors"

// Dictionary creates a map[string]interface{} from the given parameters by
// walking the parameters and treating them as key-value pairs.  The number
// of parameters must be even.
func Dictionary(values ...interface{}) (map[string]interface{}, error) {
	if len(values)%2 != 0 {
		return nil, errors.New("invalid dict call")
	}
	dict := make(map[string]interface{}, len(values)/2)
	for i := 0; i < len(values); i += 2 {
		key, ok := values[i].(string)
		if !ok {
			return nil, errors.New("dict keys must be strings")
		}
		dict[key] = values[i+1]
	}
	return dict, nil
}
<commit_msg>Add dict<commit_after>package transform

import "errors"

// Dictionary creates a map[string]interface{} from the given parameters by
// walking the parameters and treating them as key-value pairs.  The number
// of parameters must be even.
//
// Copyright The Hugo Authors
// License Apache 2
// https://github.com/spf13/hugo/blob/master/LICENSE.md
// https://github.com/spf13/hugo/blob/master/tpl/template_funcs.go
func Dictionary(values ...interface{}) (map[string]interface{}, error) {
	if len(values)%2 != 0 {
		return nil, errors.New("invalid dict call")
	}
	dict := make(map[string]interface{}, len(values)/2)
	for i := 0; i < len(values); i += 2 {
		key, ok := values[i].(string)
		if !ok {
			return nil, errors.New("dict keys must be strings")
		}
		dict[key] = values[i+1]
	}
	return dict, nil
}
<|endoftext|>


<commit_before>package indexer

import (
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"strings"

	"github.com/pilosa/pilosa"
)

// Frame represents a string field of an index. Refers to pilosa.Frame and pilosa.View.
type Frame struct {
	mu    sync.Mutex
	path  string
	index string
	name  string

	fragments map[uint64]*pilosa.Fragment //map slice to Fragment
	td        *TermDict
}

// NewFrame returns a new instance of frame, and initializes it.
func NewFrame(path, index, name string) *Frame {
	f := &Frame{
		path:      path,
		index:     index,
		name:      name,
		td:        &TermDict{Dir: path},
		fragments: make(map[uint64]*pilosa.Fragment),
	}
	for slice := uint64(0); ; slice++ {
		fp := f.FragmentPath(slice)
		if _, err := os.Stat(fp); os.IsNotExist(err) {
			//path does not exist
			break
		}
		fragment := pilosa.NewFragment(fp, f.index, f.name, pilosa.ViewStandard, slice)
		f.fragments[slice] = fragment
	}
	return f
}

// FragmentPath returns the path to a fragment
func (f *Frame) FragmentPath(slice uint64) string {
	return filepath.Join(f.path, "fragments", strconv.FormatUint(slice, 10))
}

// Fragment returns a fragment in the view by slice.
func (f *Frame) Fragment(slice uint64) *pilosa.Fragment {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.fragments[slice]
}

// Name returns the name the frame was initialized with.
func (f *Frame) Name() string { return f.name }

// Index returns the index name the frame was initialized with.
func (f *Frame) Index() string { return f.index }

// Path returns the path the frame was initialized with.
func (f *Frame) Path() string { return f.path }

// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
	f.mu.Lock()
	defer f.mu.Unlock()

	return uint64(len(f.fragments))
}

// SetBit sets a bit on a view within the frame.
func (f *Frame) SetBit(rowID, colID uint64) (changed bool, err error) {
	slice := colID / pilosa.SliceWidth
	fragment, ok := f.fragments[slice]
	if !ok {
		err = errors.New("column out of bounds")
		return
	}
	changed, err = fragment.SetBit(rowID, colID)
	return
}

// ClearBit clears a bit within the frame.
func (f *Frame) ClearBit(rowID, colID uint64) (changed bool, err error) {
	slice := colID / pilosa.SliceWidth
	fragment, ok := f.fragments[slice]
	if !ok {
		err = errors.New("column out of bounds")
		return
	}
	changed, err = fragment.SetBit(rowID, colID)
	return
}

// ParseAndIndex parses and index a field
func (f *Frame) ParseAndIndex(docID uint64, text string) (err error) {
	terms := strings.SplitN(text, " ", -1)
	ids, err := f.td.GetTermsID(terms)
	for _, termID := range ids {
		if _, err = f.SetBit(termID, docID); err != nil {
			return
		}
	}
	return
}
<commit_msg>getSliceList<commit_after>package indexer

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"sync"

	"strings"

	"github.com/pilosa/pilosa"
)

// Frame represents a string field of an index. Refers to pilosa.Frame and pilosa.View.
type Frame struct {
	mu    sync.Mutex
	path  string
	index string
	name  string

	fragments map[uint64]*pilosa.Fragment //map slice to Fragment
	td        *TermDict
}

// NewFrame returns a new instance of frame, and initializes it.
func NewFrame(path, index, name string) *Frame {
	f := &Frame{
		path:      path,
		index:     index,
		name:      name,
		td:        &TermDict{Dir: path},
		fragments: make(map[uint64]*pilosa.Fragment),
	}
	sliceList := getSliceList(path)
	for _, slice := range sliceList {
		fp := f.FragmentPath(slice)
		if _, err := os.Stat(fp); os.IsNotExist(err) {
			//path does not exist
			break
		}
		fragment := pilosa.NewFragment(fp, f.index, f.name, pilosa.ViewStandard, slice)
		f.fragments[slice] = fragment
	}
	return f
}

func getSliceList(dir string) (numList []uint64, err error) {
	var d *os.File
	var fns []string
	var num int
	d, err = os.Open(filepath.Join(dir, "fragments"))
	if err != nil {
		err = errors.Wrap(err, "")
		return
	}
	fns, err = d.Readdirnames(0)
	if err != nil {
		err = errors.Wrap(err, "")
		return
	}
	re := regexp.MustCompile(fmt.Sprintf("(?P<num>[0-9]+)", prefix))
	for _, fn := range fns {
		subs := re.FindStringSubmatch(fn)
		if subs == nil {
			continue
		}
		num, err = strconv.ParseUint(subs[1])
		if err != nil {
			err = errors.Wrap(err, "")
			return
		}
		numList = append(numList, num)
	}
	sort.Ints(numList)
	return
}

// FragmentPath returns the path to a fragment
func (f *Frame) FragmentPath(slice uint64) string {
	return filepath.Join(f.path, "fragments", strconv.FormatUint(slice, 10))
}

// Fragment returns a fragment in the view by slice.
func (f *Frame) Fragment(slice uint64) *pilosa.Fragment {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.fragments[slice]
}

// Name returns the name the frame was initialized with.
func (f *Frame) Name() string { return f.name }

// Index returns the index name the frame was initialized with.
func (f *Frame) Index() string { return f.index }

// Path returns the path the frame was initialized with.
func (f *Frame) Path() string { return f.path }

// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
	f.mu.Lock()
	defer f.mu.Unlock()

	return uint64(len(f.fragments))
}

// SetBit sets a bit on a view within the frame.
func (f *Frame) SetBit(rowID, colID uint64) (changed bool, err error) {
	slice := colID / pilosa.SliceWidth
	fragment, ok := f.fragments[slice]
	if !ok {
		err = errors.New("column out of bounds")
		return
	}
	changed, err = fragment.SetBit(rowID, colID)
	return
}

// ClearBit clears a bit within the frame.
func (f *Frame) ClearBit(rowID, colID uint64) (changed bool, err error) {
	slice := colID / pilosa.SliceWidth
	fragment, ok := f.fragments[slice]
	if !ok {
		err = errors.New("column out of bounds")
		return
	}
	changed, err = fragment.SetBit(rowID, colID)
	return
}

// ParseAndIndex parses and index a field
func (f *Frame) ParseAndIndex(docID uint64, text string) (err error) {
	terms := strings.SplitN(text, " ", -1)
	ids, err := f.td.GetTermsID(terms)
	for _, termID := range ids {
		if _, err = f.SetBit(termID, docID); err != nil {
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/user"
)

type ArchiveData struct {
	Years   *map[string]Year
	Months  *[]string
	Posts   *[]Entry
	IsAdmin bool
}

type Year map[string]Month
type Month []Day
type Day int64

var Months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveTaskHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	years := make(map[string]Year)

	oldest := (*entries)[len(*entries)-1].Datetime
	newest := (*entries)[0].Datetime

	log.Infof(c, "Oldest: %v, Newest: %v", oldest, newest)

	for year := oldest.Year(); year <= newest.Year(); year += 1 {
		ystr := strconv.Itoa(year)
		years[ystr] = make(Year)
		log.Infof(c, "Adding %d.", year)

		startMonth := time.January
		endMonth := time.December
		if year == oldest.Year() {
			startMonth = oldest.Month()
		}
		if year == newest.Year() {
			endMonth = newest.Month()
		}

		for month := startMonth; month <= endMonth; month += 1 {
			mstr := month.String()
			years[ystr][mstr] = make([]Day, daysIn(month, year)+1)
			log.Debugf(c, "Adding %d/%d - %d days.", year, month, len(years[ystr][mstr]))

			for day := range years[ystr][mstr] {
				if day > 0 {
					e, err := PostsForDay(c, int64(year), int64(month), int64(day))
					if err != nil {
						log.Errorf(c, err.Error())
						http.Error(w, err.Error(), 500)
						return
					}
					years[ystr][mstr][day] = Day(len(*e))
				}
			}
		}
	}
	log.Infof(c, "Added posts.")

	// https://blog.golang.org/json-and-go
	b, err := json.Marshal(years)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
	}

	item := &memcache.Item{
		Key:   "archive_data",
		Value: b,
	}

	// Set the item, unconditionally
	if err := memcache.Set(c, item); err != nil {
		log.Errorf(c, "error setting item: %v", err)
	}
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	// Get the item from the memcache
	var years map[string]Year
	if year_data, err := memcache.Get(c, "archive_data"); err == memcache.ErrCacheMiss {
		log.Infof(c, "item not in the cache")
	} else if err != nil {
		log.Errorf(c, "error getting item: %v", err)
	} else {
		err := json.Unmarshal(year_data.Value, &years)
		if err != nil {
			log.Errorf(c, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}
	}

	data := &ArchiveData{
		Years: &years,
		Months: &[]string{
			time.January.String(),
			time.February.String(),
			time.March.String(),
			time.April.String(),
			time.May.String(),
			time.June.String(),
			time.July.String(),
			time.August.String(),
			time.September.String(),
			time.October.String(),
			time.November.String(),
			time.December.String(),
		},
		IsAdmin: user.IsAdmin(c),
		Posts:   entries,
	}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}
<commit_msg>progress on posts handler<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/user"
)

type ArchiveData struct {
	Years   *map[string]Year
	Months  *[]string
	Posts   *[]Entry
	IsAdmin bool
}

type Year map[string]Month
type Month []Day
type Day int64

var Months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveTaskHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	years := make(map[string]Year)

	oldest := (*entries)[len(*entries)-1].Datetime
	newest := (*entries)[0].Datetime

	log.Infof(c, "Oldest: %v, Newest: %v", oldest, newest)

	for year := oldest.Year(); year <= newest.Year(); year += 1 {
		ystr := strconv.Itoa(year)
		years[ystr] = make(Year)
		log.Infof(c, "Adding %d.", year)

		startMonth := time.January
		endMonth := time.December
		if year == oldest.Year() {
			startMonth = oldest.Month()
		}
		if year == newest.Year() {
			endMonth = newest.Month()
		}

		for month := startMonth; month <= endMonth; month += 1 {
			mstr := month.String()
			years[ystr][mstr] = make([]Day, daysIn(month, year)+1)
			log.Debugf(c, "Adding %d/%d - %d days.", year, month, len(years[ystr][mstr]))

			for day := range years[ystr][mstr] {
				if day > 0 {
					e, err := PostsForDay(c, int64(year), int64(month), int64(day))
					if err != nil {
						log.Errorf(c, err.Error())
						http.Error(w, err.Error(), 500)
						return
					}
					years[ystr][mstr][day] = Day(len(*e))
				}
			}
		}
	}
	log.Infof(c, "Added posts.")

	// https://blog.golang.org/json-and-go
	b, err := json.Marshal(years)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
	}

	item := &memcache.Item{
		Key:   "archive_data",
		Value: b,
	}

	// Set the item, unconditionally
	if err := memcache.Set(c, item); err != nil {
		log.Errorf(c, "error setting item: %v", err)
	}
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Get the item from the memcache
	var years map[string]Year
	if year_data, err := memcache.Get(c, "archive_data"); err == memcache.ErrCacheMiss {
		log.Infof(c, "item not in the cache")
	} else if err != nil {
		log.Errorf(c, "error getting item: %v", err)
	} else {
		err := json.Unmarshal(year_data.Value, &years)
		if err != nil {
			log.Errorf(c, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}
	}

	data := &ArchiveData{
		Years: &years,
		Months: &[]string{
			time.January.String(),
			time.February.String(),
			time.March.String(),
			time.April.String(),
			time.May.String(),
			time.June.String(),
			time.July.String(),
			time.August.String(),
			time.September.String(),
			time.October.String(),
			time.November.String(),
			time.December.String(),
		},
		IsAdmin: user.IsAdmin(c),
		Posts:   entries,
	}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}

func PostsHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	urls := []string{}

	entries, err := AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	for _, e := range *entries {
		urls = append(urls, fmt.Sprintf("/post/%d", e.Id))
	}

	w.WriteJSON(urls)
}
<|endoftext|>


<commit_before>package handlers

import (
	"errors"
	"io"
	"net/http"

	ctxu "github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/api/errcode"
)

// closeResources closes all the provided resources after running the target
// handler.
func closeResources(handler http.Handler, closers ...io.Closer) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for _, closer := range closers {
			defer closer.Close()
		}
		handler.ServeHTTP(w, r)
	})
}

// copyFullPayload copies the payload of a HTTP request to destWriter. If it
// receives less content than expected, and the client disconnected during the
// upload, it avoids sending a 400 error to keep the logs cleaner.
func copyFullPayload(responseWriter http.ResponseWriter, r *http.Request, destWriter io.Writer, context ctxu.Context, action string, errSlice *errcode.Errors) error {
	// Get a channel that tells us if the client disconnects
	var clientClosed <-chan bool
	if notifier, ok := responseWriter.(http.CloseNotifier); ok {
		clientClosed = notifier.CloseNotify()
	} else {
		panic("the ResponseWriter does not implement CloseNotifier")
	}

	// Read in the data, if any.
	copied, err := io.Copy(destWriter, r.Body)
	if clientClosed != nil && (err != nil || (r.ContentLength > 0 && copied < r.ContentLength)) {
		// Didn't recieve as much content as expected. Did the client
		// disconnect during the request? If so, avoid returning a 400
		// error to keep the logs cleaner.
		select {
		case <-clientClosed:
			ctxu.GetLogger(context).Error("client disconnected during " + action)
			return errors.New("client disconnected")
		default:
		}
	}

	if err != nil {
		ctxu.GetLogger(context).Errorf("unknown error reading request payload: %v", err)
		*errSlice = append(*errSlice, errcode.ErrorCodeUnknown.WithDetail(err))
		return err
	}

	return nil
}
<commit_msg>Set the response code to 499 when a client disconnects during an upload<commit_after>package handlers

import (
	"errors"
	"io"
	"net/http"

	ctxu "github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/api/errcode"
)

// closeResources closes all the provided resources after running the target
// handler.
func closeResources(handler http.Handler, closers ...io.Closer) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for _, closer := range closers {
			defer closer.Close()
		}
		handler.ServeHTTP(w, r)
	})
}

// copyFullPayload copies the payload of a HTTP request to destWriter. If it
// receives less content than expected, and the client disconnected during the
// upload, it avoids sending a 400 error to keep the logs cleaner.
func copyFullPayload(responseWriter http.ResponseWriter, r *http.Request, destWriter io.Writer, context ctxu.Context, action string, errSlice *errcode.Errors) error {
	// Get a channel that tells us if the client disconnects
	var clientClosed <-chan bool
	if notifier, ok := responseWriter.(http.CloseNotifier); ok {
		clientClosed = notifier.CloseNotify()
	} else {
		panic("the ResponseWriter does not implement CloseNotifier")
	}

	// Read in the data, if any.
	copied, err := io.Copy(destWriter, r.Body)
	if clientClosed != nil && (err != nil || (r.ContentLength > 0 && copied < r.ContentLength)) {
		// Didn't recieve as much content as expected. Did the client
		// disconnect during the request? If so, avoid returning a 400
		// error to keep the logs cleaner.
		select {
		case <-clientClosed:
			// Set the response code to "499 Client Closed Request"
			// Even though the connection has already been closed,
			// this causes the logger to pick up a 499 error
			// instead of showing 0 for the HTTP status.
			responseWriter.WriteHeader(499)

			ctxu.GetLogger(context).Error("client disconnected during " + action)
			return errors.New("client disconnected")
		default:
		}
	}

	if err != nil {
		ctxu.GetLogger(context).Errorf("unknown error reading request payload: %v", err)
		*errSlice = append(*errSlice, errcode.ErrorCodeUnknown.WithDetail(err))
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package vix

import (
	"fmt"
	"strings"

	"github.com/cloudescape/govmx"
)

// CD/DVD configuration
type CDDVDDrive struct {
	ID string
	// Either IDE, SCSI or SATA
	Bus vmx.BusType
	// Used only when attaching image files. Ex: ISO images
	// If you just want to attach a raw cdrom device leave it empty
	Filename string
}

// Attaches a CD/DVD drive to the virtual machine.
func (v *VM) AttachCDDVD(drive *CDDVDDrive) error {
	if running, _ := v.IsRunning(); running {
		return &VixError{
			Operation: "vm.AttachCDDVD",
			Code:      200000,
			Text:      "Virtual machine must be powered off in order to attach a CD/DVD drive.",
		}
	}

	// Loads VMX file in memory
	v.vmxfile.Read()
	model := v.vmxfile.model

	device := vmx.Device{}
	if drive.Filename != "" {
		device.Filename = drive.Filename
		device.Type = vmx.CDROM_IMAGE
	} else {
		device.Type = vmx.CDROM_RAW
		device.Autodetect = true
	}

	device.Present = true
	device.StartConnected = true

	if drive.Bus == "" {
		drive.Bus = vmx.IDE
	}

	switch drive.Bus {
	case vmx.IDE:
		model.IDEDevices = append(model.IDEDevices, vmx.IDEDevice{Device: device})
	case vmx.SCSI:
		model.SCSIDevices = append(model.SCSIDevices, vmx.SCSIDevice{Device: device})
	case vmx.SATA:
		model.SATADevices = append(model.SATADevices, vmx.SATADevice{Device: device})
	default:
		return &VixError{
			Operation: "vm.AttachCDDVD",
			Code:      200001,
			Text:      fmt.Sprintf("Unrecognized bus type: %s\n", drive.Bus),
		}
	}

	return v.vmxfile.Write()
}

// Detaches a CD/DVD device from the virtual machine
func (v *VM) DetachCDDVD(drive *CDDVDDrive) error {
	if running, _ := v.IsRunning(); running {
		return &VixError{
			Operation: "vm.DetachCDDVD",
			Code:      200002,
			Text:      "Virtual machine must be powered off in order to detach CD/DVD drive.",
		}
	}

	// Loads VMX file in memory
	err := v.vmxfile.Read()
	if err != nil {
		return err
	}

	model := v.vmxfile.model

	switch drive.Bus {
	case vmx.IDE:
		for i, device := range model.IDEDevices {
			if drive.ID == device.VMXID {
				// This method of removing the element avoids memory leaks
				copy(model.IDEDevices[i:], model.IDEDevices[i+1:])
				model.IDEDevices[len(model.IDEDevices)-1] = vmx.IDEDevice{}
				model.IDEDevices = model.IDEDevices[:len(model.IDEDevices)-1]
			}
		}
	case vmx.SCSI:
		for i, device := range model.SCSIDevices {
			if drive.ID == device.VMXID {
				copy(model.SCSIDevices[i:], model.SCSIDevices[i+1:])
				model.SCSIDevices[len(model.SCSIDevices)-1] = vmx.SCSIDevice{}
				model.SCSIDevices = model.SCSIDevices[:len(model.SCSIDevices)-1]
			}
		}
	case vmx.SATA:
		for i, device := range model.SATADevices {
			if drive.ID == device.VMXID {
				copy(model.SATADevices[i:], model.SATADevices[i+1:])
				model.SATADevices[len(model.SATADevices)-1] = vmx.SATADevice{}
				model.SATADevices = model.SATADevices[:len(model.SATADevices)-1]
			}
		}
	default:
		return &VixError{
			Operation: "vm.DetachCDDVD",
			Code:      200003,
			Text:      fmt.Sprintf("Unrecognized bus type: %s\n", drive.Bus),
		}
	}

	return v.vmxfile.Write()
}

// Returns an unordered slice of currently attached CD/DVD devices on any bus.
func (v *VM) CDDVDs() ([]*CDDVDDrive, error) {
	// Loads VMX file in memory
	err := v.vmxfile.Read()
	if err != nil {
		return nil, err
	}

	model := v.vmxfile.model

	var cddvds []*CDDVDDrive
	model.WalkDevices(func(d vmx.Device) {
		var bus vmx.BusType
		switch {
		case strings.HasPrefix(d.VMXID, string(vmx.IDE)):
			bus = vmx.IDE
		case strings.HasPrefix(d.VMXID, string(vmx.SCSI)):
			bus = vmx.SCSI
		case strings.HasPrefix(d.VMXID, string(vmx.SATA)):
			bus = vmx.SATA
		}

		if d.Type == vmx.CDROM_IMAGE || d.Type == vmx.CDROM_RAW {
			cddvds = append(cddvds, &CDDVDDrive{
				ID:       d.VMXID,
				Bus:      bus,
				Filename: d.Filename,
			})
		}
	})
	return cddvds, nil
}

func (v *VM) RemoveAllCDDVDDrives() error {
	drives, err := v.CDDVDs()
	if err != nil {
		return &VixError{
			Operation: "vm.RemoveAllCDDVDDrives",
			Code:      200004,
			Text:      fmt.Sprintf("Error listing CD/DVD Drives: %s\n", err),
		}
	}

	for _, d := range drives {
		err := v.DetachCDDVD(d)
		if err != nil {
			return &VixError{
				Operation: "vm.RemoveAllCDDVDDrives",
				Code:      200004,
				Text:      fmt.Sprintf("Error removing CD/DVD Drive %v, error: %s\n", d, err),
			}
		}
	}

	return nil
}

// Returns the CD/DVD drive identified by ID
// This function depends entirely on how GoVMX identifies slice's elements
func (v *VM) CDDVD(ID string) (*CDDVDDrive, error) {
	err := v.vmxfile.Read()
	if err != nil {
		return nil, err
	}

	model := v.vmxfile.model

	var bus vmx.BusType
	switch {
	case strings.HasPrefix(ID, string(vmx.IDE)):
		bus = vmx.IDE
	case strings.HasPrefix(ID, string(vmx.SCSI)):
		bus = vmx.SCSI
	case strings.HasPrefix(ID, string(vmx.SATA)):
		bus = vmx.SATA
	}

	var filename string
	found := model.FindDevice(func(d vmx.Device) bool {
		if ID == d.VMXID {
			filename = d.Filename
		}
		return ID == d.VMXID
	}, bus)

	if !found {
		return nil, nil
	}

	return &CDDVDDrive{Bus: bus, Filename: filename}, nil
}
<commit_msg>Gets BUS type from device ID<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package vix

import (
	"fmt"
	"strings"

	"github.com/cloudescape/govmx"
)

// CD/DVD configuration
type CDDVDDrive struct {
	ID string
	// Either IDE, SCSI or SATA
	Bus vmx.BusType
	// Used only when attaching image files. Ex: ISO images
	// If you just want to attach a raw cdrom device leave it empty
	Filename string
}

// Attaches a CD/DVD drive to the virtual machine.
func (v *VM) AttachCDDVD(drive *CDDVDDrive) error {
	if running, _ := v.IsRunning(); running {
		return &VixError{
			Operation: "vm.AttachCDDVD",
			Code:      200000,
			Text:      "Virtual machine must be powered off in order to attach a CD/DVD drive.",
		}
	}

	// Loads VMX file in memory
	v.vmxfile.Read()
	model := v.vmxfile.model

	device := vmx.Device{}
	if drive.Filename != "" {
		device.Filename = drive.Filename
		device.Type = vmx.CDROM_IMAGE
	} else {
		device.Type = vmx.CDROM_RAW
		device.Autodetect = true
	}

	device.Present = true
	device.StartConnected = true

	if drive.Bus == "" {
		drive.Bus = vmx.IDE
	}

	switch drive.Bus {
	case vmx.IDE:
		model.IDEDevices = append(model.IDEDevices, vmx.IDEDevice{Device: device})
	case vmx.SCSI:
		model.SCSIDevices = append(model.SCSIDevices, vmx.SCSIDevice{Device: device})
	case vmx.SATA:
		model.SATADevices = append(model.SATADevices, vmx.SATADevice{Device: device})
	default:
		return &VixError{
			Operation: "vm.AttachCDDVD",
			Code:      200001,
			Text:      fmt.Sprintf("Unrecognized bus type: %s\n", drive.Bus),
		}
	}

	return v.vmxfile.Write()
}

// Detaches a CD/DVD device from the virtual machine
func (v *VM) DetachCDDVD(drive *CDDVDDrive) error {
	if running, _ := v.IsRunning(); running {
		return &VixError{
			Operation: "vm.DetachCDDVD",
			Code:      200002,
			Text:      "Virtual machine must be powered off in order to detach CD/DVD drive.",
		}
	}

	// Loads VMX file in memory
	err := v.vmxfile.Read()
	if err != nil {
		return err
	}

	model := v.vmxfile.model

	switch drive.Bus {
	case vmx.IDE:
		for i, device := range model.IDEDevices {
			if drive.ID == device.VMXID {
				// This method of removing the element avoids memory leaks
				copy(model.IDEDevices[i:], model.IDEDevices[i+1:])
				model.IDEDevices[len(model.IDEDevices)-1] = vmx.IDEDevice{}
				model.IDEDevices = model.IDEDevices[:len(model.IDEDevices)-1]
			}
		}
	case vmx.SCSI:
		for i, device := range model.SCSIDevices {
			if drive.ID == device.VMXID {
				copy(model.SCSIDevices[i:], model.SCSIDevices[i+1:])
				model.SCSIDevices[len(model.SCSIDevices)-1] = vmx.SCSIDevice{}
				model.SCSIDevices = model.SCSIDevices[:len(model.SCSIDevices)-1]
			}
		}
	case vmx.SATA:
		for i, device := range model.SATADevices {
			if drive.ID == device.VMXID {
				copy(model.SATADevices[i:], model.SATADevices[i+1:])
				model.SATADevices[len(model.SATADevices)-1] = vmx.SATADevice{}
				model.SATADevices = model.SATADevices[:len(model.SATADevices)-1]
			}
		}
	default:
		return &VixError{
			Operation: "vm.DetachCDDVD",
			Code:      200003,
			Text:      fmt.Sprintf("Unrecognized bus type: %s\n", drive.Bus),
		}
	}

	return v.vmxfile.Write()
}

// Returns an unordered slice of currently attached CD/DVD devices on any bus.
func (v *VM) CDDVDs() ([]*CDDVDDrive, error) {
	// Loads VMX file in memory
	err := v.vmxfile.Read()
	if err != nil {
		return nil, err
	}

	model := v.vmxfile.model

	var cddvds []*CDDVDDrive
	model.WalkDevices(func(d vmx.Device) {
		bus := BusTypeFromID(d.VMXID)

		if d.Type == vmx.CDROM_IMAGE || d.Type == vmx.CDROM_RAW {
			cddvds = append(cddvds, &CDDVDDrive{
				ID:       d.VMXID,
				Bus:      bus,
				Filename: d.Filename,
			})
		}
	})
	return cddvds, nil
}

func (v *VM) RemoveAllCDDVDDrives() error {
	drives, err := v.CDDVDs()
	if err != nil {
		return &VixError{
			Operation: "vm.RemoveAllCDDVDDrives",
			Code:      200004,
			Text:      fmt.Sprintf("Error listing CD/DVD Drives: %s\n", err),
		}
	}

	for _, d := range drives {
		err := v.DetachCDDVD(d)
		if err != nil {
			return &VixError{
				Operation: "vm.RemoveAllCDDVDDrives",
				Code:      200004,
				Text:      fmt.Sprintf("Error removing CD/DVD Drive %v, error: %s\n", d, err),
			}
		}
	}

	return nil
}

// Gets BusType from device ID
func BusTypeFromID(ID string) vmx.BusType {
	var bus vmx.BusType
	switch {
	case strings.HasPrefix(ID, string(vmx.IDE)):
		bus = vmx.IDE
	case strings.HasPrefix(ID, string(vmx.SCSI)):
		bus = vmx.SCSI
	case strings.HasPrefix(ID, string(vmx.SATA)):
		bus = vmx.SATA
	}

	return bus
}

// Returns the CD/DVD drive identified by ID
// This function depends entirely on how GoVMX identifies slice's elements
func (v *VM) CDDVD(ID string) (*CDDVDDrive, error) {
	err := v.vmxfile.Read()
	if err != nil {
		return nil, err
	}

	model := v.vmxfile.model
	bus := BusTypeFromID(ID)

	var filename string
	found := model.FindDevice(func(d vmx.Device) bool {
		if ID == d.VMXID {
			filename = d.Filename
		}
		return ID == d.VMXID
	}, bus)

	if !found {
		return nil, nil
	}

	return &CDDVDDrive{Bus: bus, Filename: filename}, nil
}
<|endoftext|>


<commit_before>// +build darwin linux
// run

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that maps don't go quadratic for NaNs and other values.

package main

import (
	"fmt"
	"math"
	"time"
)

// checkLinear asserts that the running time of f(n) is in O(n).
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	t0 := time.Now()

	n := tries
	fails := 0
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)

		// should be 2x (linear); allow up to 3x
		if t2 < 3*t1 {
			if false {
				fmt.Println(typ, "\t", time.Since(t0))
			}
			return
		}
		fails++
		if fails == 6 {
			panic(fmt.Sprintf("%s: too slow: %d inserts: %v; %d inserts: %v\n",
				typ, n, t1, 2*n, t2))
		}
		if fails < 4 {
			n *= 2
		}
	}
}

type I interface {
	f()
}

type C int

func (C) f() {}

func main() {
	// NaNs. ~31ms on a 1.6GHz Zeon.
	checkLinear("NaN", 30000, func(n int) {
		m := map[float64]int{}
		nan := math.NaN()
		for i := 0; i < n; i++ {
			m[nan] = 1
		}
		if len(m) != n {
			panic("wrong size map after nan insertion")
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("eface", 10000, func(n int) {
		m := map[interface{}]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~7ms on a 1.6GHz Zeon.
	// Regression test for CL 119360043.
	checkLinear("iface", 10000, func(n int) {
		m := map[I]int{}
		for i := 0; i < n; i++ {
			m[C(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("int", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~18ms on a 1.6GHz Zeon.
	checkLinear("string", 10000, func(n int) {
		m := map[string]int{}
		for i := 0; i < n; i++ {
			m[fmt.Sprint(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float32", 10000, func(n int) {
		m := map[float32]int{}
		for i := 0; i < n; i++ {
			m[float32(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float64", 10000, func(n int) {
		m := map[float64]int{}
		for i := 0; i < n; i++ {
			m[float64(i)] = 1
		}
	})

	// ~22ms on a 1.6GHz Zeon.
	checkLinear("complex64", 10000, func(n int) {
		m := map[complex64]int{}
		for i := 0; i < n; i++ {
			m[complex(float32(i), float32(i))] = 1
		}
	})

	// ~32ms on a 1.6GHz Zeon.
	checkLinear("complex128", 10000, func(n int) {
		m := map[complex128]int{}
		for i := 0; i < n; i++ {
			m[complex(float64(i), float64(i))] = 1
		}
	})

	// ~70ms on a 1.6GHz Zeon.
	// The iterate/delete idiom currently takes expected
	// O(n lg n) time.  Fortunately, the checkLinear test
	// leaves enough wiggle room to include n lg n time
	// (it actually tests for O(n^log_2(3)).
	// To prevent false positives, average away variation
	// by doing multiple rounds within a single run.
	checkLinear("iterdelete", 2500, func(n int) {
		for round := 0; round < 4; round++ {
			m := map[int]int{}
			for i := 0; i < n; i++ {
				m[i] = i
			}
			for i := 0; i < n; i++ {
				for k := range m {
					delete(m, k)
					break
				}
			}
		}
	})
}
<commit_msg>[dev.garbage] runtime: Linear map test give false negative due to GC. This test gives a false negative at an observed rate of 1 in a 1000 due to the fact that it runs for < 100 ms. allowing GC pauses to warp the results. Changed the test so that it triggers only if it remains non-linear for much larger problem sizes.<commit_after>// +build darwin linux
// run

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that maps don't go quadratic for NaNs and other values.

package main

import (
	"fmt"
	"math"
	"time"
)

// checkLinear asserts that the running time of f(n) is in O(n).
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	t0 := time.Now()

	n := tries
	fails := 0
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)

		// should be 2x (linear); allow up to 3x
		if t2 < 3*t1 {
			if false {
				fmt.Println(typ, "\t", time.Since(t0))
			}
			return
		}
		fails++
		if fails == 12 {
			panic(fmt.Sprintf("%s: too slow: %d inserts: %v; %d inserts: %v\n",
				typ, n, t1, 2*n, t2))
		}
		if fails < 10 {
			n *= 2
		}
	}
}

type I interface {
	f()
}

type C int

func (C) f() {}

func main() {
	// NaNs. ~31ms on a 1.6GHz Zeon.
	checkLinear("NaN", 30000, func(n int) {
		m := map[float64]int{}
		nan := math.NaN()
		for i := 0; i < n; i++ {
			m[nan] = 1
		}
		if len(m) != n {
			panic("wrong size map after nan insertion")
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("eface", 10000, func(n int) {
		m := map[interface{}]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~7ms on a 1.6GHz Zeon.
	// Regression test for CL 119360043.
	checkLinear("iface", 10000, func(n int) {
		m := map[I]int{}
		for i := 0; i < n; i++ {
			m[C(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("int", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~18ms on a 1.6GHz Zeon.
	checkLinear("string", 10000, func(n int) {
		m := map[string]int{}
		for i := 0; i < n; i++ {
			m[fmt.Sprint(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float32", 10000, func(n int) {
		m := map[float32]int{}
		for i := 0; i < n; i++ {
			m[float32(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float64", 10000, func(n int) {
		m := map[float64]int{}
		for i := 0; i < n; i++ {
			m[float64(i)] = 1
		}
	})

	// ~22ms on a 1.6GHz Zeon.
	checkLinear("complex64", 10000, func(n int) {
		m := map[complex64]int{}
		for i := 0; i < n; i++ {
			m[complex(float32(i), float32(i))] = 1
		}
	})

	// ~32ms on a 1.6GHz Zeon.
	checkLinear("complex128", 10000, func(n int) {
		m := map[complex128]int{}
		for i := 0; i < n; i++ {
			m[complex(float64(i), float64(i))] = 1
		}
	})

	// ~70ms on a 1.6GHz Zeon.
	// The iterate/delete idiom currently takes expected
	// O(n lg n) time.  Fortunately, the checkLinear test
	// leaves enough wiggle room to include n lg n time
	// (it actually tests for O(n^log_2(3)).
	// To prevent false positives, average away variation
	// by doing multiple rounds within a single run.
	checkLinear("iterdelete", 2500, func(n int) {
		for round := 0; round < 4; round++ {
			m := map[int]int{}
			for i := 0; i < n; i++ {
				m[i] = i
			}
			for i := 0; i < n; i++ {
				for k := range m {
					delete(m, k)
					break
				}
			}
		}
	})
}
<|endoftext|>


<commit_before>// +build darwin linux
// run

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that maps don't go quadratic for NaNs and other values.

package main

import (
	"fmt"
	"math"
	"time"
)

// checkLinear asserts that the running time of f(n) is in O(n).
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	t0 := time.Now()

	n := tries
	fails := 0
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)

		// should be 2x (linear); allow up to 3x
		if t2 < 3*t1 {
			if false {
				fmt.Println(typ, "\t", time.Since(t0))
			}
			return
		}
		fails++
		if fails == 6 {
			panic(fmt.Sprintf("%s: too slow: %d inserts: %v; %d inserts: %v\n",
				typ, n, t1, 2*n, t2))
		}
		if fails < 4 {
			n *= 2
		}
	}
}

type I interface {
	f()
}

type C int

func (C) f() {}

func main() {
	// NaNs. ~31ms on a 1.6GHz Zeon.
	checkLinear("NaN", 30000, func(n int) {
		m := map[float64]int{}
		nan := math.NaN()
		for i := 0; i < n; i++ {
			m[nan] = 1
		}
		if len(m) != n {
			panic("wrong size map after nan insertion")
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("eface", 10000, func(n int) {
		m := map[interface{}]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~7ms on a 1.6GHz Zeon.
	// Regression test for CL 119360043.
	checkLinear("iface", 10000, func(n int) {
		m := map[I]int{}
		for i := 0; i < n; i++ {
			m[C(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("int", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~18ms on a 1.6GHz Zeon.
	checkLinear("string", 10000, func(n int) {
		m := map[string]int{}
		for i := 0; i < n; i++ {
			m[fmt.Sprint(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float32", 10000, func(n int) {
		m := map[float32]int{}
		for i := 0; i < n; i++ {
			m[float32(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float64", 10000, func(n int) {
		m := map[float64]int{}
		for i := 0; i < n; i++ {
			m[float64(i)] = 1
		}
	})

	// ~22ms on a 1.6GHz Zeon.
	checkLinear("complex64", 10000, func(n int) {
		m := map[complex64]int{}
		for i := 0; i < n; i++ {
			m[complex(float32(i), float32(i))] = 1
		}
	})

	// ~32ms on a 1.6GHz Zeon.
	checkLinear("complex128", 10000, func(n int) {
		m := map[complex128]int{}
		for i := 0; i < n; i++ {
			m[complex(float64(i), float64(i))] = 1
		}
	})
}
<commit_msg>runtime: add timing test for iterate/delete map idiom.<commit_after>// +build darwin linux
// run

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that maps don't go quadratic for NaNs and other values.

package main

import (
	"fmt"
	"math"
	"time"
)

// checkLinear asserts that the running time of f(n) is in O(n).
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	t0 := time.Now()

	n := tries
	fails := 0
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)

		// should be 2x (linear); allow up to 3x
		if t2 < 3*t1 {
			if false {
				fmt.Println(typ, "\t", time.Since(t0))
			}
			return
		}
		fails++
		if fails == 6 {
			panic(fmt.Sprintf("%s: too slow: %d inserts: %v; %d inserts: %v\n",
				typ, n, t1, 2*n, t2))
		}
		if fails < 4 {
			n *= 2
		}
	}
}

type I interface {
	f()
}

type C int

func (C) f() {}

func main() {
	// NaNs. ~31ms on a 1.6GHz Zeon.
	checkLinear("NaN", 30000, func(n int) {
		m := map[float64]int{}
		nan := math.NaN()
		for i := 0; i < n; i++ {
			m[nan] = 1
		}
		if len(m) != n {
			panic("wrong size map after nan insertion")
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("eface", 10000, func(n int) {
		m := map[interface{}]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~7ms on a 1.6GHz Zeon.
	// Regression test for CL 119360043.
	checkLinear("iface", 10000, func(n int) {
		m := map[I]int{}
		for i := 0; i < n; i++ {
			m[C(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("int", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~18ms on a 1.6GHz Zeon.
	checkLinear("string", 10000, func(n int) {
		m := map[string]int{}
		for i := 0; i < n; i++ {
			m[fmt.Sprint(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float32", 10000, func(n int) {
		m := map[float32]int{}
		for i := 0; i < n; i++ {
			m[float32(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float64", 10000, func(n int) {
		m := map[float64]int{}
		for i := 0; i < n; i++ {
			m[float64(i)] = 1
		}
	})

	// ~22ms on a 1.6GHz Zeon.
	checkLinear("complex64", 10000, func(n int) {
		m := map[complex64]int{}
		for i := 0; i < n; i++ {
			m[complex(float32(i), float32(i))] = 1
		}
	})

	// ~32ms on a 1.6GHz Zeon.
	checkLinear("complex128", 10000, func(n int) {
		m := map[complex128]int{}
		for i := 0; i < n; i++ {
			m[complex(float64(i), float64(i))] = 1
		}
	})

	// ~70ms on a 1.6GHz Zeon.
	// The iterate/delete idiom currently takes expected
	// O(n lg n) time.  Fortunately, the checkLinear test
	// leaves enough wiggle room to include n lg n time
	// (it actually tests for O(n^log_2(3)).
	checkLinear("iterdelete", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = i
		}
		for i := 0; i < n; i++ {
			for k := range m {
				delete(m, k)
				break
			}
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// TODO(jacobsa): Implement ForgetInode support in the fuse package, then
	// implement the method here and clean up these maps.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: For all keys k, inodes[k].ID() == k
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]inode.Inode

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, isDirName(k)
	// INVARIANT: For each key k, dirIndex[k].Name() == k
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirInode.
	//
	// GUARDED_BY(mu)
	dirIndex map[string]*inode.DirInode

	// An index of all file inodes by (Name(), SourceGeneration()) pairs.
	//
	// INVARIANT: For each key k, !isDirName(k)
	// INVARIANT: For each key k, fileIndex[k].Name() == k.name
	// INVARIANT: For each key k, fileIndex[k].SourceGeneration() == k.gen
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.FileInode.
	//
	// GUARDED_BY(mu)
	fileIndex map[nameAndGen]*inode.FileInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

type nameAndGen struct {
	name string
	gen  int64
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:       clock,
		bucket:      bucket,
		inodes:      make(map[fuse.InodeID]inode.Inode),
		nextInodeID: fuse.RootInodeID + 1,
		dirIndex:    make(map[string]*inode.DirInode),
		fileIndex:   make(map[nameAndGen]*inode.FileInode),
		handles:     make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, fuse.RootInodeID, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for id, in := range fs.inodes {
		// Check the ID.
		if in.ID() != id {
			panic(fmt.Sprintf("ID mismatch: %v vs. %v", in.ID(), id))
		}

		// Check type-specific stuff.
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++

			if !isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected directory name: %s", typed.Name()))
			}

			if fs.dirIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

			if isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected file name: %s", typed.Name()))
			}

			nandg := nameAndGen{typed.Name(), typed.SourceGeneration()}
			if fs.fileIndex[nandg] != typed {
				panic(
					fmt.Sprintf(
						"fileIndex mismatch: %s, %v",
						typed.Name(),
						typed.SourceGeneration()))
			}

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirIndex length mismatch: %v vs. %v",
				len(fs.dirIndex),
				dirsSeen))
	}

	if len(fs.fileIndex) != filesSeen {
		panic(
			fmt.Sprintf(
				"fileIndex length mismatch: %v vs. %v",
				len(fs.fileIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Get attributes for the inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCKS_REQUIRED(in)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	in inode.Inode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = in.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Find a directory inode for the given object record. Create one if there
// isn't already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateDirInode(
	ctx context.Context,
	o *storage.Object) (in *inode.DirInode, err error) {
	// Do we already have an inode for this name?
	if in = fs.dirIndex[o.Name]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewDirInode(fs.bucket, id, o.Name)
	fs.inodes[id] = in
	fs.dirIndex[in.Name()] = in

	return
}

// Find a file inode for the given object record. Create one if there isn't
// already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateFileInode(
	ctx context.Context,
	o *storage.Object) (in *inode.FileInode, err error) {
	nandg := nameAndGen{
		name: o.Name,
		gen:  o.Generation,
	}

	// Do we already have an inode for this (name, generation) pair?
	if in = fs.fileIndex[nandg]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewFileInode(fs.bucket, id, o)
	fs.inodes[id] = in
	fs.fileIndex[nandg] = in

	return
}

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		return
	}

	// Is the child a directory or a file?
	var in inode.Inode
	if isDirName(o.Name) {
		in, err = fs.lookUpOrCreateDirInode(ctx, o)
	} else {
		in, err = fs.lookUpOrCreateFileInode(ctx, o)
	}

	if err != nil {
		return
	}

	in.Lock()
	defer in.Unlock()

	// Fill out the response.
	resp.Entry.Child = in.ID()
	if resp.Entry.Attributes, err = fs.getAttributes(ctx, in); err != nil {
		return
	}

	return
}

func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	// Grab its attributes.
	switch typed := in.(type) {
	case *inode.DirInode:
		resp.Attributes, err = fs.getAttributes(ctx, typed)
		if err != nil {
			err = fmt.Errorf("DirInode.Attributes: %v", err)
			return
		}

	default:
		panic(
			fmt.Sprintf(
				"Unknown inode type for ID %v: %v",
				req.Inode,
				reflect.TypeOf(in)))
	}

	return
}

func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.inodes[req.Inode].(*inode.DirInode)
	in.Lock()
	defer in.Unlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<commit_msg>Fixed a crash.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// TODO(jacobsa): Implement ForgetInode support in the fuse package, then
	// implement the method here and clean up these maps.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: For all keys k, inodes[k].ID() == k
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]inode.Inode

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, isDirName(k)
	// INVARIANT: For each key k, dirIndex[k].Name() == k
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirInode.
	//
	// GUARDED_BY(mu)
	dirIndex map[string]*inode.DirInode

	// An index of all file inodes by (Name(), SourceGeneration()) pairs.
	//
	// INVARIANT: For each key k, !isDirName(k)
	// INVARIANT: For each key k, fileIndex[k].Name() == k.name
	// INVARIANT: For each key k, fileIndex[k].SourceGeneration() == k.gen
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.FileInode.
	//
	// GUARDED_BY(mu)
	fileIndex map[nameAndGen]*inode.FileInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

type nameAndGen struct {
	name string
	gen  int64
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:       clock,
		bucket:      bucket,
		inodes:      make(map[fuse.InodeID]inode.Inode),
		nextInodeID: fuse.RootInodeID + 1,
		dirIndex:    make(map[string]*inode.DirInode),
		fileIndex:   make(map[nameAndGen]*inode.FileInode),
		handles:     make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, fuse.RootInodeID, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for id, in := range fs.inodes {
		// Check the ID.
		if in.ID() != id {
			panic(fmt.Sprintf("ID mismatch: %v vs. %v", in.ID(), id))
		}

		// Check type-specific stuff.
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++

			if !isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected directory name: %s", typed.Name()))
			}

			if fs.dirIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

			if isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected file name: %s", typed.Name()))
			}

			nandg := nameAndGen{typed.Name(), typed.SourceGeneration()}
			if fs.fileIndex[nandg] != typed {
				panic(
					fmt.Sprintf(
						"fileIndex mismatch: %s, %v",
						typed.Name(),
						typed.SourceGeneration()))
			}

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirIndex length mismatch: %v vs. %v",
				len(fs.dirIndex),
				dirsSeen))
	}

	if len(fs.fileIndex) != filesSeen {
		panic(
			fmt.Sprintf(
				"fileIndex length mismatch: %v vs. %v",
				len(fs.fileIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Get attributes for the inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCKS_REQUIRED(in)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	in inode.Inode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = in.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Find a directory inode for the given object record. Create one if there
// isn't already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateDirInode(
	ctx context.Context,
	o *storage.Object) (in *inode.DirInode, err error) {
	// Do we already have an inode for this name?
	if in = fs.dirIndex[o.Name]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewDirInode(fs.bucket, id, o.Name)
	fs.inodes[id] = in
	fs.dirIndex[in.Name()] = in

	return
}

// Find a file inode for the given object record. Create one if there isn't
// already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateFileInode(
	ctx context.Context,
	o *storage.Object) (in *inode.FileInode, err error) {
	nandg := nameAndGen{
		name: o.Name,
		gen:  o.Generation,
	}

	// Do we already have an inode for this (name, generation) pair?
	if in = fs.fileIndex[nandg]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewFileInode(fs.bucket, id, o)
	fs.inodes[id] = in
	fs.fileIndex[nandg] = in

	return
}

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		return
	}

	// Is the child a directory or a file?
	var in inode.Inode
	if isDirName(o.Name) {
		in, err = fs.lookUpOrCreateDirInode(ctx, o)
	} else {
		in, err = fs.lookUpOrCreateFileInode(ctx, o)
	}

	if err != nil {
		return
	}

	in.Lock()
	defer in.Unlock()

	// Fill out the response.
	resp.Entry.Child = in.ID()
	if resp.Entry.Attributes, err = fs.getAttributes(ctx, in); err != nil {
		return
	}

	return
}

func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	in.Lock()
	defer in.Unlock()

	// Grab its attributes.
	resp.Attributes, err = fs.getAttributes(ctx, in)
	if err != nil {
		return
	}

	return
}

func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.inodes[req.Inode].(*inode.DirInode)
	in.Lock()
	defer in.Unlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<|endoftext|>