<commit_before>package persist

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"strings"

	"gitlab.com/NebulousLabs/Sia/build"
	"gitlab.com/NebulousLabs/Sia/crypto"
)

// readJSON will try to read a persisted json object from a file.
func readJSON(meta Metadata, object interface{}, filename string) error {
	// Open the file.
	file, err := os.Open(filename)
	if os.IsNotExist(err) {
		return err
	}
	if err != nil {
		return build.ExtendErr("unable to open persisted json object file", err)
	}
	defer file.Close()

	// Read the metadata from the file.
	var header, version string
	dec := json.NewDecoder(file)
	if err := dec.Decode(&header); err != nil {
		return build.ExtendErr("unable to read header from persisted json object file", err)
	}
	if header != meta.Header {
		return ErrBadHeader
	}
	if err := dec.Decode(&version); err != nil {
		return build.ExtendErr("unable to read version from persisted json object file", err)
	}
	if version != meta.Version {
		return ErrBadVersion
	}

	// Read everything else.
	remainingBytes, err := ioutil.ReadAll(dec.Buffered())
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	// The buffer may or may not have read the rest of the file, read the rest
	// of the file to be certain.
	remainingBytesExtra, err := ioutil.ReadAll(file)
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	remainingBytes = append(remainingBytes, remainingBytesExtra...)

	// Determine whether the leading bytes contain a checksum. A proper checksum
	// will be 67 bytes (quote, 64 byte checksum, quote, newline). A manual
	// checksum will be the characters "manual\n" (9 characters). If neither
	// decode correctly, it is assumed that there is no checksum at all.
	var checksum crypto.Hash
	err = json.Unmarshal(remainingBytes[:67], &checksum)
	if err == nil && checksum == crypto.HashBytes(remainingBytes[68:]) {
		// Checksum is proper, and matches the data. Update the data portion to
		// exclude the checksum.
		remainingBytes = remainingBytes[68:]
	} else {
		// Cryptographic checksum failed, try interpreting a manual checksum.
		var manualChecksum string
		err := json.Unmarshal(remainingBytes[:8], &manualChecksum)
		if err == nil && manualChecksum == "manual" {
			// Manual checksum is proper. Update the remaining data to exclude
			// the manual checksum.
			remainingBytes = remainingBytes[9:]
		}
	}

	// Any valid checksum has been stripped off. There is also the case that no
	// checksum was written at all, which is ignored as a case - it's needed to
	// preserve compatibility with previous persist files.

	// Parse the json object.
	return json.Unmarshal(remainingBytes, &object)
}

// LoadJSON will load a persisted json object from disk.
func LoadJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Try opening the primary file.
	err = readJSON(meta, object, filename)
	if err == ErrBadHeader || err == ErrBadVersion || os.IsNotExist(err) {
		return err
	}
	if err != nil {
		// Try opening the temp file.
		err := readJSON(meta, object, filename+tempSuffix)
		if err != nil {
			return build.ExtendErr("unable to read persisted json object from disk", err)
		}
	}

	// Success.
	return nil
}

// SaveJSON will save a json object to disk in a durable, atomic way. The
// resulting file will have a checksum of the data as the third line. If
// manually editing files, the checksum line can be replaced with the 8
// characters "manual". This will cause the reader to accept the checksum even
// though the file has been changed.
func SaveJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Write the metadata to the buffer.
	buf := new(bytes.Buffer)
	enc := json.NewEncoder(buf)
	if err := enc.Encode(meta.Header); err != nil {
		return build.ExtendErr("unable to encode metadata header", err)
	}
	if err := enc.Encode(meta.Version); err != nil {
		return build.ExtendErr("unable to encode metadata version", err)
	}

	// Marshal the object into json and write the checksum + result to the
	// buffer.
	objBytes, err := json.MarshalIndent(object, "", "\t")
	if err != nil {
		return build.ExtendErr("unable to marshal the provided object", err)
	}
	checksum := crypto.HashBytes(objBytes)
	if err := enc.Encode(checksum); err != nil {
		return build.ExtendErr("unable to encode checksum", err)
	}
	buf.Write(objBytes)

	// Write out the data to the temp file, with a sync.
	data := buf.Bytes()
	err = func() (err error) {
		file, err := os.OpenFile(filename+tempSuffix, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open temp file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write temp file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Write out the data to the real file, with a sync.
	err = func() (err error) {
		file, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Success
	return nil
}
<commit_msg>truncate file before writing to it<commit_after>package persist

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"strings"

	"gitlab.com/NebulousLabs/Sia/build"
	"gitlab.com/NebulousLabs/Sia/crypto"

	"gitlab.com/NebulousLabs/errors"
)

// readJSON will try to read a persisted json object from a file.
func readJSON(meta Metadata, object interface{}, filename string) error {
	// Open the file.
	file, err := os.Open(filename)
	if os.IsNotExist(err) {
		return err
	}
	if err != nil {
		return build.ExtendErr("unable to open persisted json object file", err)
	}
	defer file.Close()

	// Read the metadata from the file.
	var header, version string
	dec := json.NewDecoder(file)
	if err := dec.Decode(&header); err != nil {
		return build.ExtendErr("unable to read header from persisted json object file", err)
	}
	if header != meta.Header {
		return ErrBadHeader
	}
	if err := dec.Decode(&version); err != nil {
		return build.ExtendErr("unable to read version from persisted json object file", err)
	}
	if version != meta.Version {
		return ErrBadVersion
	}

	// Read everything else.
	remainingBytes, err := ioutil.ReadAll(dec.Buffered())
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	// The buffer may or may not have read the rest of the file, read the rest
	// of the file to be certain.
	remainingBytesExtra, err := ioutil.ReadAll(file)
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	remainingBytes = append(remainingBytes, remainingBytesExtra...)

	// Determine whether the leading bytes contain a checksum. A proper checksum
	// will be 67 bytes (quote, 64 byte checksum, quote, newline). A manual
	// checksum will be the characters "manual\n" (9 characters). If neither
	// decode correctly, it is assumed that there is no checksum at all.
	var checksum crypto.Hash
	err = json.Unmarshal(remainingBytes[:67], &checksum)
	if err == nil && checksum == crypto.HashBytes(remainingBytes[68:]) {
		// Checksum is proper, and matches the data. Update the data portion to
		// exclude the checksum.
		remainingBytes = remainingBytes[68:]
	} else {
		// Cryptographic checksum failed, try interpreting a manual checksum.
		var manualChecksum string
		err := json.Unmarshal(remainingBytes[:8], &manualChecksum)
		if err == nil && manualChecksum == "manual" {
			// Manual checksum is proper. Update the remaining data to exclude
			// the manual checksum.
			remainingBytes = remainingBytes[9:]
		}
	}

	// Any valid checksum has been stripped off. There is also the case that no
	// checksum was written at all, which is ignored as a case - it's needed to
	// preserve compatibility with previous persist files.

	// Parse the json object.
	return json.Unmarshal(remainingBytes, &object)
}

// LoadJSON will load a persisted json object from disk.
func LoadJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Try opening the primary file.
	err = readJSON(meta, object, filename)
	if err == ErrBadHeader || err == ErrBadVersion || os.IsNotExist(err) {
		return err
	}
	if err != nil {
		// Try opening the temp file.
		err := readJSON(meta, object, filename+tempSuffix)
		if err != nil {
			return build.ExtendErr("unable to read persisted json object from disk", err)
		}
	}

	// Success.
	return nil
}

// SaveJSON will save a json object to disk in a durable, atomic way. The
// resulting file will have a checksum of the data as the third line. If
// manually editing files, the checksum line can be replaced with the 8
// characters "manual". This will cause the reader to accept the checksum even
// though the file has been changed.
func SaveJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Write the metadata to the buffer.
	buf := new(bytes.Buffer)
	enc := json.NewEncoder(buf)
	if err := enc.Encode(meta.Header); err != nil {
		return build.ExtendErr("unable to encode metadata header", err)
	}
	if err := enc.Encode(meta.Version); err != nil {
		return build.ExtendErr("unable to encode metadata version", err)
	}

	// Marshal the object into json and write the checksum + result to the
	// buffer.
	objBytes, err := json.MarshalIndent(object, "", "\t")
	if err != nil {
		return build.ExtendErr("unable to marshal the provided object", err)
	}
	checksum := crypto.HashBytes(objBytes)
	if err := enc.Encode(checksum); err != nil {
		return build.ExtendErr("unable to encode checksum", err)
	}
	buf.Write(objBytes)

	// Write out the data to the temp file, with a sync.
	data := buf.Bytes()
	err = func() (err error) {
		file, err := os.OpenFile(filename+tempSuffix, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open temp file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// If the data is greater than the file, we truncate it first to make
		// sure we don't run out of disk space mid-write.
		fi, err := file.Stat()
		if err != nil {
			return errors.AddContext(err, "failed to get FileInfo")
		}
		if int64(len(data)) > fi.Size() {
			if err := file.Truncate(int64(len(data))); err != nil {
				return errors.AddContext(err, "failed to reserve space for file")
			}
		}

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write temp file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Write out the data to the real file, with a sync.
	err = func() (err error) {
		file, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// If the data is greater than the file, we truncate it first to make
		// sure we don't run out of disk space mid-write.
		fi, err := file.Stat()
		if err != nil {
			return errors.AddContext(err, "failed to get FileInfo")
		}
		if int64(len(data)) > fi.Size() {
			if err := file.Truncate(int64(len(data))); err != nil {
				return errors.AddContext(err, "failed to reserve space for file")
			}
		}

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Success
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"strings"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on an object in GCS that allows random access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// All methods are safe for concurrent access. Concurrent readers and writers
// within process receive the same guarantees as with POSIX files.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not exist in
	// the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be nil if NoteLatest was never called.
	//
	// INVARIANT: If source != nil, source.Size >= 0
	// INVARIANT: If source != nil, source.Name == name
	source *storage.Object // GUARDED_BY(mu)

	// A local temporary file containing the contents of our source (or the empty
	// string if no source) along with any local modifications. The authority on
	// our view of the object when non-nil.
	//
	// A nil file is to be regarded as empty, but is not authoritative unless
	// source is also nil.
	localFile *os.File // GUARDED_BY(mu)

	// false if the contents of localFile may be different from the contents of
	// the object referred to by source. Sync needs to do work iff this is true.
	//
	// INVARIANT: If false, then source != nil.
	dirty bool // GUARDED_BY(mu)
}

var _ io.ReaderAt = &ObjectProxy{}
var _ io.WriterAt = &ObjectProxy{}

// Create a new view on the GCS object with the given name. The remote object
// is assumed to be non-existent, so that the local contents are empty. Use
// NoteLatest to change that if necessary.
func NewObjectProxy(
	bucket gcs.Bucket,
	name string) (op *ObjectProxy, err error) {
	op = &ObjectProxy{
		logger: getLogger(),
		bucket: bucket,
		name:   name,

		// Initial state: empty contents, dirty. (The remote object needs to be
		// truncated.)
		source:    nil,
		localFile: nil,
		dirty:     true,
	}

	op.mu = syncutil.NewInvariantMutex(op.checkInvariants)
	return
}

// SHARED_LOCKS_REQUIRED(op.mu)
func (op *ObjectProxy) checkInvariants() {
	if op.source != nil && op.source.Size <= 0 {
		if op.source.Size <= 0 {
			panic(fmt.Sprintf("Non-sensical source size: %v", op.source.Size))
		}

		if op.source.Name != op.name {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", op.source.Name, op.name))
		}
	}

	if !op.dirty && op.source == nil {
		panic("A clean proxy must have a source set.")
	}
}

// Inform the proxy object of the most recently observed generation of the
// object of interest in GCS.
//
// If this is no newer than the newest generation that has previously been
// observed, it is ignored. Otherwise, it becomes the definitive source of data
// for the object. Any local-only state is clobbered, including local
// modifications.
func (op *ObjectProxy) NoteLatest(o storage.Object) (err error) {
	// Sanity check the input.
	if o.Size < 0 {
		err = fmt.Errorf("Object contains negative size: %v", o.Size)
		return
	}

	if o.Name != op.name {
		err = fmt.Errorf("Object name mismatch: %s vs. %s", o.Name, op.name)
		return
	}

	// Do nothing if nothing has changed.
	if op.source != nil && op.source.Generation == o.Generation {
		return
	}

	// Throw out the local file, if any.
	if op.localFile != nil {
		path := op.localFile.Name()

		if err = op.localFile.Close(); err != nil {
			err = fmt.Errorf("Closing local file: %v", err)
			return
		}

		if err = os.Remove(path); err != nil {
			err = fmt.Errorf("Unlinking local file: %v", err)
			return
		}
	}

	// Reset state.
	op.source = &o
	op.localFile = nil
	op.dirty = false

	return
}

// Return the current size in bytes of our view of the content.
func (op *ObjectProxy) Size() (n uint64, err error) {
	// If we have a local file, it is authoritative.
	if op.localFile != nil {
		var fi os.FileInfo
		if fi, err = op.localFile.Stat(); err != nil {
			err = fmt.Errorf("localFile.Stat: %v", err)
			return
		}

		nSigned := fi.Size()
		if nSigned < 0 {
			err = fmt.Errorf("Stat returned nonsense size: %v", nSigned)
			return
		}

		n = uint64(nSigned)
		return
	}

	// Otherwise, if we have a source then it is authoritative.
	if op.source != nil {
		n = uint64(op.source.Size)
		return
	}

	// Otherwise, we are empty.
	return
}

// Make a random access read into our view of the content. May block for
// network access.
func (op *ObjectProxy) ReadAt(buf []byte, offset int64) (n int, err error) {
	if err = op.ensureLocalFile(); err != nil {
		return
	}

	n, err = op.localFile.ReadAt(buf, offset)
	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
func (op *ObjectProxy) WriteAt(buf []byte, offset int64) (n int, err error) {
	if err = op.ensureLocalFile(); err != nil {
		return
	}

	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)
	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than Size(). May block for network access. Not guaranteed to be
// reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(n uint64) (err error) {
	if err = op.ensureLocalFile(); err != nil {
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	op.dirty = true
	err = op.localFile.Truncate(int64(n))
	return
}

// Ensure that the remote object reflects the local state, returning a record
// for a generation that does. Clobbers the remote version. Does no work if the
// remote version is already up to date.
func (op *ObjectProxy) Sync(ctx context.Context) (o storage.Object, err error) {
	// Is there anything to do?
	if !op.dirty {
		o = *op.source
		return
	}

	// Choose a reader.
	var contents io.Reader
	if op.localFile != nil {
		contents = op.localFile
	} else {
		contents = strings.NewReader("")
	}

	// Create a new generation of the object.
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.name,
		},
		Contents: contents,
	}

	created, err := op.bucket.CreateObject(ctx, req)
	if err != nil {
		return
	}

	o = *created

	// Update local state.
	op.source = created
	op.dirty = false

	return
}

// Ensure that op.localFile != nil and contains the correct contents.
func (op *ObjectProxy) ensureLocalFile() (err error) {
	err = errors.New("TODO: ObjectProxy.ensureLocalFile")
	return
}
<commit_msg>Require external synchronization for ObjectProxy.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"strings"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on an object in GCS that allows random access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not exist in
	// the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be nil if NoteLatest was never called.
	//
	// INVARIANT: If source != nil, source.Size >= 0
	// INVARIANT: If source != nil, source.Name == name
	source *storage.Object

	// A local temporary file containing the contents of our source (or the empty
	// string if no source) along with any local modifications. The authority on
	// our view of the object when non-nil.
	//
	// A nil file is to be regarded as empty, but is not authoritative unless
	// source is also nil.
	localFile *os.File

	// false if the contents of localFile may be different from the contents of
	// the object referred to by source. Sync needs to do work iff this is true.
	//
	// INVARIANT: If false, then source != nil.
	dirty bool
}

var _ io.ReaderAt = &ObjectProxy{}
var _ io.WriterAt = &ObjectProxy{}

// Create a new view on the GCS object with the given name. The remote object
// is assumed to be non-existent, so that the local contents are empty. Use
// NoteLatest to change that if necessary.
func NewObjectProxy(
	bucket gcs.Bucket,
	name string) (op *ObjectProxy, err error) {
	op = &ObjectProxy{
		logger: getLogger(),
		bucket: bucket,
		name:   name,

		// Initial state: empty contents, dirty. (The remote object needs to be
		// truncated.)
		source:    nil,
		localFile: nil,
		dirty:     true,
	}

	return
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	if op.source != nil && op.source.Size <= 0 {
		if op.source.Size <= 0 {
			panic(fmt.Sprintf("Non-sensical source size: %v", op.source.Size))
		}

		if op.source.Name != op.name {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", op.source.Name, op.name))
		}
	}

	if !op.dirty && op.source == nil {
		panic("A clean proxy must have a source set.")
	}
}

// Inform the proxy object of the most recently observed generation of the
// object of interest in GCS.
//
// If this is no newer than the newest generation that has previously been
// observed, it is ignored. Otherwise, it becomes the definitive source of data
// for the object. Any local-only state is clobbered, including local
// modifications.
func (op *ObjectProxy) NoteLatest(o storage.Object) (err error) {
	// Sanity check the input.
	if o.Size < 0 {
		err = fmt.Errorf("Object contains negative size: %v", o.Size)
		return
	}

	if o.Name != op.name {
		err = fmt.Errorf("Object name mismatch: %s vs. %s", o.Name, op.name)
		return
	}

	// Do nothing if nothing has changed.
	if op.source != nil && op.source.Generation == o.Generation {
		return
	}

	// Throw out the local file, if any.
	if op.localFile != nil {
		path := op.localFile.Name()

		if err = op.localFile.Close(); err != nil {
			err = fmt.Errorf("Closing local file: %v", err)
			return
		}

		if err = os.Remove(path); err != nil {
			err = fmt.Errorf("Unlinking local file: %v", err)
			return
		}
	}

	// Reset state.
	op.source = &o
	op.localFile = nil
	op.dirty = false

	return
}

// Return the current size in bytes of our view of the content.
func (op *ObjectProxy) Size() (n uint64, err error) {
	// If we have a local file, it is authoritative.
	if op.localFile != nil {
		var fi os.FileInfo
		if fi, err = op.localFile.Stat(); err != nil {
			err = fmt.Errorf("localFile.Stat: %v", err)
			return
		}

		nSigned := fi.Size()
		if nSigned < 0 {
			err = fmt.Errorf("Stat returned nonsense size: %v", nSigned)
			return
		}

		n = uint64(nSigned)
		return
	}

	// Otherwise, if we have a source then it is authoritative.
	if op.source != nil {
		n = uint64(op.source.Size)
		return
	}

	// Otherwise, we are empty.
	return
}

// Make a random access read into our view of the content. May block for
// network access.
func (op *ObjectProxy) ReadAt(buf []byte, offset int64) (n int, err error) {
	if err = op.ensureLocalFile(); err != nil {
		return
	}

	n, err = op.localFile.ReadAt(buf, offset)
	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
func (op *ObjectProxy) WriteAt(buf []byte, offset int64) (n int, err error) {
	if err = op.ensureLocalFile(); err != nil {
		return
	}

	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)
	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than Size(). May block for network access. Not guaranteed to be
// reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(n uint64) (err error) {
	if err = op.ensureLocalFile(); err != nil {
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	op.dirty = true
	err = op.localFile.Truncate(int64(n))
	return
}

// Ensure that the remote object reflects the local state, returning a record
// for a generation that does. Clobbers the remote version. Does no work if the
// remote version is already up to date.
func (op *ObjectProxy) Sync(ctx context.Context) (o storage.Object, err error) {
	// Is there anything to do?
	if !op.dirty {
		o = *op.source
		return
	}

	// Choose a reader.
	var contents io.Reader
	if op.localFile != nil {
		contents = op.localFile
	} else {
		contents = strings.NewReader("")
	}

	// Create a new generation of the object.
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.name,
		},
		Contents: contents,
	}

	created, err := op.bucket.CreateObject(ctx, req)
	if err != nil {
		return
	}

	o = *created

	// Update local state.
	op.source = created
	op.dirty = false

	return
}

// Ensure that op.localFile != nil and contains the correct contents.
func (op *ObjectProxy) ensureLocalFile() (err error) {
	err = errors.New("TODO: ObjectProxy.ensureLocalFile")
	return
}
<|endoftext|>


<commit_before>package curses

// struct _win_st{};
// struct ldat{};
// #define _Bool int
// #define NCURSES_OPAQUE 1
// #include <curses.h>
// #cgo LDFLAGS: -lncurses
import "C"

import (
	"fmt"
	"unsafe"
)

type void unsafe.Pointer
type Window C.WINDOW

type CursesError struct {
	message string
}

func (ce CursesError) Error() string {
	return ce.message
}

// Cursor options.
const (
	CURS_HIDE = iota
	CURS_NORM
	CURS_HIGH
)

// Pointers to the values in curses, which may change values.
var Cols *int = nil
var Rows *int = nil

var Colors *int = nil
var ColorPairs *int = nil

var Tabsize *int = nil

// The window returned from C.initscr()
var Stdwin *Window = nil

// Initializes gocurses
func init() {
	Cols = (*int)(void(&C.COLS))
	Rows = (*int)(void(&C.LINES))

	Colors = (*int)(void(&C.COLORS))
	ColorPairs = (*int)(void(&C.COLOR_PAIRS))

	Tabsize = (*int)(void(&C.TABSIZE))
}

func Initscr() (*Window, error) {
	Stdwin = (*Window)(C.initscr())

	if Stdwin == nil {
		return nil, CursesError{"Initscr failed"}
	}

	return Stdwin, nil
}

func Newwin(rows int, cols int, starty int, startx int) (*Window, error) {
	nw := (*Window)(C.newwin(C.int(rows), C.int(cols), C.int(starty), C.int(startx)))

	if nw == nil {
		return nil, CursesError{"Failed to create window"}
	}

	return nw, nil
}

func (win *Window) Del() error {
	if int(C.delwin((*C.WINDOW)(win))) == 0 {
		return CursesError{"delete failed"}
	}
	return nil
}

func (win *Window) Subwin(rows int, cols int, starty int, startx int) (*Window, error) {
	sw := (*Window)(C.subwin((*C.WINDOW)(win), C.int(rows), C.int(cols), C.int(starty), C.int(startx)))

	if sw == nil {
		return nil, CursesError{"Failed to create window"}
	}

	return sw, nil
}

func (win *Window) Derwin(rows int, cols int, starty int, startx int) (*Window, error) {
	dw := (*Window)(C.derwin((*C.WINDOW)(win), C.int(rows), C.int(cols), C.int(starty), C.int(startx)))

	if dw == nil {
		return nil, CursesError{"Failed to create window"}
	}

	return dw, nil
}

func Start_color() error {
	if int(C.has_colors()) == 0 {
		return CursesError{"terminal does not support color"}
	}
	C.start_color()

	return nil
}

func Init_pair(pair int, fg int, bg int) error {
	if C.init_pair(C.short(pair), C.short(fg), C.short(bg)) == 0 {
		return CursesError{"Init_pair failed"}
	}
	return nil
}

func Color_pair(pair int) int32 {
	return int32(C.COLOR_PAIR(C.int(pair)))
}

func Noecho() error {
	if int(C.noecho()) == 0 {
		return CursesError{"Noecho failed"}
	}
	return nil
}

func DoUpdate() error {
	if int(C.doupdate()) == 0 {
		return CursesError{"Doupdate failed"}
	}
	return nil
}

func Echo() error {
	if int(C.noecho()) == 0 {
		return CursesError{"Echo failed"}
	}
	return nil
}

func Curs_set(c int) error {
	if C.curs_set(C.int(c)) == 0 {
		return CursesError{"Curs_set failed"}
	}
	return nil
}

func Nocbreak() error {
	if C.nocbreak() == 0 {
		return CursesError{"Nocbreak failed"}
	}
	return nil
}

func Cbreak() error {
	if C.cbreak() == 0 {
		return CursesError{"Cbreak failed"}
	}
	return nil
}

func Endwin() error {
	if C.endwin() == 0 {
		return CursesError{"Endwin failed"}
	}
	return nil
}

func (win *Window) Getch() int {
	return int(C.wgetch((*C.WINDOW)(win)))
}

func (win *Window) Addch(x, y int, c int32, flags int32) {
	C.mvwaddch((*C.WINDOW)(win), C.int(y), C.int(x), C.chtype(c)|C.chtype(flags))
}

// Since CGO currently can't handle varg C functions we'll mimic the
// ncurses addstr functions.
func (win *Window) Addstr(x, y int, str string, flags int32, v ...interface{}) {
	var newstr string
	if v != nil {
		newstr = fmt.Sprintf(str, v)
	} else {
		newstr = str
	}

	win.Move(x, y)

	for i := 0; i < len(newstr); i++ {
		C.waddch((*C.WINDOW)(win), C.chtype(newstr[i])|C.chtype(flags))
	}
}

// Normally Y is the first parameter passed in curses.
func (win *Window) Move(x, y int) {
	C.wmove((*C.WINDOW)(win), C.int(y), C.int(x))
}

func (win *Window) Resize(rows, cols int) {
	C.wresize((*C.WINDOW)(win), C.int(rows), C.int(cols))
}

func (w *Window) Keypad(tf bool) error {
	var outint int
	if tf == true {
		outint = 1
	}
	if tf == false {
		outint = 0
	}
	if C.keypad((*C.WINDOW)(w), C.int(outint)) == 0 {
		return CursesError{"Keypad failed"}
	}
	return nil
}

func (win *Window) Refresh() error {
	if C.wrefresh((*C.WINDOW)(win)) == 0 {
		return CursesError{"refresh failed"}
	}
	return nil
}

func (win *Window) Redrawln(beg_line, num_lines int) {
	C.wredrawln((*C.WINDOW)(win), C.int(beg_line), C.int(num_lines))
}

func (win *Window) Redraw() {
	C.redrawwin((*C.WINDOW)(win))
}

func (win *Window) Clear() {
	C.wclear((*C.WINDOW)(win))
}

func (win *Window) Erase() {
	C.werase((*C.WINDOW)(win))
}

func (win *Window) Clrtobot() {
	C.wclrtobot((*C.WINDOW)(win))
}

func (win *Window) Clrtoeol() {
	C.wclrtoeol((*C.WINDOW)(win))
}

func (win *Window) Box(verch, horch int) {
	C.box((*C.WINDOW)(win), C.chtype(verch), C.chtype(horch))
}

func (win *Window) Background(colour int32) {
	C.wbkgd((*C.WINDOW)(win), C.chtype(colour))
}

func (win *Window) Attron(flags int32) {
	C.wattron((*C.WINDOW)(win), C.int(flags))
}

func (win *Window) Attroff(flags int32) {
	C.wattroff((*C.WINDOW)(win), C.int(flags))
}
<commit_msg>Change line to properly allow variadic parameters to Addstr().<commit_after>package curses

// struct _win_st{};
// struct ldat{};
// #define _Bool int
// #define NCURSES_OPAQUE 1
// #include <curses.h>
// #cgo LDFLAGS: -lncurses
import "C"

import (
	"fmt"
	"unsafe"
)

type void unsafe.Pointer
type Window C.WINDOW

type CursesError struct {
	message string
}

func (ce CursesError) Error() string {
	return ce.message
}

// Cursor options.
const (
	CURS_HIDE = iota
	CURS_NORM
	CURS_HIGH
)

// Pointers to the values in curses, which may change values.
var Cols *int = nil
var Rows *int = nil

var Colors *int = nil
var ColorPairs *int = nil

var Tabsize *int = nil

// The window returned from C.initscr()
var Stdwin *Window = nil

// Initializes gocurses
func init() {
	Cols = (*int)(void(&C.COLS))
	Rows = (*int)(void(&C.LINES))

	Colors = (*int)(void(&C.COLORS))
	ColorPairs = (*int)(void(&C.COLOR_PAIRS))

	Tabsize = (*int)(void(&C.TABSIZE))
}

func Initscr() (*Window, error) {
	Stdwin = (*Window)(C.initscr())

	if Stdwin == nil {
		return nil, CursesError{"Initscr failed"}
	}

	return Stdwin, nil
}

func Newwin(rows int, cols int, starty int, startx int) (*Window, error) {
	nw := (*Window)(C.newwin(C.int(rows), C.int(cols), C.int(starty), C.int(startx)))

	if nw == nil {
		return nil, CursesError{"Failed to create window"}
	}

	return nw, nil
}

func (win *Window) Del() error {
	if int(C.delwin((*C.WINDOW)(win))) == 0 {
		return CursesError{"delete failed"}
	}
	return nil
}

func (win *Window) Subwin(rows int, cols int, starty int, startx int) (*Window, error) {
	sw := (*Window)(C.subwin((*C.WINDOW)(win), C.int(rows), C.int(cols), C.int(starty), C.int(startx)))

	if sw == nil {
		return nil, CursesError{"Failed to create window"}
	}

	return sw, nil
}

func (win *Window) Derwin(rows int, cols int, starty int, startx int) (*Window, error) {
	dw := (*Window)(C.derwin((*C.WINDOW)(win), C.int(rows), C.int(cols), C.int(starty), C.int(startx)))

	if dw == nil {
		return nil, CursesError{"Failed to create window"}
	}

	return dw, nil
}

func Start_color() error {
	if int(C.has_colors()) == 0 {
		return CursesError{"terminal does not support color"}
	}
	C.start_color()

	return nil
}

func Init_pair(pair int, fg int, bg int) error {
	if C.init_pair(C.short(pair), C.short(fg), C.short(bg)) == 0 {
		return CursesError{"Init_pair failed"}
	}
	return nil
}

func Color_pair(pair int) int32 {
	return int32(C.COLOR_PAIR(C.int(pair)))
}

func Noecho() error {
	if int(C.noecho()) == 0 {
		return CursesError{"Noecho failed"}
	}
	return nil
}

func DoUpdate() error {
	if int(C.doupdate()) == 0 {
		return CursesError{"Doupdate failed"}
	}
	return nil
}

func Echo() error {
	if int(C.noecho()) == 0 {
		return CursesError{"Echo failed"}
	}
	return nil
}

func Curs_set(c int) error {
	if C.curs_set(C.int(c)) == 0 {
		return CursesError{"Curs_set failed"}
	}
	return nil
}

func Nocbreak() error {
	if C.nocbreak() == 0 {
		return CursesError{"Nocbreak failed"}
	}
	return nil
}

func Cbreak() error {
	if C.cbreak() == 0 {
		return CursesError{"Cbreak failed"}
	}
	return nil
}

func Endwin() error {
	if C.endwin() == 0 {
		return CursesError{"Endwin failed"}
	}
	return nil
}

func (win *Window) Getch() int {
	return int(C.wgetch((*C.WINDOW)(win)))
}

func (win *Window) Addch(x, y int, c int32, flags int32) {
	C.mvwaddch((*C.WINDOW)(win), C.int(y), C.int(x), C.chtype(c)|C.chtype(flags))
}

// Since CGO currently can't handle varg C functions we'll mimic the
// ncurses addstr functions.
func (win *Window) Addstr(x, y int, str string, flags int32, v ...interface{}) {
	var newstr string
	if v != nil {
		newstr = fmt.Sprintf(str, v...)
	} else {
		newstr = str
	}

	win.Move(x, y)

	for i := 0; i < len(newstr); i++ {
		C.waddch((*C.WINDOW)(win), C.chtype(newstr[i])|C.chtype(flags))
	}
}

// Normally Y is the first parameter passed in curses.
func (win *Window) Move(x, y int) {
	C.wmove((*C.WINDOW)(win), C.int(y), C.int(x))
}

func (win *Window) Resize(rows, cols int) {
	C.wresize((*C.WINDOW)(win), C.int(rows), C.int(cols))
}

func (w *Window) Keypad(tf bool) error {
	var outint int
	if tf == true {
		outint = 1
	}
	if tf == false {
		outint = 0
	}
	if C.keypad((*C.WINDOW)(w), C.int(outint)) == 0 {
		return CursesError{"Keypad failed"}
	}
	return nil
}

func (win *Window) Refresh() error {
	if C.wrefresh((*C.WINDOW)(win)) == 0 {
		return CursesError{"refresh failed"}
	}
	return nil
}

func (win *Window) Redrawln(beg_line, num_lines int) {
	C.wredrawln((*C.WINDOW)(win), C.int(beg_line), C.int(num_lines))
}

func (win *Window) Redraw() {
	C.redrawwin((*C.WINDOW)(win))
}

func (win *Window) Clear() {
	C.wclear((*C.WINDOW)(win))
}

func (win *Window) Erase() {
	C.werase((*C.WINDOW)(win))
}

func (win *Window) Clrtobot() {
	C.wclrtobot((*C.WINDOW)(win))
}

func (win *Window) Clrtoeol() {
	C.wclrtoeol((*C.WINDOW)(win))
}

func (win *Window) Box(verch, horch int) {
	C.box((*C.WINDOW)(win), C.chtype(verch), C.chtype(horch))
}

func (win *Window) Background(colour int32) {
	C.wbkgd((*C.WINDOW)(win), C.chtype(colour))
}

func (win *Window) Attron(flags int32) {
	C.wattron((*C.WINDOW)(win), C.int(flags))
}

func (win *Window) Attroff(flags int32) {
	C.wattroff((*C.WINDOW)(win), C.int(flags))
}
<|endoftext|>


<commit_before>package aero

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/OneOfOne/xxhash"
	"github.com/aerogo/session"
	"github.com/fatih/color"
	"github.com/julienschmidt/httprouter"
	cache "github.com/patrickmn/go-cache"
	"github.com/tomasen/realip"
	"github.com/valyala/fasthttp"
)

// This should be close to the MTU size of a TCP packet.
// Regarding performance it makes no sense to compress smaller files.
// Bandwidth can be saved however the savings are minimal for small files
// and the overhead of compressing can lead up to a 75% reduction
// in server speed under high load. Therefore in this case
// we're trying to optimize for performance, not bandwidth.
const gzipThreshold = 1450

const (
	serverHeader                  = "Server"
	server                        = "Aero"
	cacheControlHeader            = "Cache-Control"
	cacheControlAlwaysValidate    = "must-revalidate"
	cacheControlMedia             = "public, max-age=864000"
	contentTypeOptionsHeader      = "X-Content-Type-Options"
	contentTypeOptions            = "nosniff"
	xssProtectionHeader           = "X-XSS-Protection"
	xssProtection                 = "1; mode=block"
	etagHeader                    = "ETag"
	contentTypeHeader             = "Content-Type"
	contentTypeHTML               = "text/html; charset=utf-8"
	contentTypeJavaScript         = "application/javascript; charset=utf-8"
	contentTypeJSON               = "application/json; charset=utf-8"
	contentTypePlainText          = "text/plain; charset=utf-8"
	contentEncodingHeader         = "Content-Encoding"
	contentEncodingGzip           = "gzip"
	contentLengthHeader           = "Content-Length"
	responseTimeHeader            = "X-Response-Time"
	ifNoneMatchHeader             = "If-None-Match"
	xFrameOptionsHeader           = "X-Frame-Options"
	xFrameOptions                 = "SAMEORIGIN"
	referrerPolicyHeader          = "Referrer-Policy"
	referrerPolicySameOrigin      = "no-referrer"
	strictTransportSecurityHeader = "Strict-Transport-Security"
	strictTransportSecurity       = "max-age=31536000; includeSubDomains; preload"
	contentSecurityPolicyHeader   = "Content-Security-Policy"
)

// Context ...
type Context struct {
	// net/http
	request  *http.Request
	response http.ResponseWriter
	params   httprouter.Params

	// A pointer to the application this request occured on.
	App *Application

	// Status code
	StatusCode int

	// Custom data
	Data interface{}

	// Start time
	start time.Time

	// User session
	session *session.Session
}

// Session returns the session of the context or creates and caches a new session.
func (ctx *Context) Session() *session.Session {
	// Return cached session if available.
	if ctx.session != nil {
		return ctx.session
	}

	// Check if the client has a session cookie already.
	cookie, err := ctx.request.Cookie("sid")

	if err == nil {
		sid := cookie.Value

		if session.IsValidID(sid) {
			ctx.session, err = ctx.App.Sessions.Store.Get(sid)

			if err != nil {
				color.Red(err.Error())
			}

			if ctx.session != nil {
				return ctx.session
			}
		}
	}

	// Create a new session
	ctx.session = ctx.App.Sessions.New()

	// Create a session cookie in the client
	ctx.createSessionCookie()

	return ctx.session
}

// createSessionCookie creates a session cookie in the client.
func (ctx *Context) createSessionCookie() {
	sessionCookie := http.Cookie{
		Name:     "sid",
		Value:    ctx.session.ID(),
		HttpOnly: true,
		Secure:   true,
		MaxAge:   ctx.App.Sessions.Duration,
		Path:     "/",
	}

	http.SetCookie(ctx.response, &sessionCookie)

	// HACK: Add SameSite attribute
	// Remove this once it's available inside http.Cookie
	// cookieData := ctx.response.Header().Get("Set-Cookie")
	// cookieData += "; SameSite=lax"
	// ctx.response.Header().Set("Set-Cookie", cookieData)
}

// HasSession indicates whether the client has a valid session or not.
func (ctx *Context) HasSession() bool {
	if ctx.session != nil {
		return true
	}

	cookie, err := ctx.request.Cookie("sid")

	if err != nil || !session.IsValidID(cookie.Value) {
		return false
	}

	ctx.session, _ = ctx.App.Sessions.Store.Get(cookie.Value)

	return ctx.session != nil
}

// JSON encodes the object to a JSON string and responds.
func (ctx *Context) JSON(value interface{}) string {
	bytes, _ := json.Marshal(value)

	ctx.SetResponseHeader(contentTypeHeader, contentTypeJSON)
	return string(bytes)
}

// HTML sends a HTML string.
func (ctx *Context) HTML(html string) string {
	ctx.SetResponseHeader(contentTypeHeader, contentTypeHTML)
	ctx.SetResponseHeader(contentTypeOptionsHeader, contentTypeOptions)
	ctx.SetResponseHeader(xssProtectionHeader, xssProtection)
	// ctx.SetResponseHeader(xFrameOptionsHeader, xFrameOptions)
	ctx.SetResponseHeader(referrerPolicyHeader, referrerPolicySameOrigin)

	if ctx.App.Security.Certificate != "" {
		ctx.SetResponseHeader(strictTransportSecurityHeader, strictTransportSecurity)
		ctx.SetResponseHeader(contentSecurityPolicyHeader, ctx.App.contentSecurityPolicy)
	}

	return html
}

// Text sends a plain text string.
func (ctx *Context) Text(text string) string {
	ctx.SetResponseHeader(contentTypeHeader, contentTypePlainText)
	return text
}

// JavaScript sends a script.
func (ctx *Context) JavaScript(code string) string {
	ctx.SetResponseHeader(contentTypeHeader, contentTypeJavaScript)
	return code
}

// File sends the contents of a local file and determines its mime type by extension.
func (ctx *Context) File(file string) string {
	extension := filepath.Ext(file)
	mimeType := mime.TypeByExtension(extension)
	data, _ := ioutil.ReadFile(file)

	if mimeType == "" {
		mimeType = http.DetectContentType(data)
	}

	ctx.SetResponseHeader(contentTypeHeader, mimeType)
	return string(data)
}

// TryWebP tries to serve a WebP image but will fall back to the specified extension if needed.
func (ctx *Context) TryWebP(path string, extension string) string {
	if ctx.CanUseWebP() {
		extension = ".webp"
	}

	return ctx.File(path + extension)
}

// Error should be used for sending error messages to the user.
func (ctx *Context) Error(statusCode int, explanation string, err error) string {
	ctx.StatusCode = statusCode
	ctx.SetResponseHeader(contentTypeHeader, contentTypeHTML)
	// ctx.App.Logger.Error(
	// 	color.RedString(explanation),
	// 	zap.String("error", err.Error()),
	// 	zap.String("url", ctx.request.RequestURI),
	// )
	if err != nil {
		detailed := err.Error()
		color.Red(detailed)
		return fmt.Sprintf("%s (%s)", explanation, detailed)
	}

	return explanation
}

// GetRequestHeader retrieves the value for the request header.
func (ctx *Context) GetRequestHeader(header string) string {
	return ctx.request.Header.Get(header)
}

// SetRequestHeader set the value for the request header.
func (ctx *Context) SetRequestHeader(header string, value string) {
	ctx.request.Header.Set(header, value)
}

// GetResponseHeader sets response header to value.
func (ctx *Context) GetResponseHeader(header string) string {
	return ctx.response.Header().Get(header)
}

// SetResponseHeader sets response header to value.
func (ctx *Context) SetResponseHeader(header string, value string) {
	ctx.response.Header().Set(header, value)
}

// URI returns the relative path, e.g. /blog/post/123.
func (ctx *Context) URI() string {
	return ctx.request.URL.Path
}

// SetURI sets the relative path, e.g. /blog/post/123.
func (ctx *Context) SetURI(b string) {
	ctx.request.URL.Path = b
}

// Get retrieves an URL parameter.
func (ctx *Context) Get(param string) string {
	return ctx.params.ByName(param)
}

// GetInt retrieves an URL parameter as an integer.
func (ctx *Context) GetInt(param string) (int, error) {
	return strconv.Atoi(ctx.Get(param))
}

// RequestBody returns the request body as a string.
func (ctx *Context) RequestBody() []byte {
	body, err := ioutil.ReadAll(ctx.request.Body)

	if err != nil {
		panic(err)
	}

	return body
}

// RequestBodyJSON returns the JSON parsed request body as map[string]interface{} or []interface{}.
func (ctx *Context) RequestBodyJSON() (interface{}, error) {
	var data interface{}
	err := json.Unmarshal(ctx.RequestBody(), &data)
	return data, err
}

// RequestBodyJSONObject returns the JSON parsed request body as map[string]interface{}.
func (ctx *Context) RequestBodyJSONObject() (map[string]interface{}, error) {
	json, err := ctx.RequestBodyJSON()

	if err != nil {
		return nil, err
	}

	data, formatOK := json.(map[string]interface{})

	if !formatOK {
		return nil, errors.New("Invalid format: Expected JSON object")
	}

	return data, nil
}

// RealIP tries to determine the real IP address of the request.
func (ctx *Context) RealIP() string {
	return realip.RealIP(ctx.request)
}

// UserAgent retrieves the user agent for the given request.
func (ctx *Context) UserAgent() string {
	ctx.request.URL.Query()
	return ctx.request.UserAgent()
}

// Query retrieves the value for the given URL query parameter.
func (ctx *Context) Query(param string) string {
	return ctx.request.URL.Query().Get(param)
}

// Redirect redirects to the given URL using status code 302.
func (ctx *Context) Redirect(url string) string {
	ctx.StatusCode = http.StatusFound
	ctx.SetResponseHeader("Location", url)
	return ""
}

// RedirectPermanently redirects to the given URL and indicates that this is a permanent change using status code 301.
func (ctx *Context) RedirectPermanently(url string) string {
	ctx.StatusCode = http.StatusPermanentRedirect
	ctx.SetResponseHeader("Location", url)
	return ""
}

// CanUseWebP checks the Accept header to find out if WebP is supported by the client's browser.
func (ctx *Context) CanUseWebP() bool {
	accept := ctx.GetRequestHeader("Accept")

	if strings.Index(accept, "image/webp") != -1 {
		return true
	}

	return false
}

// IsMediaResponse returns whether the given context has already set its content type to a media type.
func (ctx *Context) IsMediaResponse() bool {
	contentType := ctx.response.Header().Get(contentTypeHeader)
	return strings.HasPrefix(contentType, "image/") || strings.HasPrefix(contentType, "video/")
}

// respond responds either with raw code or gzipped if the
// code length is greater than the gzip threshold.
func (ctx *Context) respond(code string) {
	ctx.respondBytes(StringToBytesUnsafe(code))
}

// respondBytes responds either with raw code or gzipped if the
// code length is greater than the gzip threshold. Requires a byte slice.
func (ctx *Context) respondBytes(b []byte) {
	response := ctx.response
	header := response.Header()
	isMedia := ctx.IsMediaResponse()

	// Headers
	if isMedia {
		header.Set(cacheControlHeader, cacheControlMedia)
	} else {
		header.Set(cacheControlHeader, cacheControlAlwaysValidate)
		header.Set(serverHeader, server)
		header.Set(responseTimeHeader, strconv.FormatInt(time.Since(ctx.start).Nanoseconds()/1000, 10)+" us")
	}

	// Small response
	if len(b) < gzipThreshold {
		header.Set(contentLengthHeader, strconv.Itoa(len(b)))
		response.WriteHeader(ctx.StatusCode)
		response.Write(b)
		return
	}

	// ETag generation
	h := xxhash.NewS64(0)
	h.Write(b)
	etag := strconv.FormatUint(h.Sum64(), 16)

	// If client cache is up to date, send 304 with no response body.
	clientETag := ctx.request.Header.Get(ifNoneMatchHeader)

	if etag == clientETag {
		response.WriteHeader(304)
		return
	}

	// Set ETag
	header.Set(etagHeader, etag)

	// No GZip?
	if !ctx.App.Config.GZip || isMedia {
		header.Set(contentLengthHeader, strconv.Itoa(len(b)))
		response.WriteHeader(ctx.StatusCode)
		response.Write(b)
		return
	}

	// GZip
	header.Set(contentEncodingHeader, contentEncodingGzip)

	if ctx.App.Config.GZipCache {
		cachedResponse, found := ctx.App.gzipCache.Get(etag)

		if found {
			cachedResponseBytes := cachedResponse.([]byte)
			header.Set(contentLengthHeader, strconv.Itoa(len(cachedResponseBytes)))
			response.WriteHeader(ctx.StatusCode)
			response.Write(cachedResponseBytes)
			return
		}
	}

	var buffer bytes.Buffer
	writer := bufio.NewWriter(&buffer)
	fasthttp.WriteGzipLevel(writer, b, 9)
	writer.Flush()
	gzippedBytes := buffer.Bytes()

	header.Set(contentLengthHeader, strconv.Itoa(len(gzippedBytes)))
	response.WriteHeader(ctx.StatusCode)
	response.Write(gzippedBytes)

	if ctx.App.Config.GZipCache {
		ctx.App.gzipCache.Set(etag, gzippedBytes, cache.DefaultExpiration)
	}
}
<commit_msg>Removed X-Response-Time header<commit_after>package aero

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/OneOfOne/xxhash"
	"github.com/aerogo/session"
	"github.com/fatih/color"
	"github.com/julienschmidt/httprouter"
	cache "github.com/patrickmn/go-cache"
	"github.com/tomasen/realip"
	"github.com/valyala/fasthttp"
)

// This should be close to the MTU size of a TCP packet.
// Regarding performance it makes no sense to compress smaller files.
// Bandwidth can be saved however the savings are minimal for small files
// and the overhead of compressing can lead up to a 75% reduction
// in server speed under high load. Therefore in this case
// we're trying to optimize for performance, not bandwidth.
const gzipThreshold = 1450

const (
	serverHeader                  = "Server"
	server                        = "Aero"
	cacheControlHeader            = "Cache-Control"
	cacheControlAlwaysValidate    = "must-revalidate"
	cacheControlMedia             = "public, max-age=864000"
	contentTypeOptionsHeader      = "X-Content-Type-Options"
	contentTypeOptions            = "nosniff"
	xssProtectionHeader           = "X-XSS-Protection"
	xssProtection                 = "1; mode=block"
	etagHeader                    = "ETag"
	contentTypeHeader             = "Content-Type"
	contentTypeHTML               = "text/html; charset=utf-8"
	contentTypeJavaScript         = "application/javascript; charset=utf-8"
	contentTypeJSON               = "application/json; charset=utf-8"
	contentTypePlainText          = "text/plain; charset=utf-8"
	contentEncodingHeader         = "Content-Encoding"
	contentEncodingGzip           = "gzip"
	contentLengthHeader           = "Content-Length"
	responseTimeHeader            = "X-Response-Time"
	ifNoneMatchHeader             = "If-None-Match"
	xFrameOptionsHeader           = "X-Frame-Options"
	xFrameOptions                 = "SAMEORIGIN"
	referrerPolicyHeader          = "Referrer-Policy"
	referrerPolicySameOrigin      = "no-referrer"
	strictTransportSecurityHeader = "Strict-Transport-Security"
	strictTransportSecurity       = "max-age=31536000; includeSubDomains; preload"
	contentSecurityPolicyHeader   = "Content-Security-Policy"
)

// Context ...
type Context struct {
	// net/http
	request  *http.Request
	response http.ResponseWriter
	params   httprouter.Params

	// A pointer to the application this request occured on.
	App *Application

	// Status code
	StatusCode int

	// Custom data
	Data interface{}

	// Start time
	start time.Time

	// User session
	session *session.Session
}

// Session returns the session of the context or creates and caches a new session.
func (ctx *Context) Session() *session.Session {
	// Return cached session if available.
	if ctx.session != nil {
		return ctx.session
	}

	// Check if the client has a session cookie already.
	cookie, err := ctx.request.Cookie("sid")

	if err == nil {
		sid := cookie.Value

		if session.IsValidID(sid) {
			ctx.session, err = ctx.App.Sessions.Store.Get(sid)

			if err != nil {
				color.Red(err.Error())
			}

			if ctx.session != nil {
				return ctx.session
			}
		}
	}

	// Create a new session
	ctx.session = ctx.App.Sessions.New()

	// Create a session cookie in the client
	ctx.createSessionCookie()

	return ctx.session
}

// createSessionCookie creates a session cookie in the client.
func (ctx *Context) createSessionCookie() {
	sessionCookie := http.Cookie{
		Name:     "sid",
		Value:    ctx.session.ID(),
		HttpOnly: true,
		Secure:   true,
		MaxAge:   ctx.App.Sessions.Duration,
		Path:     "/",
	}

	http.SetCookie(ctx.response, &sessionCookie)

	// HACK: Add SameSite attribute
	// Remove this once it's available inside http.Cookie
	// cookieData := ctx.response.Header().Get("Set-Cookie")
	// cookieData += "; SameSite=lax"
	// ctx.response.Header().Set("Set-Cookie", cookieData)
}

// HasSession indicates whether the client has a valid session or not.
func (ctx *Context) HasSession() bool {
	if ctx.session != nil {
		return true
	}

	cookie, err := ctx.request.Cookie("sid")

	if err != nil || !session.IsValidID(cookie.Value) {
		return false
	}

	ctx.session, _ = ctx.App.Sessions.Store.Get(cookie.Value)

	return ctx.session != nil
}

// JSON encodes the object to a JSON string and responds.
func (ctx *Context) JSON(value interface{}) string {
	bytes, _ := json.Marshal(value)

	ctx.SetResponseHeader(contentTypeHeader, contentTypeJSON)
	return string(bytes)
}

// HTML sends a HTML string.
func (ctx *Context) HTML(html string) string {
	ctx.SetResponseHeader(contentTypeHeader, contentTypeHTML)
	ctx.SetResponseHeader(contentTypeOptionsHeader, contentTypeOptions)
	ctx.SetResponseHeader(xssProtectionHeader, xssProtection)
	// ctx.SetResponseHeader(xFrameOptionsHeader, xFrameOptions)
	ctx.SetResponseHeader(referrerPolicyHeader, referrerPolicySameOrigin)

	if ctx.App.Security.Certificate != "" {
		ctx.SetResponseHeader(strictTransportSecurityHeader, strictTransportSecurity)
		ctx.SetResponseHeader(contentSecurityPolicyHeader, ctx.App.contentSecurityPolicy)
	}

	return html
}

// Text sends a plain text string.
func (ctx *Context) Text(text string) string {
	ctx.SetResponseHeader(contentTypeHeader, contentTypePlainText)
	return text
}

// JavaScript sends a script.
func (ctx *Context) JavaScript(code string) string {
	ctx.SetResponseHeader(contentTypeHeader, contentTypeJavaScript)
	return code
}

// File sends the contents of a local file and determines its mime type by extension.
func (ctx *Context) File(file string) string {
	extension := filepath.Ext(file)
	mimeType := mime.TypeByExtension(extension)
	data, _ := ioutil.ReadFile(file)

	if mimeType == "" {
		mimeType = http.DetectContentType(data)
	}

	ctx.SetResponseHeader(contentTypeHeader, mimeType)
	return string(data)
}

// TryWebP tries to serve a WebP image but will fall back to the specified extension if needed.
func (ctx *Context) TryWebP(path string, extension string) string {
	if ctx.CanUseWebP() {
		extension = ".webp"
	}

	return ctx.File(path + extension)
}

// Error should be used for sending error messages to the user.
func (ctx *Context) Error(statusCode int, explanation string, err error) string {
	ctx.StatusCode = statusCode
	ctx.SetResponseHeader(contentTypeHeader, contentTypeHTML)
	// ctx.App.Logger.Error(
	// 	color.RedString(explanation),
	// 	zap.String("error", err.Error()),
	// 	zap.String("url", ctx.request.RequestURI),
	// )
	if err != nil {
		detailed := err.Error()
		color.Red(detailed)
		return fmt.Sprintf("%s (%s)", explanation, detailed)
	}

	return explanation
}

// GetRequestHeader retrieves the value for the request header.
func (ctx *Context) GetRequestHeader(header string) string {
	return ctx.request.Header.Get(header)
}

// SetRequestHeader set the value for the request header.
func (ctx *Context) SetRequestHeader(header string, value string) {
	ctx.request.Header.Set(header, value)
}

// GetResponseHeader sets response header to value.
func (ctx *Context) GetResponseHeader(header string) string {
	return ctx.response.Header().Get(header)
}

// SetResponseHeader sets response header to value.
func (ctx *Context) SetResponseHeader(header string, value string) {
	ctx.response.Header().Set(header, value)
}

// URI returns the relative path, e.g. /blog/post/123.
func (ctx *Context) URI() string {
	return ctx.request.URL.Path
}

// SetURI sets the relative path, e.g. /blog/post/123.
func (ctx *Context) SetURI(b string) {
	ctx.request.URL.Path = b
}

// Get retrieves an URL parameter.
func (ctx *Context) Get(param string) string {
	return ctx.params.ByName(param)
}

// GetInt retrieves an URL parameter as an integer.
func (ctx *Context) GetInt(param string) (int, error) {
	return strconv.Atoi(ctx.Get(param))
}

// RequestBody returns the request body as a string.
func (ctx *Context) RequestBody() []byte {
	body, err := ioutil.ReadAll(ctx.request.Body)

	if err != nil {
		panic(err)
	}

	return body
}

// RequestBodyJSON returns the JSON parsed request body as map[string]interface{} or []interface{}.
func (ctx *Context) RequestBodyJSON() (interface{}, error) {
	var data interface{}
	err := json.Unmarshal(ctx.RequestBody(), &data)
	return data, err
}

// RequestBodyJSONObject returns the JSON parsed request body as map[string]interface{}.
func (ctx *Context) RequestBodyJSONObject() (map[string]interface{}, error) {
	json, err := ctx.RequestBodyJSON()

	if err != nil {
		return nil, err
	}

	data, formatOK := json.(map[string]interface{})

	if !formatOK {
		return nil, errors.New("Invalid format: Expected JSON object")
	}

	return data, nil
}

// RealIP tries to determine the real IP address of the request.
func (ctx *Context) RealIP() string {
	return realip.RealIP(ctx.request)
}

// UserAgent retrieves the user agent for the given request.
func (ctx *Context) UserAgent() string {
	ctx.request.URL.Query()
	return ctx.request.UserAgent()
}

// Query retrieves the value for the given URL query parameter.
func (ctx *Context) Query(param string) string {
	return ctx.request.URL.Query().Get(param)
}

// Redirect redirects to the given URL using status code 302.
func (ctx *Context) Redirect(url string) string {
	ctx.StatusCode = http.StatusFound
	ctx.SetResponseHeader("Location", url)
	return ""
}

// RedirectPermanently redirects to the given URL and indicates that this is a permanent change using status code 301.
func (ctx *Context) RedirectPermanently(url string) string {
	ctx.StatusCode = http.StatusPermanentRedirect
	ctx.SetResponseHeader("Location", url)
	return ""
}

// CanUseWebP checks the Accept header to find out if WebP is supported by the client's browser.
func (ctx *Context) CanUseWebP() bool {
	accept := ctx.GetRequestHeader("Accept")

	if strings.Index(accept, "image/webp") != -1 {
		return true
	}

	return false
}

// IsMediaResponse returns whether the given context has already set its content type to a media type.
func (ctx *Context) IsMediaResponse() bool {
	contentType := ctx.response.Header().Get(contentTypeHeader)
	return strings.HasPrefix(contentType, "image/") || strings.HasPrefix(contentType, "video/")
}

// respond responds either with raw code or gzipped if the
// code length is greater than the gzip threshold.
func (ctx *Context) respond(code string) {
	ctx.respondBytes(StringToBytesUnsafe(code))
}

// respondBytes responds either with raw code or gzipped if the
// code length is greater than the gzip threshold. Requires a byte slice.
func (ctx *Context) respondBytes(b []byte) {
	response := ctx.response
	header := response.Header()
	isMedia := ctx.IsMediaResponse()

	// Headers
	if isMedia {
		header.Set(cacheControlHeader, cacheControlMedia)
	} else {
		header.Set(cacheControlHeader, cacheControlAlwaysValidate)
		header.Set(serverHeader, server)
		// header.Set(responseTimeHeader, strconv.FormatInt(time.Since(ctx.start).Nanoseconds()/1000, 10)+" us")
	}

	// Small response
	if len(b) < gzipThreshold {
		header.Set(contentLengthHeader, strconv.Itoa(len(b)))
		response.WriteHeader(ctx.StatusCode)
		response.Write(b)
		return
	}

	// ETag generation
	h := xxhash.NewS64(0)
	h.Write(b)
	etag := strconv.FormatUint(h.Sum64(), 16)

	// If client cache is up to date, send 304 with no response body.
	clientETag := ctx.request.Header.Get(ifNoneMatchHeader)

	if etag == clientETag {
		response.WriteHeader(304)
		return
	}

	// Set ETag
	header.Set(etagHeader, etag)

	// No GZip?
	if !ctx.App.Config.GZip || isMedia {
		header.Set(contentLengthHeader, strconv.Itoa(len(b)))
		response.WriteHeader(ctx.StatusCode)
		response.Write(b)
		return
	}

	// GZip
	header.Set(contentEncodingHeader, contentEncodingGzip)

	if ctx.App.Config.GZipCache {
		cachedResponse, found := ctx.App.gzipCache.Get(etag)

		if found {
			cachedResponseBytes := cachedResponse.([]byte)
			header.Set(contentLengthHeader, strconv.Itoa(len(cachedResponseBytes)))
			response.WriteHeader(ctx.StatusCode)
			response.Write(cachedResponseBytes)
			return
		}
	}

	var buffer bytes.Buffer
	writer := bufio.NewWriter(&buffer)
	fasthttp.WriteGzipLevel(writer, b, 9)
	writer.Flush()
	gzippedBytes := buffer.Bytes()

	header.Set(contentLengthHeader, strconv.Itoa(len(gzippedBytes)))
	response.WriteHeader(ctx.StatusCode)
	response.Write(gzippedBytes)

	if ctx.App.Config.GZipCache {
		ctx.App.gzipCache.Set(etag, gzippedBytes, cache.DefaultExpiration)
	}
}
<|endoftext|>


<commit_before>package igc

import (
	"bufio"
	"fmt"
	"github.com/twpayne/gogeom/geom"
	"io"
	"strconv"
	"strings"
	"time"
)

type Errors map[int]error

func (es Errors) Error() string {
	ss := make([]string, len(es))
	for i, e := range es {
		ss[i] = e.Error()
	}
	return strings.Join(ss, "\n")
}

type parser struct {
	pointMs          []geom.PointM
	year, month, day int
}

func newParser() *parser {
	p := new(parser)
	p.year = 2000
	p.month = 1
	p.day = 1
	return p
}

func (p *parser) parseB(line string) error {
	var err error
	var hour, minute, second int
	if hour, err = strconv.Atoi(line[1:3]); err != nil {
		return err
	}
	if minute, err = strconv.Atoi(line[3:5]); err != nil {
		return err
	}
	if second, err = strconv.Atoi(line[5:7]); err != nil {
		return err
	}
	var latDeg, latMin int
	if latDeg, err = strconv.Atoi(line[7:9]); err != nil {
		return err
	}
	if latMin, err = strconv.Atoi(line[9:14]); err != nil {
		return err
	}
	lat := float64(latDeg) + float64(latMin)/60000.
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	var lngDeg, lngMin int
	lngDeg, err = strconv.Atoi(line[15:18])
	if err != nil {
		return err
	}
	lngMin, err = strconv.Atoi(line[18:23])
	if err != nil {
		return err
	}
	lng := float64(lngDeg) + float64(lngMin)/60000.
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, 0, time.UTC)
	pointM := geom.PointM{lng, lat, float64(date.UnixNano()) / 1e9}
	p.pointMs = append(p.pointMs, pointM)
	return nil
}

func (p *parser) parseH(line string) error {
	switch {
	case strings.HasPrefix(line, "HFDTE"):
		return p.parseHFDTE(line)
	default:
		return nil
	}
}

func (p *parser) parseHFDTE(line string) error {
	var err error
	var day, month, year int
	if day, err = strconv.Atoi(line[5:7]); err != nil {
		return err
	}
	if month, err = strconv.Atoi(line[7:9]); err != nil {
		return err
	}
	if year, err = strconv.Atoi(line[9:11]); err != nil {
		return err
	}
	// FIXME check for invalid dates
	p.day = day
	p.month = month
	if year < 70 {
		p.year = 2000 + year
	} else {
		p.year = 1970 + year
	}
	return nil
}

func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	default:
		return nil
	}
}

func Read(r io.Reader) ([]geom.PointM, error) {
	errors := make(Errors)
	p := newParser()
	s := bufio.NewScanner(r)
	line := 0
	for s.Scan() {
		line++
		if err := p.parseLine(s.Text()); err != nil {
			errors[line] = err
		}
	}
	if len(errors) == 0 {
		return p.pointMs, nil
	} else {
		return p.pointMs, errors
	}
}
<commit_msg>Add parseDec<commit_after>package igc

import (
	"bufio"
	"fmt"
	"github.com/twpayne/gogeom/geom"
	"io"
	"strconv"
	"strings"
	"time"
)

type Errors map[int]error

func (es Errors) Error() string {
	ss := make([]string, len(es))
	for i, e := range es {
		ss[i] = e.Error()
	}
	return strings.Join(ss, "\n")
}

func parseDec(s string, start, stop int) (int, error) {
	result := 0
	for i := start; i < stop; i++ {
		if c := s[i]; '0' <= c && c <= '9' {
			result = 10*result + int(c) - '0'
		} else {
			return 0, fmt.Errorf("invalid")
		}
	}
	return result, nil
}

type parser struct {
	pointMs          []geom.PointM
	year, month, day int
}

func newParser() *parser {
	p := new(parser)
	p.year = 2000
	p.month = 1
	p.day = 1
	return p
}

func (p *parser) parseB(line string) error {
	var err error
	var hour, minute, second int
	if hour, err = strconv.Atoi(line[1:3]); err != nil {
		return err
	}
	if minute, err = strconv.Atoi(line[3:5]); err != nil {
		return err
	}
	if second, err = strconv.Atoi(line[5:7]); err != nil {
		return err
	}
	var latDeg, latMin int
	if latDeg, err = strconv.Atoi(line[7:9]); err != nil {
		return err
	}
	if latMin, err = strconv.Atoi(line[9:14]); err != nil {
		return err
	}
	lat := float64(latDeg) + float64(latMin)/60000.
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	var lngDeg, lngMin int
	lngDeg, err = strconv.Atoi(line[15:18])
	if err != nil {
		return err
	}
	lngMin, err = strconv.Atoi(line[18:23])
	if err != nil {
		return err
	}
	lng := float64(lngDeg) + float64(lngMin)/60000.
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, 0, time.UTC)
	pointM := geom.PointM{lng, lat, float64(date.UnixNano()) / 1e9}
	p.pointMs = append(p.pointMs, pointM)
	return nil
}

func (p *parser) parseH(line string) error {
	switch {
	case strings.HasPrefix(line, "HFDTE"):
		return p.parseHFDTE(line)
	default:
		return nil
	}
}

func (p *parser) parseHFDTE(line string) error {
	var err error
	var day, month, year int
	if day, err = strconv.Atoi(line[5:7]); err != nil {
		return err
	}
	if month, err = strconv.Atoi(line[7:9]); err != nil {
		return err
	}
	if year, err = strconv.Atoi(line[9:11]); err != nil {
		return err
	}
	// FIXME check for invalid dates
	p.day = day
	p.month = month
	if year < 70 {
		p.year = 2000 + year
	} else {
		p.year = 1970 + year
	}
	return nil
}

func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	default:
		return nil
	}
}

func Read(r io.Reader) ([]geom.PointM, error) {
	errors := make(Errors)
	p := newParser()
	s := bufio.NewScanner(r)
	line := 0
	for s.Scan() {
		line++
		if err := p.parseLine(s.Text()); err != nil {
			errors[line] = err
		}
	}
	if len(errors) == 0 {
		return p.pointMs, nil
	} else {
		return p.pointMs, errors
	}
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"
	"testing"

	"github.com/NebulousLabs/Sia/build"
)

type updateHandler struct {
	version string
}

func (uh *updateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.URL.String() {
	case "/current/MANIFEST":
		fmt.Fprintf(w, "%s\nsiad\n", uh.version)
	case "/current/siad":
		fmt.Fprint(w, "yep this is siad")
	case "/current/siad.sig":
		fmt.Fprint(w, "and this is totally a signature")
	default:
		http.NotFound(w, r)
	}
}

// TestUpdate checks that updates work properly.
func TestSignedUpdate(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestSignedUpdate")
	if err != nil {
		t.Fatal(err)
	}

	// to test the update process, we need to spoof the update server
	uh := new(updateHandler)
	http.Handle("/", uh)
	go http.ListenAndServe(":8080", nil)
	updateURL = "http://localhost:8080"

	// same version
	uh.version = build.Version
	var info UpdateInfo
	st.getAPI("/daemon/updates/check", &info)
	if info.Available {
		t.Error("new version should not be available")
	}

	// newer version
	uh.version = "100.0"
	st.getAPI("/daemon/updates/check", &info)
	if !info.Available {
		t.Error("new version should be available")
	}

	// apply (bad signature)
	resp, err := HttpGET("http://" + st.server.listener.Addr().String() + "/daemon/updates/apply?version=current")
	if err != nil {
		t.Fatal("GET failed:", err)
	}
	if resp.StatusCode != http.StatusInternalServerError {
		t.Error("expected internal server error, got", resp.StatusCode)
	}
}

func TestVersion(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestSignedUpdate")
	if err != nil {
		t.Fatal(err)
	}
	var version string
	st.getAPI("/daemon/version", &version)
	if version != build.Version {
		t.Fatalf("/daemon/version reporting bad version: expected %v, got %v", build.Version, version)
	}
}
<commit_msg>fix daemon test<commit_after>package api

import (
	"fmt"
	"net/http"
	"testing"

	"github.com/NebulousLabs/Sia/build"
)

type updateHandler struct {
	version string
}

func (uh *updateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.URL.String() {
	case "/current/MANIFEST":
		fmt.Fprintf(w, "%s\nsiad\n", uh.version)
	case "/current/siad":
		fmt.Fprint(w, "yep this is siad")
	case "/current/siad.sig":
		fmt.Fprint(w, "and this is totally a signature")
	default:
		http.NotFound(w, r)
	}
}

// TestUpdate checks that updates work properly.
func TestSignedUpdate(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestSignedUpdate")
	if err != nil {
		t.Fatal(err)
	}

	// to test the update process, we need to spoof the update server
	uh := new(updateHandler)
	http.Handle("/", uh)
	go http.ListenAndServe(":8080", nil)
	updateURL = "http://localhost:8080"

	// same version
	uh.version = build.Version
	var info UpdateInfo
	st.getAPI("/daemon/updates/check", &info)
	if info.Available {
		t.Error("new version should not be available")
	}

	// newer version
	uh.version = "100.0"
	err = st.getAPI("/daemon/updates/check", &info)
	if err != nil {
		t.Error(err)
	} else if !info.Available {
		t.Error("new version should be available")
	}

	// apply (bad signature)
	resp, err := HttpGET("http://" + st.server.listener.Addr().String() + "/daemon/updates/apply?version=current")
	if err != nil {
		t.Fatal("GET failed:", err)
	}
	if resp.StatusCode != http.StatusInternalServerError {
		t.Error("expected internal server error, got", resp.StatusCode)
	}
}

func TestVersion(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestSignedUpdate")
	if err != nil {
		t.Fatal(err)
	}
	var version string
	st.getAPI("/daemon/version", &version)
	if version != build.Version {
		t.Fatalf("/daemon/version reporting bad version: expected %v, got %v", build.Version, version)
	}
}
<|endoftext|>


<commit_before>package rainspub

import (
	"fmt"
	"sort"
	"time"

	"golang.org/x/crypto/ed25519"

	log "github.com/inconshreveable/log15"
	"github.com/netsec-ethz/rains/rainsSiglib"
	"github.com/netsec-ethz/rains/rainslib"
	"github.com/netsec-ethz/rains/utils/protoParser"
	"github.com/netsec-ethz/rains/utils/zoneFileParser"
)

//Init starts the zone information publishing process according to the provided config.
func Init(inputConfig Config) {
	config = inputConfig
	parser = zoneFileParser.Parser{}
	signatureEncoder = zoneFileParser.Parser{}
	publish()
}

//publish calls the relevant library function to publish information according to the provided
//config during initialization.
func publish() {
	sections, err := loadZonefile()
	if err != nil {
		return
	}
	if config.DoSharding {
		var assertions []*rainslib.AssertionSection
		zone, context := "", ""
		for i, section := range sections {
			switch sec := section.(type) {
			case *rainslib.ZoneSection:
				zone = sec.SubjectZone
				context = sec.Context
			case *rainslib.ShardSection:
				if !config.KeepExistingShards {
					//remove shard
					sections = append(sections[:i], sections[i+1:]...)
				}
			case *rainslib.AssertionSection:
				assertions = append(assertions, sec)
			}
		}
		if config.MaxShardSize > 0 {
			//TODO CFE to implement
		} else if config.NofAssertionsPerShard > 0 {
			//TODO CFE how to combine the return value with previous set of shard/assertion
			groupAssertionsToShards(zone, context, assertions, config.NofAssertionsPerShard)
		} else {
			log.Error("MaxShardSize or NofAssertionsPerShard must be specified to do sharding")
			return
		}
	}
	if config.AddSignatureMetaData {
		//addSignatureMetaData()
	}
	if config.DoConsistencyCheck {
		//consistencyCheck()
	}
	if config.SortShards {
		//sort shards
	}
	if config.DoSigning {
		if err := signSections(sections); err != nil {
			return
		}
	}
	if config.OutputPath != "" {
		if err := writeZonefile(config.OutputPath, sections); err != nil {
			log.Error("Was not able to write zonefile to disk", "path", config.OutputPath, "error", err)
		}
	}
	if config.DoPublish {
		encoding, err := createRainsMessage(sections)
		if err != nil {
			return
		}
		unreachableServers := publishSections(encoding)
		if unreachableServers != nil {
			log.Warn("Was not able to connect to all authoritative servers", "unreachableServers", unreachableServers)
		}
	}
}

func signSections(sections []rainslib.MessageSectionWithSigForward) error {
	keys, err := loadPrivateKeys()
	if err != nil {
		return err
	}
	signingkeyID := rainslib.PublicKeyID{
		Algorithm: config.SignatureAlgorithm,
		KeySpace:  rainslib.RainsKeySpace,
		KeyPhase:  config.KeyPhase,
	}
	if config.SignatureAlgorithm != rainslib.Ed25519 {
		log.Error("Not supported signature algorithm type")
		return fmt.Errorf("Not supported signature algorithm type")
	}
	if key := keys[signingkeyID]; key != nil {
		for _, section := range sections {
			err := signSectionUnsafe(section, key.(ed25519.PrivateKey))
			if err != nil {
				log.Error("Was not able to sign section", "section", section, "error", err)
			}
		}
	} else {
		log.Error("no private key found for provided algo and phase", "algo",
			config.SignatureAlgorithm, "phase", config.KeyPhase, "keymap", keys)
		return fmt.Errorf("no private key found for provided algo and phase")
	}
	return nil
}

//groupAssertionsToShards creates shards containing a maximum number of different assertion names
//according to the configuration. Before grouping the assertions, it sorts them. It returns a zone
//section containing the created shards. The contained shards and assertions still have non empty
//subjectZone and context values as these values are needed to generate a signatures
func groupAssertionsToShards(subjectZone, context string, assertions []*rainslib.AssertionSection, nofAssertionsPerShard int) *rainslib.ZoneSection {
	//the assertion compareTo function sorts first by subjectName. Thus we can use it here.
	sort.Slice(assertions, func(i, j int) bool { return assertions[i].CompareTo(assertions[j]) < 0 })
	shards := []rainslib.MessageSectionWithSigForward{}
	nameCount := 0
	prevAssertionSubjectName := ""
	prevShardAssertionSubjectName := ""
	shard := newShard(subjectZone, context)
	for i, a := range assertions {
		if a.SubjectZone != subjectZone || a.Context != context {
			//log.Error("assertion's subjectZone or context does not match with the zone's", "assertion", a)
		}
		if prevAssertionSubjectName != a.SubjectName {
			nameCount++
			prevAssertionSubjectName = a.SubjectName
		}
		if nameCount > nofAssertionsPerShard {
			shard.RangeFrom = prevShardAssertionSubjectName
			shard.RangeTo = a.SubjectName
			shards = append(shards, shard)
			nameCount = 1
			shard = newShard(subjectZone, context)
			prevShardAssertionSubjectName = assertions[i-1].SubjectName
		}
		shard.Content = append(shard.Content, a)
	}
	shard.RangeFrom = prevShardAssertionSubjectName
	shard.RangeTo = ""
	shards = append(shards, shard)

	section := &rainslib.ZoneSection{
		Context:     context,
		SubjectZone: subjectZone,
		Content:     shards,
	}
	return section
}

func newShard(subjectZone, context string) *rainslib.ShardSection {
	return &rainslib.ShardSection{
		SubjectZone: subjectZone,
		Context:     context,
		Content:     []*rainslib.AssertionSection{},
	}
}

//publishZone performs the following steps:
//1) Loads the rains zone file.
//2) Adds Signature MetaData and perform consistency checks on the zone and its
//   signatures
//3) Let rainspub sign the zone
//4) Query the superordinate zone for the new delegation and push it to all
//   rains servers
//5) After rainspub signed the zone, send the signed zone to all rains servers
//   specified in the config
//returns an error if something goes wrong
/*func publishZone(keyPhase int) error {

	//TODO CFE be able to add multiple signature to a section
	addSignatureMetaData(zone, keyPhase)
	if ConsistencyCheck(zone) {
		return errors.New("Inconsistent section")
	}
	//TODO CFE do this in a go routine
	if err = SignSectionUnsafe(zone, keyPhaseToPath); err != nil {
		return err
	}
	//TODO CFE: query new delegation from superordinate server and push them to all rains servers
	msg, err := CreateRainsMessage(zone)
	if err != nil {
		log.Warn("Was not able to parse the zone to a rains message.", "error", err)
		return err
	}
	connErrors := PublishSections(msg, config.ServerAddresses)
	for _, connErr := range connErrors {
		log.Warn("Was not able to send signed zone to this server.", "server", connErr.TCPAddr.String())
		//TODO CFE: Implement error handling
	}
	return nil
}
*/
//TODO CFE change it such that it can be used as envisioned in the
//design-scalable-signature-updates.md
//especially that not all assertions are expiring at the same time
func addSignatureMetaData(zone *rainslib.ZoneSection, keyPhase int) {
	//TODO CFE consider from config, validUntil, validSince, duration
	signature := rainslib.Signature{
		PublicKeyID: rainslib.PublicKeyID{
			Algorithm: rainslib.Ed25519,
			KeySpace:  rainslib.RainsKeySpace,
			KeyPhase:  keyPhase,
		},
		ValidSince: time.Now().Unix(),
		ValidUntil: time.Now().Unix(),
	}
	zone.AddSig(signature)
	for _, sec := range zone.Content {
		switch sec := sec.(type) {
		case *rainslib.AssertionSection:
			if sec.Content[0].Type == rainslib.OTDelegation {
				signature.ValidSince = time.Now().Unix()
				signature.ValidUntil = time.Now().Unix()
			} else {
				signature.ValidSince = time.Now().Unix()
				signature.ValidUntil = time.Now().Unix()
			}
		case *rainslib.ShardSection:
			signature.ValidSince = time.Now().Unix()
			signature.ValidUntil = time.Now().Unix()
		default:
			log.Error("Invalid zone content")
		}
		sec.AddSig(signature)
	}
}

//consistencyCheck returns true if there are no inconsistencies in the section. It
//also makes sure that the section is sorted
func consistencyCheck(section rainslib.MessageSectionWithSig) bool {
	//TODO consider config.SigNotExpired and config.checkStringFields
	switch section := section.(type) {
	case *rainslib.AssertionSection:
		return rainsSiglib.ValidSectionAndSignature(section)
	case *rainslib.ShardSection:
		return shardConsistencyCheck(section)
	case *rainslib.ZoneSection:
		if !rainsSiglib.ValidSectionAndSignature(section) {
			return false
		}
		for _, sec := range section.Content {
			switch sec := sec.(type) {
			case *rainslib.AssertionSection:
				if !rainsSiglib.ValidSectionAndSignature(sec) {
					return false
				}
			case *rainslib.ShardSection:
				if !shardConsistencyCheck(sec) {
					return false
				}
			default:
				log.Error("Invalid zone content", "zone", section)
				return false
			}
		}
	case *rainslib.AddressAssertionSection:
		log.Warn("Not yet implemented")
		return false
	default:
		log.Error("Invalid section type")
		return false
	}
	return true
}

//shardConsistencyCheck returns true if the shard and all contained
//assertions are consistent and sorted
func shardConsistencyCheck(shard *rainslib.ShardSection) bool {
	if !rainsSiglib.ValidSectionAndSignature(shard) {
		return false
	}
	for _, a := range shard.Content {
		if !rainsSiglib.ValidSectionAndSignature(a) {
			return false
		}
	}
	return true
}

//createRainsMessage creates a rainsMessage containing the given zone and
//returns the byte representation of this rainsMessage ready to send out.
func createRainsMessage(sections []rainslib.MessageSectionWithSigForward) ([]byte, error) {
	msg := rainslib.RainsMessage{Token: rainslib.GenerateToken()} //no capabilities
	for _, section := range sections {
		msg.Content = append(msg.Content, section)
	}
	//FIXME CFE use CBOR
	msgParser := new(protoParser.ProtoParserAndFramer)
	byteMsg, err := msgParser.Encode(msg)
	if err != nil {
		return []byte{}, err
	}
	return byteMsg, nil
}

//publishSections establishes connections to all authoritative servers according to the config. It
//then sends sections to all of them. It returns the connection information of those servers it was
//not able to push sections, otherwise nil is returned.
func publishSections(sections []byte) []rainslib.ConnInfo {
	var errorConns []rainslib.ConnInfo
	results := make(chan *rainslib.ConnInfo, len(config.AuthServers))
	for _, conn := range config.AuthServers {
		go connectAndSendMsg(sections, conn, results)
	}
	for i := 0; i < len(config.AuthServers); i++ {
		if errorConn := <-results; errorConn != nil {
			errorConns = append(errorConns, *errorConn)
		}
	}
	return errorConns
}
<commit_msg>sharding output as zone. Added TODO's<commit_after>package rainspub

import (
	"fmt"
	"sort"
	"time"

	"golang.org/x/crypto/ed25519"

	log "github.com/inconshreveable/log15"
	"github.com/netsec-ethz/rains/rainsSiglib"
	"github.com/netsec-ethz/rains/rainslib"
	"github.com/netsec-ethz/rains/utils/protoParser"
	"github.com/netsec-ethz/rains/utils/zoneFileParser"
)

//Init starts the zone information publishing process according to the provided config.
func Init(inputConfig Config) {
	config = inputConfig
	parser = zoneFileParser.Parser{}
	signatureEncoder = zoneFileParser.Parser{}
	publish()
}

//publish calls the relevant library function to publish information according to the provided
//config during initialization.
//FIXME CFE this implementation assumes that there is exactly one zone per zonefile.
func publish() {
	sections, err := loadZonefile()
	if err != nil {
		return
	}
	if config.DoSharding {
		var assertions []*rainslib.AssertionSection
		var shards []rainslib.MessageSectionWithSigForward
		zone, context := "", ""
		for _, section := range sections {
			switch sec := section.(type) {
			case *rainslib.ZoneSection:
				zone = sec.SubjectZone
				context = sec.Context
			case *rainslib.ShardSection:
				if config.KeepExistingShards {
					shards = append(shards, sec)
				}
			case *rainslib.AssertionSection:
				assertions = append(assertions, sec)
			}
		}
		if config.MaxShardSize > 0 {
			//TODO CFE to implement
		} else if config.NofAssertionsPerShard > 0 {
			shards = append(shards, groupAssertionsToShards(zone, context, assertions, config.NofAssertionsPerShard)...)
		} else {
			log.Error("MaxShardSize or NofAssertionsPerShard must be specified to do sharding")
			return
		}
		//TODO CFE check if sections are small enough to fit into a zone
		sections = []rainslib.MessageSectionWithSigForward{
			&rainslib.ZoneSection{
				SubjectZone: zone,
				Context:     context,
				Content:     shards,
			},
		}
	}
	if config.AddSignatureMetaData {
		//TODO CFE where to add signature meta data and spreading it uniformly over given interval.
		//addSignatureMetaData()
	}
	if config.DoConsistencyCheck {
		//consistencyCheck()
	}
	//TODO CFE add other two consistency checks
	if config.SortShards {
		//sort shards
	}
	if config.DoSigning {
		if err := signSections(sections); err != nil {
			return
		}
	}
	if config.OutputPath != "" {
		if err := writeZonefile(config.OutputPath, sections); err != nil {
			log.Error("Was not able to write zonefile to disk", "path", config.OutputPath, "error", err)
		}
	}
	if config.DoPublish {
		encoding, err := createRainsMessage(sections)
		if err != nil {
			return
		}
		unreachableServers := publishSections(encoding)
		if unreachableServers != nil {
			log.Warn("Was not able to connect to all authoritative servers", "unreachableServers", unreachableServers)
		}
	}
}

func signSections(sections []rainslib.MessageSectionWithSigForward) error {
	keys, err := loadPrivateKeys()
	if err != nil {
		return err
	}
	signingkeyID := rainslib.PublicKeyID{
		Algorithm: config.SignatureAlgorithm,
		KeySpace:  rainslib.RainsKeySpace,
		KeyPhase:  config.KeyPhase,
	}
	if config.SignatureAlgorithm != rainslib.Ed25519 {
		log.Error("Not supported signature algorithm type")
		return fmt.Errorf("Not supported signature algorithm type")
	}
	if key := keys[signingkeyID]; key != nil {
		for _, section := range sections {
			err := signSectionUnsafe(section, key.(ed25519.PrivateKey))
			if err != nil {
				log.Error("Was not able to sign section", "section", section, "error", err)
			}
		}
	} else {
		log.Error("no private key found for provided algo and phase", "algo",
			config.SignatureAlgorithm, "phase", config.KeyPhase, "keymap", keys)
		return fmt.Errorf("no private key found for provided algo and phase")
	}
	return nil
}

//groupAssertionsToShards creates shards containing a maximum number of different assertion names
//according to the configuration. Before grouping the assertions, it sorts them. It returns a zone
//section containing the created shards. The contained shards and assertions still have non empty
//subjectZone and context values as these values are needed to generate a signatures
func groupAssertionsToShards(subjectZone, context string, assertions []*rainslib.AssertionSection, nofAssertionsPerShard int) []rainslib.MessageSectionWithSigForward {
	//the assertion compareTo function sorts first by subjectName. Thus we can use it here.
	sort.Slice(assertions, func(i, j int) bool { return assertions[i].CompareTo(assertions[j]) < 0 })
	shards := []rainslib.MessageSectionWithSigForward{}
	nameCount := 0
	prevAssertionSubjectName := ""
	prevShardAssertionSubjectName := ""
	shard := newShard(subjectZone, context)
	for i, a := range assertions {
		if a.SubjectZone != subjectZone || a.Context != context {
			//log.Error("assertion's subjectZone or context does not match with the zone's", "assertion", a)
		}
		if prevAssertionSubjectName != a.SubjectName {
			nameCount++
			prevAssertionSubjectName = a.SubjectName
		}
		if nameCount > nofAssertionsPerShard {
			shard.RangeFrom = prevShardAssertionSubjectName
			shard.RangeTo = a.SubjectName
			shards = append(shards, shard)
			nameCount = 1
			shard = newShard(subjectZone, context)
			prevShardAssertionSubjectName = assertions[i-1].SubjectName
		}
		shard.Content = append(shard.Content, a)
	}
	shard.RangeFrom = prevShardAssertionSubjectName
	shard.RangeTo = ""
	shards = append(shards, shard)

	return shards
}

func newShard(subjectZone, context string) *rainslib.ShardSection {
	return &rainslib.ShardSection{
		SubjectZone: subjectZone,
		Context:     context,
		Content:     []*rainslib.AssertionSection{},
	}
}

//publishZone performs the following steps:
//1) Loads the rains zone file.
//2) Adds Signature MetaData and perform consistency checks on the zone and its
//   signatures
//3) Let rainspub sign the zone
//4) Query the superordinate zone for the new delegation and push it to all
//   rains servers
//5) After rainspub signed the zone, send the signed zone to all rains servers
//   specified in the config
//returns an error if something goes wrong
/*func publishZone(keyPhase int) error {

	//TODO CFE be able to add multiple signature to a section
	addSignatureMetaData(zone, keyPhase)
	if ConsistencyCheck(zone) {
		return errors.New("Inconsistent section")
	}
	//TODO CFE do this in a go routine
	if err = SignSectionUnsafe(zone, keyPhaseToPath); err != nil {
		return err
	}
	//TODO CFE: query new delegation from superordinate server and push them to all rains servers
	msg, err := CreateRainsMessage(zone)
	if err != nil {
		log.Warn("Was not able to parse the zone to a rains message.", "error", err)
		return err
	}
	connErrors := PublishSections(msg, config.ServerAddresses)
	for _, connErr := range connErrors {
		log.Warn("Was not able to send signed zone to this server.", "server", connErr.TCPAddr.String())
		//TODO CFE: Implement error handling
	}
	return nil
}
*/
//TODO CFE change it such that it can be used as envisioned in the
//design-scalable-signature-updates.md
//especially that not all assertions are expiring at the same time
func addSignatureMetaData(zone *rainslib.ZoneSection, keyPhase int) {
	//TODO CFE consider from config, validUntil, validSince, duration
	signature := rainslib.Signature{
		PublicKeyID: rainslib.PublicKeyID{
			Algorithm: rainslib.Ed25519,
			KeySpace:  rainslib.RainsKeySpace,
			KeyPhase:  keyPhase,
		},
		ValidSince: time.Now().Unix(),
		ValidUntil: time.Now().Unix(),
	}
	zone.AddSig(signature)
	for _, sec := range zone.Content {
		switch sec := sec.(type) {
		case *rainslib.AssertionSection:
			if sec.Content[0].Type == rainslib.OTDelegation {
				signature.ValidSince = time.Now().Unix()
				signature.ValidUntil = time.Now().Unix()
			} else {
				signature.ValidSince = time.Now().Unix()
				signature.ValidUntil = time.Now().Unix()
			}
		case *rainslib.ShardSection:
			signature.ValidSince = time.Now().Unix()
			signature.ValidUntil = time.Now().Unix()
		default:
			log.Error("Invalid zone content")
		}
		sec.AddSig(signature)
	}
}

//consistencyCheck returns true if there are no inconsistencies in the section. It
//also makes sure that the section is sorted
func consistencyCheck(section rainslib.MessageSectionWithSig) bool {
	//TODO consider config.SigNotExpired and config.checkStringFields
	switch section := section.(type) {
	case *rainslib.AssertionSection:
		return rainsSiglib.ValidSectionAndSignature(section)
	case *rainslib.ShardSection:
		return shardConsistencyCheck(section)
	case *rainslib.ZoneSection:
		if !rainsSiglib.ValidSectionAndSignature(section) {
			return false
		}
		for _, sec := range section.Content {
			switch sec := sec.(type) {
			case *rainslib.AssertionSection:
				if !rainsSiglib.ValidSectionAndSignature(sec) {
					return false
				}
			case *rainslib.ShardSection:
				if !shardConsistencyCheck(sec) {
					return false
				}
			default:
				log.Error("Invalid zone content", "zone", section)
				return false
			}
		}
	case *rainslib.AddressAssertionSection:
		log.Warn("Not yet implemented")
		return false
	default:
		log.Error("Invalid section type")
		return false
	}
	return true
}

//shardConsistencyCheck returns true if the shard and all contained
//assertions are consistent and sorted
func shardConsistencyCheck(shard *rainslib.ShardSection) bool {
	if !rainsSiglib.ValidSectionAndSignature(shard) {
		return false
	}
	for _, a := range shard.Content {
		if !rainsSiglib.ValidSectionAndSignature(a) {
			return false
		}
	}
	return true
}

//createRainsMessage creates a rainsMessage containing the given zone and
//returns the byte representation of this rainsMessage ready to send out.
func createRainsMessage(sections []rainslib.MessageSectionWithSigForward) ([]byte, error) {
	msg := rainslib.RainsMessage{Token: rainslib.GenerateToken()} //no capabilities
	for _, section := range sections {
		msg.Content = append(msg.Content, section)
	}
	//FIXME CFE use CBOR
	msgParser := new(protoParser.ProtoParserAndFramer)
	byteMsg, err := msgParser.Encode(msg)
	if err != nil {
		return []byte{}, err
	}
	return byteMsg, nil
}

//publishSections establishes connections to all authoritative servers according to the config. It
//then sends sections to all of them. It returns the connection information of those servers it was
//not able to push sections, otherwise nil is returned.
func publishSections(sections []byte) []rainslib.ConnInfo {
	var errorConns []rainslib.ConnInfo
	results := make(chan *rainslib.ConnInfo, len(config.AuthServers))
	for _, conn := range config.AuthServers {
		go connectAndSendMsg(sections, conn, results)
	}
	for i := 0; i < len(config.AuthServers); i++ {
		if errorConn := <-results; errorConn != nil {
			errorConns = append(errorConns, *errorConn)
		}
	}
	return errorConns
}
<|endoftext|>