<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"fmt"
	"sync"
	"time"

	"github.com/keybase/client/go/logger"
	"golang.org/x/net/context"
)

type dirtyBlockID struct {
	id       BlockID
	refNonce BlockRefNonce
	branch   BranchName
}

type dirtyReq struct {
	respChan chan<- struct{}
	bytes    int64
	start    time.Time
	deadline time.Time
}

// DirtyBlockCacheStandard implements the DirtyBlockCache interface by
// storing blocks in an in-memory cache.  Dirty blocks are identified
// by their block ID, branch name, and reference nonce, since the same
// block may be forked and modified on different branches and under
// different references simultaneously.
//
// DirtyBlockCacheStandard controls how fast uses can write into KBFS,
// and does so with a TCP-like slow-start algorithm that adjusts
// itself according to how fast bytes are synced to the server.  It
// keeps track of a "sync buffer" that holds all of the bytes that
// have been written but not yet synced to the server.  When the
// buffer is full, write requests get blocked by a fraction of the
// write timeout, according to how close the user is to filling up
// their "dirty buffer" -- i.e., the total number of bytes for which
// an overall Sync operation has not yet completed (even if their
// individual block Puts to the server have completed).
//
// The size of the sync buffer can vary between a minimum and maximum.
// It starts out at the minimum, and grows exponentially as Puts
// succeed.  However, if a write request is blocked for too long by
// the backpressure and buffer-fullness checks, the sync buffer size
// is cut in half.
type DirtyBlockCacheStandard struct {
	clock   Clock
	makeLog func(string) logger.Logger
	log     logger.Logger

	// requestsChan is a queue for channels that should be closed when
	// permission is granted to dirty new data.
	requestsChan chan dirtyReq
	// bytesDecreasedChan is signalled when syncs have finished or dirty
	// blocks have been deleted.
	bytesDecreasedChan chan struct{}
	// shutdownChan is closed when Shutdown is called.
	shutdownChan chan struct{}
	// blockedChanForTesting sends out the number of bytes of the
	// request currently waiting.  Sends out -1 when the request is
	// accepted. Used only for testing.
	blockedChanForTesting chan<- int64

	// The minimum (and initial) size of the sync buffer.
	minSyncBufferSize int64
	// The maximum size of the sync buffer.  Also used as the
	// denominator when calculating backpressure, such that the closer
	// we are to reaching the maximum size (over and above the current
	// sync buffer), the more write requests will be delayed.
	maxSyncBufferSize int64

	lock               sync.RWMutex
	cache              map[dirtyBlockID]Block
	unsyncedDirtyBytes int64
	syncingDirtyBytes  int64 // just for bookkeeping, not actually used
	totalDirtyBytes    int64
	syncBufferSize     int64
}

// NewDirtyBlockCacheStandard constructs a new BlockCacheStandard
// instance.  makeLog is a function that will be called later to make
// a log (we don't take an actual log, to allow the DirtyBlockCache to
// be created before logging is initialized).  The min and max buffer
// sizes define the possible range of how many bytes we'll try to sync
// in any one sync.
func NewDirtyBlockCacheStandard(clock Clock,
	makeLog func(string) logger.Logger, minSyncBufferSize int64,
	maxSyncBufferSize int64) *DirtyBlockCacheStandard {
	d := &DirtyBlockCacheStandard{
		clock:              clock,
		makeLog:            makeLog,
		requestsChan:       make(chan dirtyReq, 1000),
		bytesDecreasedChan: make(chan struct{}, 1),
		shutdownChan:       make(chan struct{}),
		cache:              make(map[dirtyBlockID]Block),
		minSyncBufferSize:  minSyncBufferSize,
		maxSyncBufferSize:  maxSyncBufferSize,
		syncBufferSize:     minSyncBufferSize,
	}
	go d.processPermission()
	return d
}

// Get implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Get(ptr BlockPointer, branch BranchName) (
	Block, error) {
	block := func() Block {
		dirtyID := dirtyBlockID{
			id:       ptr.ID,
			refNonce: ptr.RefNonce,
			branch:   branch,
		}
		d.lock.RLock()
		defer d.lock.RUnlock()
		return d.cache[dirtyID]
	}()
	if block != nil {
		return block, nil
	}

	return nil, NoSuchBlockError{ptr.ID}
}

// Put implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Put(ptr BlockPointer, branch BranchName,
	block Block) error {
	dirtyID := dirtyBlockID{
		id:       ptr.ID,
		refNonce: ptr.RefNonce,
		branch:   branch,
	}

	d.lock.Lock()
	defer d.lock.Unlock()
	d.cache[dirtyID] = block
	return nil
}

// Delete implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Delete(ptr BlockPointer,
	branch BranchName) error {
	dirtyID := dirtyBlockID{
		id:       ptr.ID,
		refNonce: ptr.RefNonce,
		branch:   branch,
	}

	d.lock.Lock()
	defer d.lock.Unlock()
	delete(d.cache, dirtyID)
	return nil
}

// IsDirty implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) IsDirty(
	ptr BlockPointer, branch BranchName) (isDirty bool) {
	dirtyID := dirtyBlockID{
		id:       ptr.ID,
		refNonce: ptr.RefNonce,
		branch:   branch,
	}

	d.lock.RLock()
	defer d.lock.RUnlock()
	_, isDirty = d.cache[dirtyID]
	return
}

const backpressureSlack = 1 * time.Second

// calcBackpressure returns how much longer a given request should be
// blocked, as a function of its deadline and how past full the
// syncing buffer is.  In its lifetime, the request should be blocked
// by roughly the same fraction of its total deadline as how past full
// the buffer is.  This will let KBFS slow down writes according to
// how slow the background Syncs are, so we don't accumulate more
// bytes to Sync than we can handle.  See KBFS-731.
func (d *DirtyBlockCacheStandard) calcBackpressure(start time.Time,
	deadline time.Time) time.Duration {
	d.lock.RLock()
	defer d.lock.RUnlock()
	// We don't want to use the whole deadline, so cut it some slack.
	totalReqTime := deadline.Sub(start) - backpressureSlack
	if totalReqTime <= 0 {
		return 0
	}

	// Keep the window full in preparation for the next sync, after
	// it's full start applying backpressure.
	if d.unsyncedDirtyBytes < d.syncBufferSize {
		return 0
	}

	// The backpressure is proportional to how far our overage is
	// towards the max sync buffer size.
	backpressureFrac := float64(d.unsyncedDirtyBytes-d.syncBufferSize) /
		float64(d.maxSyncBufferSize-d.syncBufferSize)
	if backpressureFrac > 1.0 {
		backpressureFrac = 1.0
	}
	totalBackpressure := time.Duration(
		float64(totalReqTime) * backpressureFrac)

	// How much time do we have left, given how much time this request
	// has waited so far?
	backpressureLeft := totalBackpressure - d.clock.Now().Sub(start)
	if backpressureLeft < 0 {
		return 0
	}
	return backpressureLeft
}

func (d *DirtyBlockCacheStandard) logLocked(fmt string, arg ...interface{}) {
	if d.log == nil {
		log := d.makeLog("")
		if log != nil {
			d.log = log.CloneWithAddedDepth(1)
		}
	}
	if d.log != nil {
		// TODO: pass contexts all the way here just for logging? It's
		// extremely inconvenient to do that for the permission check
		// messages which happen in the background.
		d.log.CDebugf(nil, fmt, arg...)
	}
}

func (d *DirtyBlockCacheStandard) acceptNewWrite(newBytes int64,
	start time.Time, deadline time.Time) bool {
	d.lock.Lock()
	defer d.lock.Unlock()
	// Accept any write, as long as we're not already over the limits.
	canAccept := d.totalDirtyBytes < d.syncBufferSize
	if canAccept {
		d.unsyncedDirtyBytes += newBytes
		d.totalDirtyBytes += newBytes

		// Update syncBufferSize if the write has been blocked for more than
		// half of its deadline.
		allowedDeadline := float64(deadline.Sub(start))
		deadlineUsed := d.clock.Now().Sub(start)
		fracDeadlineUsed := float64(deadlineUsed) / allowedDeadline
		if fracDeadlineUsed > 0.5 {
			d.syncBufferSize /= 2
			if d.syncBufferSize < d.minSyncBufferSize {
				d.syncBufferSize = d.minSyncBufferSize
			}
			d.logLocked("Write blocked for %s (%f%% of deadline), "+
				"syncBufferSize=%d", deadlineUsed, fracDeadlineUsed*100,
				d.syncBufferSize)
		}
	}
	return canAccept
}

func (d *DirtyBlockCacheStandard) processPermission() {
	// Keep track of the most-recently seen request across loop
	// iterations, because we aren't necessarily going to be able to
	// deal with it as soon as we see it (since we might be past our
	// limits already).
	var currentReq dirtyReq
	var backpressure time.Duration
	for {
		reqChan := d.requestsChan
		if currentReq.respChan != nil {
			// We are already waiting on a request, so don't bother
			// trying to read another request from the requests chan.
			reqChan = nil
		}

		var bpTimer <-chan time.Time
		if backpressure > 0 {
			bpTimer = time.After(backpressure)
		}

		newReq := false
		select {
		case <-d.shutdownChan:
			return
		case <-d.bytesDecreasedChan:
		case <-bpTimer:
		case r := <-reqChan:
			currentReq = r
			newReq = true
		}

		if currentReq.respChan != nil {
			// Apply any backpressure?
			backpressure = d.calcBackpressure(currentReq.start,
				currentReq.deadline)
			if backpressure == 0 && d.acceptNewWrite(currentReq.bytes,
				currentReq.start, currentReq.deadline) {
				// If we have an active request, and we have room in
				// our buffers to deal with it, grant permission to
				// the requestor by closing the response channel.
				close(currentReq.respChan)
				currentReq = dirtyReq{}
				if d.blockedChanForTesting != nil {
					d.blockedChanForTesting <- -1
				}
			} else if d.blockedChanForTesting != nil && newReq {
				// Otherwise, if this is the first time we've
				// considered this request, inform any tests that the
				// request is blocked.
				d.blockedChanForTesting <- currentReq.bytes
			}
		}
	}
}

// RequestPermissionToDirty implements the DirtyBlockCache interface
// for DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) RequestPermissionToDirty(
	ctx context.Context, estimatedDirtyBytes int64) (DirtyPermChan, error) {
	if estimatedDirtyBytes < 0 {
		panic("Must request permission for a non-negative number of bytes.")
	}
	c := make(chan struct{})
	deadline, ok := ctx.Deadline()
	now := d.clock.Now()
	if !ok {
		deadline = now.Add(backgroundTaskTimeout)
	}
	req := dirtyReq{c, estimatedDirtyBytes, now, deadline}
	select {
	case d.requestsChan <- req:
		return c, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (d *DirtyBlockCacheStandard) signalDecreasedBytes() {
	select {
	case d.bytesDecreasedChan <- struct{}{}:
	default:
		// Already something queued there, and one is enough.
	}
}

// UpdateUnsyncedBytes implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) UpdateUnsyncedBytes(newUnsyncedBytes int64,
	wasSyncing bool) {
	d.lock.Lock()
	defer d.lock.Unlock()
	d.unsyncedDirtyBytes += newUnsyncedBytes
	d.totalDirtyBytes += newUnsyncedBytes
	if wasSyncing {
		d.syncingDirtyBytes += newUnsyncedBytes
	}
	if d.unsyncedDirtyBytes < 0 {
		d.signalDecreasedBytes()
	}
}

// UpdateSyncingBytes implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) UpdateSyncingBytes(size int64) {
	d.lock.Lock()
	defer d.lock.Unlock()
	d.syncingDirtyBytes += size
}

// BlockSyncFinished implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) BlockSyncFinished(size int64) {
	d.lock.Lock()
	defer d.lock.Unlock()
	d.unsyncedDirtyBytes -= size
	if size > 0 {
		d.syncingDirtyBytes -= size
		d.signalDecreasedBytes()
	}
}

// SyncFinished implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) SyncFinished(size int64) {
	d.lock.Lock()
	defer d.lock.Unlock()
	if size <= 0 {
		return
	}
	d.totalDirtyBytes -= size
	// Only increase the buffer size if we sent over a lot of bytes.
	// We don't want a series of small writes to increase the buffer
	// size, since that doesn't give us any real information about the
	// throughput of the connection.
	if size >= d.minSyncBufferSize {
		d.syncBufferSize += size
		if d.syncBufferSize > d.maxSyncBufferSize {
			d.syncBufferSize = d.maxSyncBufferSize
		}
	}
	d.signalDecreasedBytes()
	d.logLocked("Finished syncing %d bytes, syncBufferSize=%d, "+
		"totalDirty=%d", size, d.syncBufferSize, d.totalDirtyBytes)
}

// ShouldForceSync implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) ShouldForceSync() bool {
	d.lock.Lock()
	defer d.lock.Unlock()
	return d.unsyncedDirtyBytes > d.syncBufferSize
}

// Shutdown implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Shutdown() error {
	d.lock.Lock()
	defer d.lock.Unlock()
	close(d.shutdownChan)
	if d.unsyncedDirtyBytes != 0 || d.totalDirtyBytes != 0 ||
		d.syncingDirtyBytes != 0 {
		return fmt.Errorf("Unexpected dirty bytes leftover on shutdown: "+
			"unsynced=%d, syncing=%d, total=%d", d.unsyncedDirtyBytes,
			d.syncingDirtyBytes, d.totalDirtyBytes)
	}
	return nil
}
<commit_msg>dirty_bcache: avoid negative backpressures and add log<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"fmt"
	"sync"
	"time"

	"github.com/keybase/client/go/logger"
	"golang.org/x/net/context"
)

type dirtyBlockID struct {
	id       BlockID
	refNonce BlockRefNonce
	branch   BranchName
}

type dirtyReq struct {
	respChan chan<- struct{}
	bytes    int64
	start    time.Time
	deadline time.Time
}

// DirtyBlockCacheStandard implements the DirtyBlockCache interface by
// storing blocks in an in-memory cache.  Dirty blocks are identified
// by their block ID, branch name, and reference nonce, since the same
// block may be forked and modified on different branches and under
// different references simultaneously.
//
// DirtyBlockCacheStandard controls how fast uses can write into KBFS,
// and does so with a TCP-like slow-start algorithm that adjusts
// itself according to how fast bytes are synced to the server.  It
// keeps track of a "sync buffer" that holds all of the bytes that
// have been written but not yet synced to the server.  When the
// buffer is full, write requests get blocked by a fraction of the
// write timeout, according to how close the user is to filling up
// their "dirty buffer" -- i.e., the total number of bytes for which
// an overall Sync operation has not yet completed (even if their
// individual block Puts to the server have completed).
//
// The size of the sync buffer can vary between a minimum and maximum.
// It starts out at the minimum, and grows exponentially as Puts
// succeed.  However, if a write request is blocked for too long by
// the backpressure and buffer-fullness checks, the sync buffer size
// is cut in half.
type DirtyBlockCacheStandard struct {
	clock   Clock
	makeLog func(string) logger.Logger
	log     logger.Logger

	// requestsChan is a queue for channels that should be closed when
	// permission is granted to dirty new data.
	requestsChan chan dirtyReq
	// bytesDecreasedChan is signalled when syncs have finished or dirty
	// blocks have been deleted.
	bytesDecreasedChan chan struct{}
	// shutdownChan is closed when Shutdown is called.
	shutdownChan chan struct{}
	// blockedChanForTesting sends out the number of bytes of the
	// request currently waiting.  Sends out -1 when the request is
	// accepted. Used only for testing.
	blockedChanForTesting chan<- int64

	// The minimum (and initial) size of the sync buffer.
	minSyncBufferSize int64
	// The maximum size of the sync buffer.  Also used as the
	// denominator when calculating backpressure, such that the closer
	// we are to reaching the maximum size (over and above the current
	// sync buffer), the more write requests will be delayed.
	maxSyncBufferSize int64

	lock               sync.RWMutex
	cache              map[dirtyBlockID]Block
	unsyncedDirtyBytes int64
	syncingDirtyBytes  int64 // just for bookkeeping, not actually used
	totalDirtyBytes    int64
	syncBufferSize     int64
}

// NewDirtyBlockCacheStandard constructs a new BlockCacheStandard
// instance.  makeLog is a function that will be called later to make
// a log (we don't take an actual log, to allow the DirtyBlockCache to
// be created before logging is initialized).  The min and max buffer
// sizes define the possible range of how many bytes we'll try to sync
// in any one sync.
func NewDirtyBlockCacheStandard(clock Clock,
	makeLog func(string) logger.Logger, minSyncBufferSize int64,
	maxSyncBufferSize int64) *DirtyBlockCacheStandard {
	d := &DirtyBlockCacheStandard{
		clock:              clock,
		makeLog:            makeLog,
		requestsChan:       make(chan dirtyReq, 1000),
		bytesDecreasedChan: make(chan struct{}, 1),
		shutdownChan:       make(chan struct{}),
		cache:              make(map[dirtyBlockID]Block),
		minSyncBufferSize:  minSyncBufferSize,
		maxSyncBufferSize:  maxSyncBufferSize,
		syncBufferSize:     minSyncBufferSize,
	}
	go d.processPermission()
	return d
}

// Get implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Get(ptr BlockPointer, branch BranchName) (
	Block, error) {
	block := func() Block {
		dirtyID := dirtyBlockID{
			id:       ptr.ID,
			refNonce: ptr.RefNonce,
			branch:   branch,
		}
		d.lock.RLock()
		defer d.lock.RUnlock()
		return d.cache[dirtyID]
	}()
	if block != nil {
		return block, nil
	}

	return nil, NoSuchBlockError{ptr.ID}
}

// Put implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Put(ptr BlockPointer, branch BranchName,
	block Block) error {
	dirtyID := dirtyBlockID{
		id:       ptr.ID,
		refNonce: ptr.RefNonce,
		branch:   branch,
	}

	d.lock.Lock()
	defer d.lock.Unlock()
	d.cache[dirtyID] = block
	return nil
}

// Delete implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Delete(ptr BlockPointer,
	branch BranchName) error {
	dirtyID := dirtyBlockID{
		id:       ptr.ID,
		refNonce: ptr.RefNonce,
		branch:   branch,
	}

	d.lock.Lock()
	defer d.lock.Unlock()
	delete(d.cache, dirtyID)
	return nil
}

// IsDirty implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) IsDirty(
	ptr BlockPointer, branch BranchName) (isDirty bool) {
	dirtyID := dirtyBlockID{
		id:       ptr.ID,
		refNonce: ptr.RefNonce,
		branch:   branch,
	}

	d.lock.RLock()
	defer d.lock.RUnlock()
	_, isDirty = d.cache[dirtyID]
	return
}

const backpressureSlack = 1 * time.Second

// calcBackpressure returns how much longer a given request should be
// blocked, as a function of its deadline and how past full the
// syncing buffer is.  In its lifetime, the request should be blocked
// by roughly the same fraction of its total deadline as how past full
// the buffer is.  This will let KBFS slow down writes according to
// how slow the background Syncs are, so we don't accumulate more
// bytes to Sync than we can handle.  See KBFS-731.
func (d *DirtyBlockCacheStandard) calcBackpressure(start time.Time,
	deadline time.Time) time.Duration {
	d.lock.RLock()
	defer d.lock.RUnlock()
	// We don't want to use the whole deadline, so cut it some slack.
	totalReqTime := deadline.Sub(start) - backpressureSlack
	if totalReqTime <= 0 {
		return 0
	}

	// Keep the window full in preparation for the next sync, after
	// it's full start applying backpressure.
	if d.unsyncedDirtyBytes < d.syncBufferSize {
		return 0
	}

	// The backpressure is proportional to how far our overage is
	// towards the max sync buffer size.
	backpressureFrac := float64(d.unsyncedDirtyBytes-d.syncBufferSize) /
		float64(d.maxSyncBufferSize-d.syncBufferSize)
	if backpressureFrac > 1.0 {
		backpressureFrac = 1.0
	}
	totalBackpressure := time.Duration(
		float64(totalReqTime) * backpressureFrac)
	timeSpentSoFar := d.clock.Now().Sub(start)
	if totalBackpressure <= timeSpentSoFar {
		return 0
	}

	// How much time do we have left, given how much time this request
	// has waited so far?
	return totalBackpressure - timeSpentSoFar
}

func (d *DirtyBlockCacheStandard) logLocked(fmt string, arg ...interface{}) {
	if d.log == nil {
		log := d.makeLog("")
		if log != nil {
			d.log = log.CloneWithAddedDepth(1)
		}
	}
	if d.log != nil {
		// TODO: pass contexts all the way here just for logging? It's
		// extremely inconvenient to do that for the permission check
		// messages which happen in the background.
		d.log.CDebugf(nil, fmt, arg...)
	}
}

func (d *DirtyBlockCacheStandard) acceptNewWrite(newBytes int64,
	start time.Time, deadline time.Time) bool {
	d.lock.Lock()
	defer d.lock.Unlock()
	// Accept any write, as long as we're not already over the limits.
	canAccept := d.totalDirtyBytes < d.syncBufferSize
	if canAccept {
		d.unsyncedDirtyBytes += newBytes
		d.totalDirtyBytes += newBytes

		// Update syncBufferSize if the write has been blocked for more than
		// half of its deadline.
		allowedDeadline := float64(deadline.Sub(start))
		deadlineUsed := d.clock.Now().Sub(start)
		fracDeadlineUsed := float64(deadlineUsed) / allowedDeadline
		if fracDeadlineUsed > 0.5 {
			d.syncBufferSize /= 2
			if d.syncBufferSize < d.minSyncBufferSize {
				d.syncBufferSize = d.minSyncBufferSize
			}
			d.logLocked("Write blocked for %s (%f%% of deadline), "+
				"syncBufferSize=%d", deadlineUsed, fracDeadlineUsed*100,
				d.syncBufferSize)
		}
	}
	return canAccept
}

func (d *DirtyBlockCacheStandard) processPermission() {
	// Keep track of the most-recently seen request across loop
	// iterations, because we aren't necessarily going to be able to
	// deal with it as soon as we see it (since we might be past our
	// limits already).
	var currentReq dirtyReq
	var backpressure time.Duration
	for {
		reqChan := d.requestsChan
		if currentReq.respChan != nil {
			// We are already waiting on a request, so don't bother
			// trying to read another request from the requests chan.
			reqChan = nil
		}

		var bpTimer <-chan time.Time
		if backpressure > 0 {
			bpTimer = time.After(backpressure)
		}

		newReq := false
		select {
		case <-d.shutdownChan:
			return
		case <-d.bytesDecreasedChan:
		case <-bpTimer:
		case r := <-reqChan:
			currentReq = r
			newReq = true
		}

		if currentReq.respChan != nil {
			// Apply any backpressure?
			backpressure = d.calcBackpressure(currentReq.start,
				currentReq.deadline)
			if backpressure == 0 && d.acceptNewWrite(currentReq.bytes,
				currentReq.start, currentReq.deadline) {
				// If we have an active request, and we have room in
				// our buffers to deal with it, grant permission to
				// the requestor by closing the response channel.
				close(currentReq.respChan)
				currentReq = dirtyReq{}
				if d.blockedChanForTesting != nil {
					d.blockedChanForTesting <- -1
				}
			} else if d.blockedChanForTesting != nil && newReq {
				// Otherwise, if this is the first time we've
				// considered this request, inform any tests that the
				// request is blocked.
				d.blockedChanForTesting <- currentReq.bytes
			} else if backpressure != 0 {
				d.logLocked("Applying backpressure %s", backpressure)
			}
		}
	}
}

// RequestPermissionToDirty implements the DirtyBlockCache interface
// for DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) RequestPermissionToDirty(
	ctx context.Context, estimatedDirtyBytes int64) (DirtyPermChan, error) {
	if estimatedDirtyBytes < 0 {
		panic("Must request permission for a non-negative number of bytes.")
	}
	c := make(chan struct{})
	deadline, ok := ctx.Deadline()
	now := d.clock.Now()
	if !ok {
		deadline = now.Add(backgroundTaskTimeout)
	}
	req := dirtyReq{c, estimatedDirtyBytes, now, deadline}
	select {
	case d.requestsChan <- req:
		return c, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (d *DirtyBlockCacheStandard) signalDecreasedBytes() {
	select {
	case d.bytesDecreasedChan <- struct{}{}:
	default:
		// Already something queued there, and one is enough.
	}
}

// UpdateUnsyncedBytes implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) UpdateUnsyncedBytes(newUnsyncedBytes int64,
	wasSyncing bool) {
	d.lock.Lock()
	defer d.lock.Unlock()
	d.unsyncedDirtyBytes += newUnsyncedBytes
	d.totalDirtyBytes += newUnsyncedBytes
	if wasSyncing {
		d.syncingDirtyBytes += newUnsyncedBytes
	}
	if d.unsyncedDirtyBytes < 0 {
		d.signalDecreasedBytes()
	}
}

// UpdateSyncingBytes implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) UpdateSyncingBytes(size int64) {
	d.lock.Lock()
	defer d.lock.Unlock()
	d.syncingDirtyBytes += size
}

// BlockSyncFinished implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) BlockSyncFinished(size int64) {
	d.lock.Lock()
	defer d.lock.Unlock()
	d.unsyncedDirtyBytes -= size
	if size > 0 {
		d.syncingDirtyBytes -= size
		d.signalDecreasedBytes()
	}
}

// SyncFinished implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) SyncFinished(size int64) {
	d.lock.Lock()
	defer d.lock.Unlock()
	if size <= 0 {
		return
	}
	d.totalDirtyBytes -= size
	// Only increase the buffer size if we sent over a lot of bytes.
	// We don't want a series of small writes to increase the buffer
	// size, since that doesn't give us any real information about the
	// throughput of the connection.
	if size >= d.minSyncBufferSize {
		d.syncBufferSize += size
		if d.syncBufferSize > d.maxSyncBufferSize {
			d.syncBufferSize = d.maxSyncBufferSize
		}
	}
	d.signalDecreasedBytes()
	d.logLocked("Finished syncing %d bytes, syncBufferSize=%d, "+
		"totalDirty=%d", size, d.syncBufferSize, d.totalDirtyBytes)
}

// ShouldForceSync implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) ShouldForceSync() bool {
	d.lock.Lock()
	defer d.lock.Unlock()
	return d.unsyncedDirtyBytes > d.syncBufferSize
}

// Shutdown implements the DirtyBlockCache interface for
// DirtyBlockCacheStandard.
func (d *DirtyBlockCacheStandard) Shutdown() error {
	d.lock.Lock()
	defer d.lock.Unlock()
	close(d.shutdownChan)
	if d.unsyncedDirtyBytes != 0 || d.totalDirtyBytes != 0 ||
		d.syncingDirtyBytes != 0 {
		return fmt.Errorf("Unexpected dirty bytes leftover on shutdown: "+
			"unsynced=%d, syncing=%d, total=%d", d.unsyncedDirtyBytes,
			d.syncingDirtyBytes, d.totalDirtyBytes)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"net"
	"strings"
)

func strcat(s1 string, s2 string) string {
	var buffer bytes.Buffer
	buffer.WriteString(s1)
	buffer.WriteString(s2)
	return buffer.String()
}

func GetUserByNick(nick string) *User {
	nick = strings.ToLower(nick)
	for _, k := range userlist {
		if strings.ToLower(k.nick) == nick {
			return k
		}
	}
	return nil
}

func GetIpFromConn(conn net.Conn) string {
	ip := conn.RemoteAddr().String()
	ip = strings.Split(ip, ":")[0]
	return ip
}

func GetChannelByName(name string) *Channel {
	for _, k := range chanlist {
		if strings.ToLower(name) == strings.ToLower(k.name) {
			return k
		}
	}
	return nil
}

func SendToMany(msg string, list []*User) {
	list2 := []*User{}
	for _, j := range list {
		match := false
		for _, k := range list2 {
			if j == k {
				match = true
				break
			}
		}
		if !match {
			list2 = append(list2, j)
		}
	}
	for _, j := range list2 {
		j.SendLine(msg)
	}
}

//SendToMany2f sends to all users in list ([]*User), does not check for dupes, accepts in printf format.
func SendToMany2f(list []*User, msg string, args ...interface{}) {
	for _, k := range list {
		k.SendLinef(msg, args...)
	}
}

func ValidChanName(name string) bool {
	if ChanHasBadChars(name) {
		return false
	}
	for _, k := range valid_chan_prefix {
		if strings.HasPrefix(name, k) {
			return true
		}
	}
	return false
}

//IMPORTANT: args must ABSOLUTELY be a valid privmsg command, or this will not work
//validity does not depend on leading ":", I don't care that much
func FormatMessageArgs(args []string) string {
	msg := strings.Join(args[2:], " ")
	msg = StripLeading(msg, ":")
	return msg
}

//removes the first instance of toremove, only if it has it
//r can be more than one char, but I wouldn't recommend it
func StripLeading(msg string, r string) string {
	if strings.HasPrefix(msg, r) {
		msg = strings.Replace(msg, r, "", 1)
	}
	return msg
}

func NickHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	for _, k := range valid_chan_prefix {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}

func ChanHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}
<commit_msg>add note to sendtomany<commit_after>package main

import (
	"bytes"
	"net"
	"strings"
)

func strcat(s1 string, s2 string) string {
	var buffer bytes.Buffer
	buffer.WriteString(s1)
	buffer.WriteString(s2)
	return buffer.String()
}

func GetUserByNick(nick string) *User {
	nick = strings.ToLower(nick)
	for _, k := range userlist {
		if strings.ToLower(k.nick) == nick {
			return k
		}
	}
	return nil
}

func GetIpFromConn(conn net.Conn) string {
	ip := conn.RemoteAddr().String()
	ip = strings.Split(ip, ":")[0]
	return ip
}

func GetChannelByName(name string) *Channel {
	for _, k := range chanlist {
		if strings.ToLower(name) == strings.ToLower(k.name) {
			return k
		}
	}
	return nil
}

//SendToMany() sends to a list of users, checking for duplicates. This is very inefficient, so should only be used for things like QUIT and NICK
func SendToMany(msg string, list []*User) {
	list2 := []*User{}
	for _, j := range list {
		match := false
		for _, k := range list2 {
			if j == k {
				match = true
				break
			}
		}
		if !match {
			list2 = append(list2, j)
		}
	}
	for _, j := range list2 {
		j.SendLine(msg)
	}
}

//SendToMany2f sends to all users in list ([]*User), does not check for dupes, accepts in printf format.
func SendToMany2f(list []*User, msg string, args ...interface{}) {
	for _, k := range list {
		k.SendLinef(msg, args...)
	}
}

func ValidChanName(name string) bool {
	if ChanHasBadChars(name) {
		return false
	}
	for _, k := range valid_chan_prefix {
		if strings.HasPrefix(name, k) {
			return true
		}
	}
	return false
}

//IMPORTANT: args must ABSOLUTELY be a valid privmsg command, or this will not work
//validity does not depend on leading ":", I don't care that much
func FormatMessageArgs(args []string) string {
	msg := strings.Join(args[2:], " ")
	msg = StripLeading(msg, ":")
	return msg
}

//removes the first instance of toremove, only if it has it
//r can be more than one char, but I wouldn't recommend it
func StripLeading(msg string, r string) string {
	if strings.HasPrefix(msg, r) {
		msg = strings.Replace(msg, r, "", 1)
	}
	return msg
}

func NickHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	for _, k := range valid_chan_prefix {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}

func ChanHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// isASCII reports whether the input is an ASCII C-style string.
func isASCII(s string) bool {
	for _, c := range s {
		if c >= 0x80 || c == 0x00 {
			return false
		}
	}
	return true
}

// toASCII converts the input to an ASCII C-style string.
// This a best effort conversion, so invalid characters are dropped.
func toASCII(s string) string {
	if isASCII(s) {
		return s
	}
	var buf bytes.Buffer
	for _, c := range s {
		if c < 0x80 && c != 0x00 {
			buf.WriteByte(byte(c))
		}
	}
	return buf.String()
}

type parser struct {
	err error // Last error seen
}

type formatter struct {
	err error // Last error seen
}

// parseString parses bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func (*parser) parseString(b []byte) string {
	n := 0
	for n < len(b) && b[n] != 0 {
		n++
	}
	return string(b[0:n])
}

// Write s into b, terminating it with a NUL if there is room.
func (f *formatter) formatString(b []byte, s string) {
	if len(s) > len(b) {
		f.err = ErrFieldTooLong
		return
	}
	ascii := toASCII(s)
	copy(b, ascii)
	if len(ascii) < len(b) {
		b[len(ascii)] = 0
	}
}

// fitsInBase256 reports whether x can be encoded into n bytes using base-256
// encoding. Unlike octal encoding, base-256 encoding does not require that the
// string ends with a NUL character. Thus, all n bytes are available for output.
//
// If operating in binary mode, this assumes strict GNU binary mode; which means
// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
// equivalent to the sign bit in two's complement form.
func fitsInBase256(n int, x int64) bool {
	var binBits = uint(n-1) * 8
	return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
}

// parseNumeric parses the input as being encoded in either base-256 or octal.
// This function may return negative numbers.
// If parsing fails or an integer overflow occurs, err will be set.
func (p *parser) parseNumeric(b []byte) int64 {
	// Check for base-256 (binary) format first.
	// If the first bit is set, then all following bits constitute a two's
	// complement encoded number in big-endian byte order.
	if len(b) > 0 && b[0]&0x80 != 0 {
		// Handling negative numbers relies on the following identity:
		//	-a-1 == ^a
		//
		// If the number is negative, we use an inversion mask to invert the
		// data bytes and treat the value as an unsigned number.
		var inv byte // 0x00 if positive or zero, 0xff if negative
		if b[0]&0x40 != 0 {
			inv = 0xff
		}

		var x uint64
		for i, c := range b {
			c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing
			if i == 0 {
				c &= 0x7f // Ignore signal bit in first byte
			}
			if (x >> 56) > 0 {
				p.err = ErrHeader // Integer overflow
				return 0
			}
			x = x<<8 | uint64(c)
		}
		if (x >> 63) > 0 {
			p.err = ErrHeader // Integer overflow
			return 0
		}
		if inv == 0xff {
			return ^int64(x)
		}
		return int64(x)
	}

	// Normal case is base-8 (octal) format.
	return p.parseOctal(b)
}

// formatNumeric encodes x into b using base-8 (octal) encoding if possible.
// Otherwise it will attempt to use base-256 (binary) encoding.
func (f *formatter) formatNumeric(b []byte, x int64) {
	if fitsInOctal(len(b), x) {
		f.formatOctal(b, x)
		return
	}

	if fitsInBase256(len(b), x) {
		for i := len(b) - 1; i >= 0; i-- {
			b[i] = byte(x)
			x >>= 8
		}
		b[0] |= 0x80 // Highest bit indicates binary format
		return
	}

	f.formatOctal(b, 0) // Last resort, just write zero
	f.err = ErrFieldTooLong
}

func (p *parser) parseOctal(b []byte) int64 {
	// Because unused fields are filled with NULs, we need
	// to skip leading NULs. Fields may also be padded with
	// spaces or NULs.
	// So we remove leading and trailing NULs and spaces to
	// be sure.
	b = bytes.Trim(b, " \x00")

	if len(b) == 0 {
		return 0
	}
	x, perr := strconv.ParseUint(p.parseString(b), 8, 64)
	if perr != nil {
		p.err = ErrHeader
	}
	return int64(x)
}

func (f *formatter) formatOctal(b []byte, x int64) {
	s := strconv.FormatInt(x, 8)
	// Add leading zeros, but leave room for a NUL.
	if n := len(b) - len(s) - 1; n > 0 {
		s = strings.Repeat("0", n) + s
	}
	f.formatString(b, s)
}

// fitsInOctal reports whether the integer x fits in a field n-bytes long
// using octal encoding with the appropriate NUL terminator.
func fitsInOctal(n int, x int64) bool {
	octBits := uint(n-1) * 3
	return x >= 0 && (n >= 22 || x < 1<<octBits)
}

// parsePAXTime takes a string of the form %d.%d as described in the PAX
// specification. Note that this implementation allows for negative timestamps,
// which is allowed for by the PAX specification, but not always portable.
func parsePAXTime(s string) (time.Time, error) {
	const maxNanoSecondDigits = 9

	// Split string into seconds and sub-seconds parts.
	ss, sn := s, ""
	if pos := strings.IndexByte(s, '.'); pos >= 0 {
		ss, sn = s[:pos], s[pos+1:]
	}

	// Parse the seconds.
	secs, err := strconv.ParseInt(ss, 10, 64)
	if err != nil {
		return time.Time{}, ErrHeader
	}
	if len(sn) == 0 {
		return time.Unix(secs, 0), nil // No sub-second values
	}

	// Parse the nanoseconds.
	if strings.Trim(sn, "0123456789") != "" {
		return time.Time{}, ErrHeader
	}
	if len(sn) < maxNanoSecondDigits {
		sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad
	} else {
		sn = sn[:maxNanoSecondDigits] // Right truncate
	}
	nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
	if len(ss) > 0 && ss[0] == '-' {
		return time.Unix(secs, -1*int64(nsecs)), nil // Negative correction
	}
	return time.Unix(secs, int64(nsecs)), nil
}

// TODO(dsnet): Implement formatPAXTime.

// parsePAXRecord parses the input PAX record string into a key-value pair.
// If parsing is successful, it will slice off the currently read record and
// return the remainder as r.
//
// A PAX record is of the following form:
//	"%d %s=%s\n" % (size, key, value)
func parsePAXRecord(s string) (k, v, r string, err error) {
	// The size field ends at the first space.
	sp := strings.IndexByte(s, ' ')
	if sp == -1 {
		return "", "", s, ErrHeader
	}

	// Parse the first token as a decimal integer.
	n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int
	if perr != nil || n < 5 || int64(len(s)) < n {
		return "", "", s, ErrHeader
	}

	// Extract everything between the space and the final newline.
	rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:]
	if nl != "\n" {
		return "", "", s, ErrHeader
	}

	// The first equals separates the key from the value.
	eq := strings.IndexByte(rec, '=')
	if eq == -1 {
		return "", "", s, ErrHeader
	}
	k, v = rec[:eq], rec[eq+1:]

	if !validPAXRecord(k, v) {
		return "", "", s, ErrHeader
	}
	return k, v, rem, nil
}

// formatPAXRecord formats a single PAX record, prefixing it with the
// appropriate length.
func formatPAXRecord(k, v string) (string, error) {
	if !validPAXRecord(k, v) {
		return "", ErrHeader
	}

	const padding = 3 // Extra padding for ' ', '=', and '\n'
	size := len(k) + len(v) + padding
	size += len(strconv.Itoa(size))
	record := fmt.Sprintf("%d %s=%s\n", size, k, v)

	// Final adjustment if adding size field increased the record size.
	if len(record) != size {
		size = len(record)
		record = fmt.Sprintf("%d %s=%s\n", size, k, v)
	}
	return record, nil
}

// validPAXRecord reports whether the key-value pair is valid.
// Keys and values should be UTF-8, but the number of bad writers out there
// forces us to be a more liberal.
// Thus, we only reject all keys with NUL, and only reject NULs in values
// for the PAX version of the USTAR string fields.
func validPAXRecord(k, v string) bool {
	switch k {
	case paxPath, paxLinkpath, paxUname, paxGname:
		return strings.IndexByte(v, 0) < 0
	default:
		return strings.IndexByte(k, 0) < 0
	}
}
<commit_msg>archive/tar: optimize formatPAXRecord() call<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

import (
	"bytes"
	"strconv"
	"strings"
	"time"
)

// isASCII reports whether the input is an ASCII C-style string.
func isASCII(s string) bool {
	for _, c := range s {
		if c >= 0x80 || c == 0x00 {
			return false
		}
	}
	return true
}

// toASCII converts the input to an ASCII C-style string.
// This a best effort conversion, so invalid characters are dropped.
func toASCII(s string) string {
	if isASCII(s) {
		return s
	}
	var buf bytes.Buffer
	for _, c := range s {
		if c < 0x80 && c != 0x00 {
			buf.WriteByte(byte(c))
		}
	}
	return buf.String()
}

type parser struct {
	err error // Last error seen
}

type formatter struct {
	err error // Last error seen
}

// parseString parses bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func (*parser) parseString(b []byte) string {
	n := 0
	for n < len(b) && b[n] != 0 {
		n++
	}
	return string(b[0:n])
}

// Write s into b, terminating it with a NUL if there is room.
func (f *formatter) formatString(b []byte, s string) {
	if len(s) > len(b) {
		f.err = ErrFieldTooLong
		return
	}
	ascii := toASCII(s)
	copy(b, ascii)
	if len(ascii) < len(b) {
		b[len(ascii)] = 0
	}
}

// fitsInBase256 reports whether x can be encoded into n bytes using base-256
// encoding. Unlike octal encoding, base-256 encoding does not require that the
// string ends with a NUL character. Thus, all n bytes are available for output.
//
// If operating in binary mode, this assumes strict GNU binary mode; which means
// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
// equivalent to the sign bit in two's complement form.
func fitsInBase256(n int, x int64) bool {
	var binBits = uint(n-1) * 8
	return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
}

// parseNumeric parses the input as being encoded in either base-256 or octal.
// This function may return negative numbers.
// If parsing fails or an integer overflow occurs, err will be set.
func (p *parser) parseNumeric(b []byte) int64 {
	// Check for base-256 (binary) format first.
	// If the first bit is set, then all following bits constitute a two's
	// complement encoded number in big-endian byte order.
	if len(b) > 0 && b[0]&0x80 != 0 {
		// Handling negative numbers relies on the following identity:
		//	-a-1 == ^a
		//
		// If the number is negative, we use an inversion mask to invert the
		// data bytes and treat the value as an unsigned number.
		var inv byte // 0x00 if positive or zero, 0xff if negative
		if b[0]&0x40 != 0 {
			inv = 0xff
		}

		var x uint64
		for i, c := range b {
			c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing
			if i == 0 {
				c &= 0x7f // Ignore signal bit in first byte
			}
			if (x >> 56) > 0 {
				p.err = ErrHeader // Integer overflow
				return 0
			}
			x = x<<8 | uint64(c)
		}
		if (x >> 63) > 0 {
			p.err = ErrHeader // Integer overflow
			return 0
		}
		if inv == 0xff {
			return ^int64(x)
		}
		return int64(x)
	}

	// Normal case is base-8 (octal) format.
	return p.parseOctal(b)
}

// formatNumeric encodes x into b using base-8 (octal) encoding if possible.
// Otherwise it will attempt to use base-256 (binary) encoding.
func (f *formatter) formatNumeric(b []byte, x int64) {
	if fitsInOctal(len(b), x) {
		f.formatOctal(b, x)
		return
	}

	if fitsInBase256(len(b), x) {
		for i := len(b) - 1; i >= 0; i-- {
			b[i] = byte(x)
			x >>= 8
		}
		b[0] |= 0x80 // Highest bit indicates binary format
		return
	}

	f.formatOctal(b, 0) // Last resort, just write zero
	f.err = ErrFieldTooLong
}

func (p *parser) parseOctal(b []byte) int64 {
	// Because unused fields are filled with NULs, we need
	// to skip leading NULs. Fields may also be padded with
	// spaces or NULs.
	// So we remove leading and trailing NULs and spaces to
	// be sure.
	b = bytes.Trim(b, " \x00")

	if len(b) == 0 {
		return 0
	}
	x, perr := strconv.ParseUint(p.parseString(b), 8, 64)
	if perr != nil {
		p.err = ErrHeader
	}
	return int64(x)
}

func (f *formatter) formatOctal(b []byte, x int64) {
	s := strconv.FormatInt(x, 8)
	// Add leading zeros, but leave room for a NUL.
	if n := len(b) - len(s) - 1; n > 0 {
		s = strings.Repeat("0", n) + s
	}
	f.formatString(b, s)
}

// fitsInOctal reports whether the integer x fits in a field n-bytes long
// using octal encoding with the appropriate NUL terminator.
func fitsInOctal(n int, x int64) bool {
	octBits := uint(n-1) * 3
	return x >= 0 && (n >= 22 || x < 1<<octBits)
}

// parsePAXTime takes a string of the form %d.%d as described in the PAX
// specification. Note that this implementation allows for negative timestamps,
// which is allowed for by the PAX specification, but not always portable.
func parsePAXTime(s string) (time.Time, error) {
	const maxNanoSecondDigits = 9

	// Split string into seconds and sub-seconds parts.
	ss, sn := s, ""
	if pos := strings.IndexByte(s, '.'); pos >= 0 {
		ss, sn = s[:pos], s[pos+1:]
	}

	// Parse the seconds.
	secs, err := strconv.ParseInt(ss, 10, 64)
	if err != nil {
		return time.Time{}, ErrHeader
	}
	if len(sn) == 0 {
		return time.Unix(secs, 0), nil // No sub-second values
	}

	// Parse the nanoseconds.
	if strings.Trim(sn, "0123456789") != "" {
		return time.Time{}, ErrHeader
	}
	if len(sn) < maxNanoSecondDigits {
		sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad
	} else {
		sn = sn[:maxNanoSecondDigits] // Right truncate
	}
	nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
	if len(ss) > 0 && ss[0] == '-' {
		return time.Unix(secs, -1*int64(nsecs)), nil // Negative correction
	}
	return time.Unix(secs, int64(nsecs)), nil
}

// TODO(dsnet): Implement formatPAXTime.

// parsePAXRecord parses the input PAX record string into a key-value pair.
// If parsing is successful, it will slice off the currently read record and
// return the remainder as r.
//
// A PAX record is of the following form:
//	"%d %s=%s\n" % (size, key, value)
func parsePAXRecord(s string) (k, v, r string, err error) {
	// The size field ends at the first space.
	sp := strings.IndexByte(s, ' ')
	if sp == -1 {
		return "", "", s, ErrHeader
	}

	// Parse the first token as a decimal integer.
	n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int
	if perr != nil || n < 5 || int64(len(s)) < n {
		return "", "", s, ErrHeader
	}

	// Extract everything between the space and the final newline.
	rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:]
	if nl != "\n" {
		return "", "", s, ErrHeader
	}

	// The first equals separates the key from the value.
	eq := strings.IndexByte(rec, '=')
	if eq == -1 {
		return "", "", s, ErrHeader
	}
	k, v = rec[:eq], rec[eq+1:]

	if !validPAXRecord(k, v) {
		return "", "", s, ErrHeader
	}
	return k, v, rem, nil
}

// formatPAXRecord formats a single PAX record, prefixing it with the
// appropriate length.
func formatPAXRecord(k, v string) (string, error) {
	if !validPAXRecord(k, v) {
		return "", ErrHeader
	}

	const padding = 3 // Extra padding for ' ', '=', and '\n'
	size := len(k) + len(v) + padding
	size += len(strconv.Itoa(size))
	record := strconv.Itoa(size) + " " + k + "=" + v + "\n"

	// Final adjustment if adding size field increased the record size.
	if len(record) != size {
		size = len(record)
		record = strconv.Itoa(size) + " " + k + "=" + v + "\n"
	}
	return record, nil
}

// validPAXRecord reports whether the key-value pair is valid.
// Keys and values should be UTF-8, but the number of bad writers out there
// forces us to be a more liberal.
// Thus, we only reject all keys with NUL, and only reject NULs in values
// for the PAX version of the USTAR string fields.
func validPAXRecord(k, v string) bool {
	switch k {
	case paxPath, paxLinkpath, paxUname, paxGname:
		return strings.IndexByte(v, 0) < 0
	default:
		return strings.IndexByte(k, 0) < 0
	}
}
<|endoftext|>


<commit_before>package controller

import (
	"net/http"

	"github.com/zenazn/goji/web"
)

type AuthController interface {
	authenticated(h web.HandlerFunc) web.HandlerFunc
}

type AuthControllerImpl struct {
	tokenizer Tokenizer
}

func NewAuthController(tokenizer Tokenizer) AuthController {
	return &AuthControllerImpl{tokenizer}
}
func (controller *AuthControllerImpl) authenticated(h web.HandlerFunc) web.HandlerFunc {
	return web.HandlerFunc(func(c web.C, w http.ResponseWriter, r *http.Request) {
		userID, err := controller.tokenizer.parse(r)
		if err == nil {
			c.Env["userID"] = userID
			h.ServeHTTPC(c, w, r)
		} else {
			unauthorized(c, w, r)
		}
	})
}

func unauthorized(c web.C, w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusUnauthorized)
}
<commit_msg>error handling first<commit_after>package controller

import (
	"net/http"

	"github.com/zenazn/goji/web"
)

type AuthController interface {
	authenticated(h web.HandlerFunc) web.HandlerFunc
}

type AuthControllerImpl struct {
	tokenizer Tokenizer
}

func NewAuthController(tokenizer Tokenizer) AuthController {
	return &AuthControllerImpl{tokenizer}
}
func (controller *AuthControllerImpl) authenticated(h web.HandlerFunc) web.HandlerFunc {
	return web.HandlerFunc(func(c web.C, w http.ResponseWriter, r *http.Request) {
		userID, err := controller.tokenizer.parse(r)
		if err != nil {
			unauthorized(c, w, r)
		} else {
			c.Env["userID"] = userID
			h.ServeHTTPC(c, w, r)
		}
	})
}

func unauthorized(c web.C, w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusUnauthorized)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cpu

import "runtime"

const cacheLineSize = 64

func initOptions() {
	options = []option{
		{Name: "fp", Feature: &ARM64.HasFP},
		{Name: "asimd", Feature: &ARM64.HasASIMD},
		{Name: "evstrm", Feature: &ARM64.HasEVTSTRM},
		{Name: "aes", Feature: &ARM64.HasAES},
		{Name: "fphp", Feature: &ARM64.HasFPHP},
		{Name: "jscvt", Feature: &ARM64.HasJSCVT},
		{Name: "lrcpc", Feature: &ARM64.HasLRCPC},
		{Name: "pmull", Feature: &ARM64.HasPMULL},
		{Name: "sha1", Feature: &ARM64.HasSHA1},
		{Name: "sha2", Feature: &ARM64.HasSHA2},
		{Name: "sha3", Feature: &ARM64.HasSHA3},
		{Name: "sha512", Feature: &ARM64.HasSHA512},
		{Name: "sm3", Feature: &ARM64.HasSM3},
		{Name: "sm4", Feature: &ARM64.HasSM4},
		{Name: "sve", Feature: &ARM64.HasSVE},
		{Name: "crc32", Feature: &ARM64.HasCRC32},
		{Name: "atomics", Feature: &ARM64.HasATOMICS},
		{Name: "asimdhp", Feature: &ARM64.HasASIMDHP},
		{Name: "cpuid", Feature: &ARM64.HasCPUID},
		{Name: "asimrdm", Feature: &ARM64.HasASIMDRDM},
		{Name: "fcma", Feature: &ARM64.HasFCMA},
		{Name: "dcpop", Feature: &ARM64.HasDCPOP},
		{Name: "asimddp", Feature: &ARM64.HasASIMDDP},
		{Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM},
	}
}

func archInit() {
	switch runtime.GOOS {
	case "android", "darwin", "ios", "netbsd", "openbsd":
		// Android and iOS don't seem to allow reading these registers.
		//
		// NetBSD:
		// ID_AA64ISAR0_EL1 is a privileged register and cannot be read from EL0.
		// It can be read via sysctl(3). Example for future implementers:
		// https://nxr.netbsd.org/xref/src/usr.sbin/cpuctl/arch/aarch64.c
		//
		// OpenBSD:
		// See https://golang.org/issue/31746
		//
		// Fake the minimal features expected by
		// TestARM64minimalFeatures.
		ARM64.HasASIMD = true
		ARM64.HasFP = true
	case "linux":
		doinit()
	default:
		readARM64Registers()
	}
}

func readARM64Registers() {
	Initialized = true

	// ID_AA64ISAR0_EL1
	isar0 := getisar0()

	switch extractBits(isar0, 4, 7) {
	case 1:
		ARM64.HasAES = true
	case 2:
		ARM64.HasAES = true
		ARM64.HasPMULL = true
	}

	switch extractBits(isar0, 8, 11) {
	case 1:
		ARM64.HasSHA1 = true
	}

	switch extractBits(isar0, 12, 15) {
	case 1:
		ARM64.HasSHA2 = true
	case 2:
		ARM64.HasSHA2 = true
		ARM64.HasSHA512 = true
	}

	switch extractBits(isar0, 16, 19) {
	case 1:
		ARM64.HasCRC32 = true
	}

	switch extractBits(isar0, 20, 23) {
	case 2:
		ARM64.HasATOMICS = true
	}

	switch extractBits(isar0, 28, 31) {
	case 1:
		ARM64.HasASIMDRDM = true
	}

	switch extractBits(isar0, 32, 35) {
	case 1:
		ARM64.HasSHA3 = true
	}

	switch extractBits(isar0, 36, 39) {
	case 1:
		ARM64.HasSM3 = true
	}

	switch extractBits(isar0, 40, 43) {
	case 1:
		ARM64.HasSM4 = true
	}

	switch extractBits(isar0, 44, 47) {
	case 1:
		ARM64.HasASIMDDP = true
	}

	// ID_AA64ISAR1_EL1
	isar1 := getisar1()

	switch extractBits(isar1, 0, 3) {
	case 1:
		ARM64.HasDCPOP = true
	}

	switch extractBits(isar1, 12, 15) {
	case 1:
		ARM64.HasJSCVT = true
	}

	switch extractBits(isar1, 16, 19) {
	case 1:
		ARM64.HasFCMA = true
	}

	switch extractBits(isar1, 20, 23) {
	case 1:
		ARM64.HasLRCPC = true
	}

	// ID_AA64PFR0_EL1
	pfr0 := getpfr0()

	switch extractBits(pfr0, 16, 19) {
	case 0:
		ARM64.HasFP = true
	case 1:
		ARM64.HasFP = true
		ARM64.HasFPHP = true
	}

	switch extractBits(pfr0, 20, 23) {
	case 0:
		ARM64.HasASIMD = true
	case 1:
		ARM64.HasASIMD = true
		ARM64.HasASIMDHP = true
	}

	switch extractBits(pfr0, 32, 35) {
	case 1:
		ARM64.HasSVE = true
	}
}

func extractBits(data uint64, start, end uint) uint {
	return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
}
<commit_msg>cpu: refactor parsing of ARM64 registers into separate function<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cpu

import "runtime"

const cacheLineSize = 64

func initOptions() {
	options = []option{
		{Name: "fp", Feature: &ARM64.HasFP},
		{Name: "asimd", Feature: &ARM64.HasASIMD},
		{Name: "evstrm", Feature: &ARM64.HasEVTSTRM},
		{Name: "aes", Feature: &ARM64.HasAES},
		{Name: "fphp", Feature: &ARM64.HasFPHP},
		{Name: "jscvt", Feature: &ARM64.HasJSCVT},
		{Name: "lrcpc", Feature: &ARM64.HasLRCPC},
		{Name: "pmull", Feature: &ARM64.HasPMULL},
		{Name: "sha1", Feature: &ARM64.HasSHA1},
		{Name: "sha2", Feature: &ARM64.HasSHA2},
		{Name: "sha3", Feature: &ARM64.HasSHA3},
		{Name: "sha512", Feature: &ARM64.HasSHA512},
		{Name: "sm3", Feature: &ARM64.HasSM3},
		{Name: "sm4", Feature: &ARM64.HasSM4},
		{Name: "sve", Feature: &ARM64.HasSVE},
		{Name: "crc32", Feature: &ARM64.HasCRC32},
		{Name: "atomics", Feature: &ARM64.HasATOMICS},
		{Name: "asimdhp", Feature: &ARM64.HasASIMDHP},
		{Name: "cpuid", Feature: &ARM64.HasCPUID},
		{Name: "asimrdm", Feature: &ARM64.HasASIMDRDM},
		{Name: "fcma", Feature: &ARM64.HasFCMA},
		{Name: "dcpop", Feature: &ARM64.HasDCPOP},
		{Name: "asimddp", Feature: &ARM64.HasASIMDDP},
		{Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM},
	}
}

func archInit() {
	switch runtime.GOOS {
	case "android", "darwin", "ios", "netbsd", "openbsd":
		// Android and iOS don't seem to allow reading these registers.
		//
		// NetBSD:
		// ID_AA64ISAR0_EL1 is a privileged register and cannot be read from EL0.
		// It can be read via sysctl(3). Example for future implementers:
		// https://nxr.netbsd.org/xref/src/usr.sbin/cpuctl/arch/aarch64.c
		//
		// OpenBSD:
		// See https://golang.org/issue/31746
		//
		// Fake the minimal features expected by
		// TestARM64minimalFeatures.
		ARM64.HasASIMD = true
		ARM64.HasFP = true
	case "linux":
		doinit()
	default:
		readARM64Registers()
	}
}

func readARM64Registers() {
	Initialized = true

	parseARM64SystemRegisters(getisar0(), getisar1(), getpfr0())
}

func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) {
	// ID_AA64ISAR0_EL1
	switch extractBits(isar0, 4, 7) {
	case 1:
		ARM64.HasAES = true
	case 2:
		ARM64.HasAES = true
		ARM64.HasPMULL = true
	}

	switch extractBits(isar0, 8, 11) {
	case 1:
		ARM64.HasSHA1 = true
	}

	switch extractBits(isar0, 12, 15) {
	case 1:
		ARM64.HasSHA2 = true
	case 2:
		ARM64.HasSHA2 = true
		ARM64.HasSHA512 = true
	}

	switch extractBits(isar0, 16, 19) {
	case 1:
		ARM64.HasCRC32 = true
	}

	switch extractBits(isar0, 20, 23) {
	case 2:
		ARM64.HasATOMICS = true
	}

	switch extractBits(isar0, 28, 31) {
	case 1:
		ARM64.HasASIMDRDM = true
	}

	switch extractBits(isar0, 32, 35) {
	case 1:
		ARM64.HasSHA3 = true
	}

	switch extractBits(isar0, 36, 39) {
	case 1:
		ARM64.HasSM3 = true
	}

	switch extractBits(isar0, 40, 43) {
	case 1:
		ARM64.HasSM4 = true
	}

	switch extractBits(isar0, 44, 47) {
	case 1:
		ARM64.HasASIMDDP = true
	}

	// ID_AA64ISAR1_EL1
	switch extractBits(isar1, 0, 3) {
	case 1:
		ARM64.HasDCPOP = true
	}

	switch extractBits(isar1, 12, 15) {
	case 1:
		ARM64.HasJSCVT = true
	}

	switch extractBits(isar1, 16, 19) {
	case 1:
		ARM64.HasFCMA = true
	}

	switch extractBits(isar1, 20, 23) {
	case 1:
		ARM64.HasLRCPC = true
	}

	// ID_AA64PFR0_EL1
	switch extractBits(pfr0, 16, 19) {
	case 0:
		ARM64.HasFP = true
	case 1:
		ARM64.HasFP = true
		ARM64.HasFPHP = true
	}

	switch extractBits(pfr0, 20, 23) {
	case 0:
		ARM64.HasASIMD = true
	case 1:
		ARM64.HasASIMD = true
		ARM64.HasASIMDHP = true
	}

	switch extractBits(pfr0, 32, 35) {
	case 1:
		ARM64.HasSVE = true
	}
}

func extractBits(data uint64, start, end uint) uint {
	return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
}
<|endoftext|>


<commit_before>package perceptron

import (
	"errors"

	"github.com/mitsuse/perceptron-go/vector"
)

type Perceptron struct {
	iteration int
}

func New(iteration byte) *Perceptron {
	p := &Perceptron{
		iteration: int(iteration),
	}

	return p
}

func (p *Perceptron) Learn(classifier *Classifier, iter InstanceIter) error {
	for iteration := 1; iteration <= p.iteration; iteration++ {
		for iter.HasNext() {
			instance := iter.Get()

			if err := p.learnInstance(classifier, instance); err != nil {
				return err
			}
		}

		if err := iter.Error(); err != nil {
			return err
		}

		if err := iter.Init(); err != nil {
			return err
		}
	}

	return nil
}

func (p *Perceptron) learnInstance(classifier *Classifier, instance Instance) error {
	inference, err := classifier.Classify(instance)
	if err != nil {
		return err
	}

	if instance.Label() == inference.Label() {
		classifier.Weight().Add(inference.Update())

		if classifier.Weight().Undefined() {
			// TODO: Write the error message.
			return errors.New("")
		}
	}

	return nil
}

type Learner interface {
	Learn(classifier *Classifier, iter InstanceIter) error
}

type Instance interface {
	Label() int
	Score() float64
	Feature() vector.Vector

	SetLabel(label int)
	SetScore(score float64)
	SetFeature(vector vector.Vector)

	Update() vector.Vector
	Clone() Instance
}

type InstanceIter interface {
	HasNext() bool
	Get() Instance
	Error() error
	Init() error
}
<commit_msg>If "iteration" equals or smaller than 0, set 1 to "iteration".<commit_after>package perceptron

import (
	"errors"

	"github.com/mitsuse/perceptron-go/vector"
)

type Perceptron struct {
	iteration int
}

func New(iteration int) *Perceptron {
	if iteration <= 0 {
		iteration = 1
	}

	p := &Perceptron{
		iteration: int(iteration),
	}

	return p
}

func (p *Perceptron) Learn(classifier *Classifier, iter InstanceIter) error {
	for iteration := 1; iteration <= p.iteration; iteration++ {
		for iter.HasNext() {
			instance := iter.Get()

			if err := p.learnInstance(classifier, instance); err != nil {
				return err
			}
		}

		if err := iter.Error(); err != nil {
			return err
		}

		if err := iter.Init(); err != nil {
			return err
		}
	}

	return nil
}

func (p *Perceptron) learnInstance(classifier *Classifier, instance Instance) error {
	inference, err := classifier.Classify(instance)
	if err != nil {
		return err
	}

	if instance.Label() == inference.Label() {
		classifier.Weight().Add(inference.Update())

		if classifier.Weight().Undefined() {
			// TODO: Write the error message.
			return errors.New("")
		}
	}

	return nil
}

type Learner interface {
	Learn(classifier *Classifier, iter InstanceIter) error
}

type Instance interface {
	Label() int
	Score() float64
	Feature() vector.Vector

	SetLabel(label int)
	SetScore(score float64)
	SetFeature(vector vector.Vector)

	Update() vector.Vector
	Clone() Instance
}

type InstanceIter interface {
	HasNext() bool
	Get() Instance
	Error() error
	Init() error
}
<|endoftext|>


<commit_before>package main

import (
	"sort"
)

type Model struct {
	weight    map[string]float64
	cumWeight map[string]float64
	count     int
}

func NewModel() *Model {
	return &Model{make(map[string]float64), make(map[string]float64), 1}
}

func GetAccuracy(gold []LabelType, predict []LabelType) float64 {
	if len(gold) != len(predict) {
		return 0.0
	}
	sum := 0.0
	for i, v := range gold {
		if v == predict[i] {
			sum += 1.0
		}
	}
	return sum / float64(len(gold))
}

func (model *Model) Learn(example Example) {
	predict := model.predictForTraining(example.Fv)
	if example.Label != predict {
		for _, f := range example.Fv {
			w, _ := model.weight[f]
			cumW, _ := model.cumWeight[f]
			model.weight[f] = w + float64(example.Label)*1.0
			model.cumWeight[f] = cumW + float64(model.count)*float64(example.Label)*1.0
		}
		model.count += 1
	}
}

func (model *Model) predictForTraining(features FeatureVector) LabelType {
	result := 0.0
	for _, f := range features {
		w, ok := model.weight[f]
		if ok {
			result = result + w*1.0
		}
	}
	if result > 0 {
		return POSITIVE
	}
	return NEGATIVE
}

func (model Model) PredictScore(features FeatureVector) float64 {
	result := 0.0
	for _, f := range features {
		w, ok := model.weight[f]
		if ok {
			result = result + w*1.0
		}

		w, ok = model.cumWeight[f]
		if ok {
			result = result - w*1.0/float64(model.count)
		}

	}
	return result
}

func (model Model) Predict(features FeatureVector) LabelType {
	if model.PredictScore(features) > 0 {
		return POSITIVE
	}
	return NEGATIVE
}

func ExtractGoldLabels(examples Examples) []LabelType {
	golds := make([]LabelType, 0, 0)
	for _, e := range examples {
		golds = append(golds, e.Label)
	}
	return golds
}

func (model Model) SortByScore(examples Examples) Examples {
	alreadyLabeled := make(map[string]bool)

	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeled[e.Url] = true
	}

	var unlabeledExamples Examples
	for _, e := range examples {
		e.Score = model.PredictScore(e.Fv)
		if _, ok := alreadyLabeled[e.Url]; ok {
			continue
		}

		if !e.IsLabeled() && e.Score != 0.0 {
			unlabeledExamples = append(unlabeledExamples, e)
		}
	}

	sort.Sort(unlabeledExamples)
	return unlabeledExamples
}

func TrainedModel(examples Examples) *Model {
	train := FilterLabeledExamples(examples)
	model := NewModel()
	for iter := 0; iter < 10; iter++ {
		shuffle(train)
		for _, example := range train {
			model.Learn(*example)
		}

		trainPredicts := make([]LabelType, len(train))
		for i, example := range train {
			trainPredicts[i] = model.Predict(example.Fv)
		}
		// fmt.Println(fmt.Sprintf("Iter:%d\tAccuracy:%0.03f", iter, GetAccuracy(ExtractGoldLabels(train), trainPredicts)))
	}
	return model
}
<commit_msg>割とゆらぐので、30回くらい回す<commit_after>package main

import (
	"sort"
)

type Model struct {
	weight    map[string]float64
	cumWeight map[string]float64
	count     int
}

func NewModel() *Model {
	return &Model{make(map[string]float64), make(map[string]float64), 1}
}

func GetAccuracy(gold []LabelType, predict []LabelType) float64 {
	if len(gold) != len(predict) {
		return 0.0
	}
	sum := 0.0
	for i, v := range gold {
		if v == predict[i] {
			sum += 1.0
		}
	}
	return sum / float64(len(gold))
}

func (model *Model) Learn(example Example) {
	predict := model.predictForTraining(example.Fv)
	if example.Label != predict {
		for _, f := range example.Fv {
			w, _ := model.weight[f]
			cumW, _ := model.cumWeight[f]
			model.weight[f] = w + float64(example.Label)*1.0
			model.cumWeight[f] = cumW + float64(model.count)*float64(example.Label)*1.0
		}
		model.count += 1
	}
}

func (model *Model) predictForTraining(features FeatureVector) LabelType {
	result := 0.0
	for _, f := range features {
		w, ok := model.weight[f]
		if ok {
			result = result + w*1.0
		}
	}
	if result > 0 {
		return POSITIVE
	}
	return NEGATIVE
}

func (model Model) PredictScore(features FeatureVector) float64 {
	result := 0.0
	for _, f := range features {
		w, ok := model.weight[f]
		if ok {
			result = result + w*1.0
		}

		w, ok = model.cumWeight[f]
		if ok {
			result = result - w*1.0/float64(model.count)
		}

	}
	return result
}

func (model Model) Predict(features FeatureVector) LabelType {
	if model.PredictScore(features) > 0 {
		return POSITIVE
	}
	return NEGATIVE
}

func ExtractGoldLabels(examples Examples) []LabelType {
	golds := make([]LabelType, 0, 0)
	for _, e := range examples {
		golds = append(golds, e.Label)
	}
	return golds
}

func (model Model) SortByScore(examples Examples) Examples {
	alreadyLabeled := make(map[string]bool)

	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeled[e.Url] = true
	}

	var unlabeledExamples Examples
	for _, e := range examples {
		e.Score = model.PredictScore(e.Fv)
		if _, ok := alreadyLabeled[e.Url]; ok {
			continue
		}

		if !e.IsLabeled() && e.Score != 0.0 {
			unlabeledExamples = append(unlabeledExamples, e)
		}
	}

	sort.Sort(unlabeledExamples)
	return unlabeledExamples
}

func TrainedModel(examples Examples) *Model {
	train := FilterLabeledExamples(examples)
	model := NewModel()
	for iter := 0; iter < 30; iter++ {
		shuffle(train)
		for _, example := range train {
			model.Learn(*example)
		}

		trainPredicts := make([]LabelType, len(train))
		for i, example := range train {
			trainPredicts[i] = model.Predict(example.Fv)
		}
		// fmt.Println(fmt.Sprintf("Iter:%d\tAccuracy:%0.03f", iter, GetAccuracy(ExtractGoldLabels(train), trainPredicts)))
	}
	return model
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/bitly/go-simplejson"
)

const pingPath = "/ping"
const signInPath = "/oauth2/sign_in"
const oauthStartPath = "/oauth2/start"
const oauthCallbackPath = "/oauth2/callback"

type OauthProxy struct {
	CookieSeed      string
	CookieKey       string
	CookieDomain    string
	CookieHttpsOnly bool
	CookieHttpOnly  bool
	CookieExpire    time.Duration
	Validator       func(string) bool

	redirectUrl         *url.URL // the url to receive requests at
	oauthRedemptionUrl  *url.URL // endpoint to redeem the code
	oauthLoginUrl       *url.URL // to redirect the user to
	oauthScope          string
	clientID            string
	clientSecret        string
	SignInMessage       string
	HtpasswdFile        *HtpasswdFile
	DisplayHtpasswdForm bool
	serveMux            *http.ServeMux
	PassBasicAuth       bool
	skipAuthRegex       []string
	compiledRegex       []*regexp.Regexp
}

func NewReverseProxy(target *url.URL) (proxy *httputil.ReverseProxy) {
    return httputil.NewSingleHostReverseProxy(target)
}

func NewOauthProxy(opts *Options, validator func(string) bool) *OauthProxy {
	login, _ := url.Parse("https://accounts.google.com/o/oauth2/auth")
	redeem, _ := url.Parse("https://accounts.google.com/o/oauth2/token")
	serveMux := http.NewServeMux()
	for _, u := range opts.proxyUrls {
		path := u.Path
		u.Path = ""
		log.Printf("mapping path %q => upstream %q", path, u)
		serveMux.Handle(path, NewReverseProxy(u))
	}
	for _, u := range opts.CompiledRegex {
		log.Printf("compiled skip-auth-regex => %q", u)
	}

	redirectUrl := opts.redirectUrl
	redirectUrl.Path = oauthCallbackPath

	log.Printf("OauthProxy configured for %s", opts.ClientID)
	domain := opts.CookieDomain
	if domain == "" {
		domain = "<default>"
	}
	log.Printf("Cookie settings: https_only: %v httponly: %v expiry: %s domain:%s", opts.CookieHttpsOnly, opts.CookieHttpOnly, opts.CookieExpire, domain)
	return &OauthProxy{
		CookieKey:       "_oauthproxy",
		CookieSeed:      opts.CookieSecret,
		CookieDomain:    opts.CookieDomain,
		CookieHttpsOnly: opts.CookieHttpsOnly,
		CookieHttpOnly:  opts.CookieHttpOnly,
		CookieExpire:    opts.CookieExpire,
		Validator:       validator,

		clientID:           opts.ClientID,
		clientSecret:       opts.ClientSecret,
		oauthScope:         "profile email",
		oauthRedemptionUrl: redeem,
		oauthLoginUrl:      login,
		serveMux:           serveMux,
		redirectUrl:        redirectUrl,
		skipAuthRegex:      opts.SkipAuthRegex,
		compiledRegex:      opts.CompiledRegex,
		PassBasicAuth:      opts.PassBasicAuth,
	}
}

func (p *OauthProxy) GetLoginURL(redirectUrl string) string {
	params := url.Values{}
	params.Add("redirect_uri", p.redirectUrl.String())
	params.Add("approval_prompt", "force")
	params.Add("scope", p.oauthScope)
	params.Add("client_id", p.clientID)
	params.Add("response_type", "code")
	if strings.HasPrefix(redirectUrl, "/") {
		params.Add("state", redirectUrl)
	}
	return fmt.Sprintf("%s?%s", p.oauthLoginUrl, params.Encode())
}

func apiRequest(req *http.Request) (*simplejson.Json, error) {
	httpclient := &http.Client{}
	resp, err := httpclient.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		log.Printf("got response code %d - %s", resp.StatusCode, body)
		return nil, errors.New("api request returned non 200 status code")
	}
	data, err := simplejson.NewJson(body)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func (p *OauthProxy) displayCustomLoginForm() bool {
	return p.HtpasswdFile != nil && p.DisplayHtpasswdForm
}

func (p *OauthProxy) redeemCode(code string) (string, string, error) {
	if code == "" {
		return "", "", errors.New("missing code")
	}
	params := url.Values{}
	params.Add("redirect_uri", p.redirectUrl.String())
	params.Add("client_id", p.clientID)
	params.Add("client_secret", p.clientSecret)
	params.Add("code", code)
	params.Add("grant_type", "authorization_code")
	req, err := http.NewRequest("POST", p.oauthRedemptionUrl.String(), bytes.NewBufferString(params.Encode()))
	if err != nil {
		log.Printf("failed building request %s", err.Error())
		return "", "", err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	json, err := apiRequest(req)
	if err != nil {
		log.Printf("failed making request %s", err)
		return "", "", err
	}
	access_token, err := json.Get("access_token").String()
	if err != nil {
		return "", "", err
	}

	idToken, err := json.Get("id_token").String()
	if err != nil {
		return "", "", err
	}

	// id_token is a base64 encode ID token payload
	// https://developers.google.com/accounts/docs/OAuth2Login#obtainuserinfo
	jwt := strings.Split(idToken, ".")
	b, err := jwtDecodeSegment(jwt[1])
	if err != nil {
		return "", "", err
	}
	data, err := simplejson.NewJson(b)
	if err != nil {
		return "", "", err
	}
	email, err := data.Get("email").String()
	if err != nil {
		return "", "", err
	}

	return access_token, email, nil
}

func jwtDecodeSegment(seg string) ([]byte, error) {
	if l := len(seg) % 4; l > 0 {
		seg += strings.Repeat("=", 4-l)
	}

	return base64.URLEncoding.DecodeString(seg)
}

func (p *OauthProxy) ClearCookie(rw http.ResponseWriter, req *http.Request) {
	domain := strings.Split(req.Host, ":")[0]
	if p.CookieDomain != "" && strings.HasSuffix(domain, p.CookieDomain) {
		domain = p.CookieDomain
	}
	cookie := &http.Cookie{
		Name:     p.CookieKey,
		Value:    "",
		Path:     "/",
		Domain:   domain,
		Expires:  time.Now().Add(time.Duration(1) * time.Hour * -1),
		HttpOnly: p.CookieHttpOnly,
	}
	http.SetCookie(rw, cookie)
}

func (p *OauthProxy) SetCookie(rw http.ResponseWriter, req *http.Request, val string) {

	domain := strings.Split(req.Host, ":")[0] // strip the port (if any)
	if p.CookieDomain != "" && strings.HasSuffix(domain, p.CookieDomain) {
		domain = p.CookieDomain
	}
	cookie := &http.Cookie{
		Name:     p.CookieKey,
		Value:    signedCookieValue(p.CookieSeed, p.CookieKey, val),
		Path:     "/",
		Domain:   domain,
		HttpOnly: p.CookieHttpOnly,
		Secure:   p.CookieHttpsOnly,
		Expires:  time.Now().Add(p.CookieExpire),
	}
	http.SetCookie(rw, cookie)
}

func (p *OauthProxy) PingPage(rw http.ResponseWriter) {
	rw.WriteHeader(http.StatusOK)
	fmt.Fprintf(rw, "OK")
}

func (p *OauthProxy) ErrorPage(rw http.ResponseWriter, code int, title string, message string) {
	log.Printf("ErrorPage %d %s %s", code, title, message)
	rw.WriteHeader(code)
	templates := getTemplates()
	t := struct {
		Title   string
		Message string
	}{
		Title:   fmt.Sprintf("%d %s", code, title),
		Message: message,
	}
	templates.ExecuteTemplate(rw, "error.html", t)
}

func (p *OauthProxy) SignInPage(rw http.ResponseWriter, req *http.Request, code int) {
	p.ClearCookie(rw, req)
	rw.WriteHeader(code)
	templates := getTemplates()

	t := struct {
		SignInMessage string
		CustomLogin   bool
		Redirect      string
		Version       string
	}{
		SignInMessage: p.SignInMessage,
		CustomLogin:   p.displayCustomLoginForm(),
		Redirect:      req.URL.RequestURI(),
		Version:       VERSION,
	}
	templates.ExecuteTemplate(rw, "sign_in.html", t)
}

func (p *OauthProxy) ManualSignIn(rw http.ResponseWriter, req *http.Request) (string, bool) {
	if req.Method != "POST" || p.HtpasswdFile == nil {
		return "", false
	}
	user := req.FormValue("username")
	passwd := req.FormValue("password")
	if user == "" {
		return "", false
	}
	// check auth
	if p.HtpasswdFile.Validate(user, passwd) {
		log.Printf("authenticated %s via manual sign in", user)
		return user, true
	}
	return "", false
}

func (p *OauthProxy) GetRedirect(req *http.Request) (string, error) {
	err := req.ParseForm()

	if err != nil {
		return "", err
	}

	redirect := req.FormValue("rd")

	if redirect == "" {
		redirect = "/"
	}

	return redirect, err
}

func (p *OauthProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// check if this is a redirect back at the end of oauth
	remoteAddr := req.RemoteAddr
	if req.Header.Get("X-Real-IP") != "" {
		remoteAddr += fmt.Sprintf(" (%q)", req.Header.Get("X-Real-IP"))
	}
	log.Printf("%s %s %s", remoteAddr, req.Method, req.URL.RequestURI())

	var ok bool
	var user string
	var email string

	if req.URL.Path == pingPath {
		p.PingPage(rw)
		return
	}

	for _, u := range p.compiledRegex {
		match := u.MatchString(req.URL.Path)
		if match {
			p.serveMux.ServeHTTP(rw, req)
			return
		}

	}

	if req.URL.Path == signInPath {
		redirect, err := p.GetRedirect(req)
		if err != nil {
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}

		user, ok = p.ManualSignIn(rw, req)
		if ok {
			p.SetCookie(rw, req, user)
			http.Redirect(rw, req, redirect, 302)
		} else {
			p.SignInPage(rw, req, 200)
		}
		return
	}
	if req.URL.Path == oauthStartPath {
		redirect, err := p.GetRedirect(req)
		if err != nil {
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}
		http.Redirect(rw, req, p.GetLoginURL(redirect), 302)
		return
	}
	if req.URL.Path == oauthCallbackPath {
		// finish the oauth cycle
		err := req.ParseForm()
		if err != nil {
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}
		errorString := req.Form.Get("error")
		if errorString != "" {
			p.ErrorPage(rw, 403, "Permission Denied", errorString)
			return
		}

		_, email, err := p.redeemCode(req.Form.Get("code"))
		if err != nil {
			log.Printf("%s error redeeming code %s", remoteAddr, err)
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}

		redirect := req.Form.Get("state")
		if redirect == "" {
			redirect = "/"
		}

		// set cookie, or deny
		if p.Validator(email) {
			log.Printf("%s authenticating %s completed", remoteAddr, email)
			p.SetCookie(rw, req, email)
			http.Redirect(rw, req, redirect, 302)
			return
		} else {
			p.ErrorPage(rw, 403, "Permission Denied", "Invalid Account")
			return
		}
	}

	if !ok {
		cookie, err := req.Cookie(p.CookieKey)
		if err == nil {
			email, ok = validateCookie(cookie, p.CookieSeed)
			user = strings.Split(email, "@")[0]
		}
	}

	if !ok {
		user, ok = p.CheckBasicAuth(req)
		// if we want to promote basic auth requests to cookie'd requests, we could do that here
		// not sure that would be ideal in all circumstances though
		// if ok {
		// 	p.SetCookie(rw, req, user)
		// }
	}

	if !ok {
		log.Printf("%s - invalid cookie session", remoteAddr)
		p.SignInPage(rw, req, 403)
		return
	}

	// At this point, the user is authenticated. proxy normally
	if p.PassBasicAuth {
		req.SetBasicAuth(user, "")
		req.Header["X-Forwarded-User"] = []string{user}
		req.Header["X-Forwarded-Email"] = []string{email}
	}

	p.serveMux.ServeHTTP(rw, req)
}

func (p *OauthProxy) CheckBasicAuth(req *http.Request) (string, bool) {
	if p.HtpasswdFile == nil {
		return "", false
	}
	s := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Basic" {
		return "", false
	}
	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return "", false
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return "", false
	}
	if p.HtpasswdFile.Validate(pair[0], pair[1]) {
		log.Printf("authenticated %s via basic auth", pair[0])
		return pair[0], true
	}
	return "", false
}
<commit_msg>Requests are proxied to the Host specified by the target.<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/bitly/go-simplejson"
)

const pingPath = "/ping"
const signInPath = "/oauth2/sign_in"
const oauthStartPath = "/oauth2/start"
const oauthCallbackPath = "/oauth2/callback"

type OauthProxy struct {
	CookieSeed      string
	CookieKey       string
	CookieDomain    string
	CookieHttpsOnly bool
	CookieHttpOnly  bool
	CookieExpire    time.Duration
	Validator       func(string) bool

	redirectUrl         *url.URL // the url to receive requests at
	oauthRedemptionUrl  *url.URL // endpoint to redeem the code
	oauthLoginUrl       *url.URL // to redirect the user to
	oauthScope          string
	clientID            string
	clientSecret        string
	SignInMessage       string
	HtpasswdFile        *HtpasswdFile
	DisplayHtpasswdForm bool
	serveMux            *http.ServeMux
	PassBasicAuth       bool
	skipAuthRegex       []string
	compiledRegex       []*regexp.Regexp
}

func NewReverseProxy(target *url.URL) (proxy *httputil.ReverseProxy) {
    proxy = httputil.NewSingleHostReverseProxy(target)
    director := proxy.Director
    proxy.Director = func(req *http.Request) {
        director(req)
        req.Host = target.Host
    }
    return proxy
}

func NewOauthProxy(opts *Options, validator func(string) bool) *OauthProxy {
	login, _ := url.Parse("https://accounts.google.com/o/oauth2/auth")
	redeem, _ := url.Parse("https://accounts.google.com/o/oauth2/token")
	serveMux := http.NewServeMux()
	for _, u := range opts.proxyUrls {
		path := u.Path
		u.Path = ""
		log.Printf("mapping path %q => upstream %q", path, u)
		serveMux.Handle(path, NewReverseProxy(u))
	}
	for _, u := range opts.CompiledRegex {
		log.Printf("compiled skip-auth-regex => %q", u)
	}

	redirectUrl := opts.redirectUrl
	redirectUrl.Path = oauthCallbackPath

	log.Printf("OauthProxy configured for %s", opts.ClientID)
	domain := opts.CookieDomain
	if domain == "" {
		domain = "<default>"
	}
	log.Printf("Cookie settings: https_only: %v httponly: %v expiry: %s domain:%s", opts.CookieHttpsOnly, opts.CookieHttpOnly, opts.CookieExpire, domain)
	return &OauthProxy{
		CookieKey:       "_oauthproxy",
		CookieSeed:      opts.CookieSecret,
		CookieDomain:    opts.CookieDomain,
		CookieHttpsOnly: opts.CookieHttpsOnly,
		CookieHttpOnly:  opts.CookieHttpOnly,
		CookieExpire:    opts.CookieExpire,
		Validator:       validator,

		clientID:           opts.ClientID,
		clientSecret:       opts.ClientSecret,
		oauthScope:         "profile email",
		oauthRedemptionUrl: redeem,
		oauthLoginUrl:      login,
		serveMux:           serveMux,
		redirectUrl:        redirectUrl,
		skipAuthRegex:      opts.SkipAuthRegex,
		compiledRegex:      opts.CompiledRegex,
		PassBasicAuth:      opts.PassBasicAuth,
	}
}

func (p *OauthProxy) GetLoginURL(redirectUrl string) string {
	params := url.Values{}
	params.Add("redirect_uri", p.redirectUrl.String())
	params.Add("approval_prompt", "force")
	params.Add("scope", p.oauthScope)
	params.Add("client_id", p.clientID)
	params.Add("response_type", "code")
	if strings.HasPrefix(redirectUrl, "/") {
		params.Add("state", redirectUrl)
	}
	return fmt.Sprintf("%s?%s", p.oauthLoginUrl, params.Encode())
}

func apiRequest(req *http.Request) (*simplejson.Json, error) {
	httpclient := &http.Client{}
	resp, err := httpclient.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		log.Printf("got response code %d - %s", resp.StatusCode, body)
		return nil, errors.New("api request returned non 200 status code")
	}
	data, err := simplejson.NewJson(body)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func (p *OauthProxy) displayCustomLoginForm() bool {
	return p.HtpasswdFile != nil && p.DisplayHtpasswdForm
}

func (p *OauthProxy) redeemCode(code string) (string, string, error) {
	if code == "" {
		return "", "", errors.New("missing code")
	}
	params := url.Values{}
	params.Add("redirect_uri", p.redirectUrl.String())
	params.Add("client_id", p.clientID)
	params.Add("client_secret", p.clientSecret)
	params.Add("code", code)
	params.Add("grant_type", "authorization_code")
	req, err := http.NewRequest("POST", p.oauthRedemptionUrl.String(), bytes.NewBufferString(params.Encode()))
	if err != nil {
		log.Printf("failed building request %s", err.Error())
		return "", "", err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	json, err := apiRequest(req)
	if err != nil {
		log.Printf("failed making request %s", err)
		return "", "", err
	}
	access_token, err := json.Get("access_token").String()
	if err != nil {
		return "", "", err
	}

	idToken, err := json.Get("id_token").String()
	if err != nil {
		return "", "", err
	}

	// id_token is a base64 encode ID token payload
	// https://developers.google.com/accounts/docs/OAuth2Login#obtainuserinfo
	jwt := strings.Split(idToken, ".")
	b, err := jwtDecodeSegment(jwt[1])
	if err != nil {
		return "", "", err
	}
	data, err := simplejson.NewJson(b)
	if err != nil {
		return "", "", err
	}
	email, err := data.Get("email").String()
	if err != nil {
		return "", "", err
	}

	return access_token, email, nil
}

func jwtDecodeSegment(seg string) ([]byte, error) {
	if l := len(seg) % 4; l > 0 {
		seg += strings.Repeat("=", 4-l)
	}

	return base64.URLEncoding.DecodeString(seg)
}

func (p *OauthProxy) ClearCookie(rw http.ResponseWriter, req *http.Request) {
	domain := strings.Split(req.Host, ":")[0]
	if p.CookieDomain != "" && strings.HasSuffix(domain, p.CookieDomain) {
		domain = p.CookieDomain
	}
	cookie := &http.Cookie{
		Name:     p.CookieKey,
		Value:    "",
		Path:     "/",
		Domain:   domain,
		Expires:  time.Now().Add(time.Duration(1) * time.Hour * -1),
		HttpOnly: p.CookieHttpOnly,
	}
	http.SetCookie(rw, cookie)
}

func (p *OauthProxy) SetCookie(rw http.ResponseWriter, req *http.Request, val string) {

	domain := strings.Split(req.Host, ":")[0] // strip the port (if any)
	if p.CookieDomain != "" && strings.HasSuffix(domain, p.CookieDomain) {
		domain = p.CookieDomain
	}
	cookie := &http.Cookie{
		Name:     p.CookieKey,
		Value:    signedCookieValue(p.CookieSeed, p.CookieKey, val),
		Path:     "/",
		Domain:   domain,
		HttpOnly: p.CookieHttpOnly,
		Secure:   p.CookieHttpsOnly,
		Expires:  time.Now().Add(p.CookieExpire),
	}
	http.SetCookie(rw, cookie)
}

func (p *OauthProxy) PingPage(rw http.ResponseWriter) {
	rw.WriteHeader(http.StatusOK)
	fmt.Fprintf(rw, "OK")
}

func (p *OauthProxy) ErrorPage(rw http.ResponseWriter, code int, title string, message string) {
	log.Printf("ErrorPage %d %s %s", code, title, message)
	rw.WriteHeader(code)
	templates := getTemplates()
	t := struct {
		Title   string
		Message string
	}{
		Title:   fmt.Sprintf("%d %s", code, title),
		Message: message,
	}
	templates.ExecuteTemplate(rw, "error.html", t)
}

func (p *OauthProxy) SignInPage(rw http.ResponseWriter, req *http.Request, code int) {
	p.ClearCookie(rw, req)
	rw.WriteHeader(code)
	templates := getTemplates()

	t := struct {
		SignInMessage string
		CustomLogin   bool
		Redirect      string
		Version       string
	}{
		SignInMessage: p.SignInMessage,
		CustomLogin:   p.displayCustomLoginForm(),
		Redirect:      req.URL.RequestURI(),
		Version:       VERSION,
	}
	templates.ExecuteTemplate(rw, "sign_in.html", t)
}

func (p *OauthProxy) ManualSignIn(rw http.ResponseWriter, req *http.Request) (string, bool) {
	if req.Method != "POST" || p.HtpasswdFile == nil {
		return "", false
	}
	user := req.FormValue("username")
	passwd := req.FormValue("password")
	if user == "" {
		return "", false
	}
	// check auth
	if p.HtpasswdFile.Validate(user, passwd) {
		log.Printf("authenticated %s via manual sign in", user)
		return user, true
	}
	return "", false
}

func (p *OauthProxy) GetRedirect(req *http.Request) (string, error) {
	err := req.ParseForm()

	if err != nil {
		return "", err
	}

	redirect := req.FormValue("rd")

	if redirect == "" {
		redirect = "/"
	}

	return redirect, err
}

func (p *OauthProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// check if this is a redirect back at the end of oauth
	remoteAddr := req.RemoteAddr
	if req.Header.Get("X-Real-IP") != "" {
		remoteAddr += fmt.Sprintf(" (%q)", req.Header.Get("X-Real-IP"))
	}
	log.Printf("%s %s %s", remoteAddr, req.Method, req.URL.RequestURI())

	var ok bool
	var user string
	var email string

	if req.URL.Path == pingPath {
		p.PingPage(rw)
		return
	}

	for _, u := range p.compiledRegex {
		match := u.MatchString(req.URL.Path)
		if match {
			p.serveMux.ServeHTTP(rw, req)
			return
		}

	}

	if req.URL.Path == signInPath {
		redirect, err := p.GetRedirect(req)
		if err != nil {
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}

		user, ok = p.ManualSignIn(rw, req)
		if ok {
			p.SetCookie(rw, req, user)
			http.Redirect(rw, req, redirect, 302)
		} else {
			p.SignInPage(rw, req, 200)
		}
		return
	}
	if req.URL.Path == oauthStartPath {
		redirect, err := p.GetRedirect(req)
		if err != nil {
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}
		http.Redirect(rw, req, p.GetLoginURL(redirect), 302)
		return
	}
	if req.URL.Path == oauthCallbackPath {
		// finish the oauth cycle
		err := req.ParseForm()
		if err != nil {
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}
		errorString := req.Form.Get("error")
		if errorString != "" {
			p.ErrorPage(rw, 403, "Permission Denied", errorString)
			return
		}

		_, email, err := p.redeemCode(req.Form.Get("code"))
		if err != nil {
			log.Printf("%s error redeeming code %s", remoteAddr, err)
			p.ErrorPage(rw, 500, "Internal Error", err.Error())
			return
		}

		redirect := req.Form.Get("state")
		if redirect == "" {
			redirect = "/"
		}

		// set cookie, or deny
		if p.Validator(email) {
			log.Printf("%s authenticating %s completed", remoteAddr, email)
			p.SetCookie(rw, req, email)
			http.Redirect(rw, req, redirect, 302)
			return
		} else {
			p.ErrorPage(rw, 403, "Permission Denied", "Invalid Account")
			return
		}
	}

	if !ok {
		cookie, err := req.Cookie(p.CookieKey)
		if err == nil {
			email, ok = validateCookie(cookie, p.CookieSeed)
			user = strings.Split(email, "@")[0]
		}
	}

	if !ok {
		user, ok = p.CheckBasicAuth(req)
		// if we want to promote basic auth requests to cookie'd requests, we could do that here
		// not sure that would be ideal in all circumstances though
		// if ok {
		// 	p.SetCookie(rw, req, user)
		// }
	}

	if !ok {
		log.Printf("%s - invalid cookie session", remoteAddr)
		p.SignInPage(rw, req, 403)
		return
	}

	// At this point, the user is authenticated. proxy normally
	if p.PassBasicAuth {
		req.SetBasicAuth(user, "")
		req.Header["X-Forwarded-User"] = []string{user}
		req.Header["X-Forwarded-Email"] = []string{email}
	}

	p.serveMux.ServeHTTP(rw, req)
}

func (p *OauthProxy) CheckBasicAuth(req *http.Request) (string, bool) {
	if p.HtpasswdFile == nil {
		return "", false
	}
	s := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Basic" {
		return "", false
	}
	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return "", false
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return "", false
	}
	if p.HtpasswdFile.Validate(pair[0], pair[1]) {
		log.Printf("authenticated %s via basic auth", pair[0])
		return pair[0], true
	}
	return "", false
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package redfish

import (
	"encoding/json"

	"github.com/stmcginnis/gofish/school/common"
)

// CompositionService defines a Composition Service which represents the
// properties for the service and links to the resources available for
// composition.
type CompositionService struct {
	common.Entity
	Status         common.Status
	ServiceEnabled string
	resourceBlocks string
	resourceZones  string
}

// UnmarshalJSON unmarshals CompositionService object from the raw JSON.
func (cs *CompositionService) UnmarshalJSON(b []byte) error {
	type temp CompositionService
	var t struct {
		temp
		ResourceBlocks common.Link
		ResourceZones  common.Link
	}

	err := json.Unmarshal(b, &t)
	if err != nil {
		return err
	}

	*cs = CompositionService(t.temp)

	// Extract the links to other entities for later
	cs.resourceBlocks = string(t.ResourceBlocks)
	cs.resourceZones = string(t.ResourceZones)

	return nil
}

// GetCompositionService will get a Composition instance from the Redfish service.
func GetCompositionService(c common.Client, uri string) (*CompositionService, error) {
	resp, err := c.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var t CompositionService
	err = json.NewDecoder(resp.Body).Decode(&t)
	if err != nil {
		return nil, err
	}

	return &t, nil
}
<commit_msg>Fix definition of CompositionService object<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package redfish

import (
	"encoding/json"

	"github.com/stmcginnis/gofish/school/common"
)

// CompositionService is used to represent the Composition Service
// Properties for a Redfish implementation.
type CompositionService struct {
	common.Entity

	// ODataContext is the odata context.
	ODataContext string `json:"@odata.context"`
	// ODataEtag is the odata etag.
	ODataEtag string `json:"@odata.etag"`
	// ODataID is the odata identifier.
	ODataID string `json:"@odata.id"`
	// ODataType is the odata type.
	ODataType string `json:"@odata.type"`
	// AllowOverprovisioning shall be a boolean indicating whether this service
	// is allowed to overprovision a composition relative to the composition request.
	AllowOverprovisioning bool
	// AllowZoneAffinity shall be a boolean indicating whether a client is
	// allowed to request that given composition request is fulfilled by a
	// specified Resource Zone.
	AllowZoneAffinity bool
	// Description provides a description of this resource.
	Description string
	// resourceBlocks shall contain the link to a collection of type ResourceBlockCollection.
	resourceBlocks string
	// resourceZones shall contain the link to a collection of type ZoneCollection.
	resourceZones string
	// ServiceEnabled shall be a boolean indicating whether this service is enabled.
	ServiceEnabled bool
	// Status shall contain any status or health properties of the resource.
	Status common.Status
}

// UnmarshalJSON unmarshals CompositionService object from the raw JSON.
func (cs *CompositionService) UnmarshalJSON(b []byte) error {
	type temp CompositionService
	var t struct {
		temp
		ResourceBlocks common.Link
		ResourceZones  common.Link
	}

	err := json.Unmarshal(b, &t)
	if err != nil {
		return err
	}

	// Extract the links to other entities for later
	*cs = CompositionService(t.temp)
	cs.resourceBlocks = string(t.ResourceBlocks)
	cs.resourceZones = string(t.ResourceZones)

	return nil
}

// GetCompositionService will get a CompositionService instance from the service.
func GetCompositionService(c common.Client, uri string) (*CompositionService, error) {
	resp, err := c.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var compositionservice CompositionService
	err = json.NewDecoder(resp.Body).Decode(&compositionservice)
	if err != nil {
		return nil, err
	}

	compositionservice.SetClient(c)
	return &compositionservice, nil
}

// ListReferencedCompositionServices gets the collection of CompositionService from
// a provided reference.
func ListReferencedCompositionServices(c common.Client, link string) ([]*CompositionService, error) {
	var result []*CompositionService
	if link == "" {
		return result, nil
	}

	links, err := common.GetCollection(c, link)
	if err != nil {
		return result, err
	}

	for _, compositionserviceLink := range links.ItemLinks {
		compositionservice, err := GetCompositionService(c, compositionserviceLink)
		if err != nil {
			return result, err
		}
		result = append(result, compositionservice)
	}

	return result, nil
}
<|endoftext|>


<commit_before>package http

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/loadimpact/speedboat/sampler"
	"github.com/valyala/fasthttp"
	"golang.org/x/net/context"
	neturl "net/url"
	"time"
)

type ContextKey int

const (
	clientKey = ContextKey(iota)
)

var ErrNoClient = errors.New("No client in context")

var mDuration *sampler.Metric
var mErrors *sampler.Metric

func init() {
	mDuration = sampler.Stats("request.duration")
	mErrors = sampler.Counter("request.error")
}

type Args struct {
	Quiet   bool              `json:"quiet"`
	Headers map[string]string `json:"headers"`
}

type Response struct {
	Status  int               `json:"status"`
	Body    string            `json:"body"`
	Headers map[string]string `json:"headers"`
}

func WithDefaultClient(ctx context.Context) context.Context {
	return WithClient(ctx, &fasthttp.Client{})
}

func WithClient(ctx context.Context, c *fasthttp.Client) context.Context {
	return context.WithValue(ctx, clientKey, c)
}

func GetClient(ctx context.Context) *fasthttp.Client {
	return ctx.Value(clientKey).(*fasthttp.Client)
}

func Request(ctx context.Context, method, url, body string, args Args) (Response, error) {
	client := GetClient(ctx)
	if client == nil {
		return Response{}, ErrNoClient
	}

	if method == "GET" && body != "" {
		u, err := neturl.Parse(url)
		if err != nil {
			return Response{}, err
		}

		var params map[string]interface{}
		if err = json.Unmarshal([]byte(body), &params); err != nil {
			return Response{}, err
		}

		q := u.Query()
		for key, val := range params {
			q.Set(key, fmt.Sprint(val))
		}
		u.RawQuery = q.Encode()
		url = u.String()
	}

	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethod(method)
	req.SetRequestURI(url)

	if method != "GET" {
		req.SetBodyString(body)
	}

	for key, value := range args.Headers {
		req.Header.Set(key, value)
	}

	res := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(res)

	startTime := time.Now()
	err := client.Do(req, res)
	duration := time.Since(startTime)

	if !args.Quiet {
		mDuration.WithFields(sampler.Fields{
			"url":    url,
			"method": method,
			"status": res.StatusCode(),
		}).Duration(duration)
	}

	if err != nil {
		if !args.Quiet {
			mErrors.WithFields(sampler.Fields{
				"url":    url,
				"method": method,
				"error":  err,
			}).Int(1)
		}
		return Response{}, err
	}

	resHeaders := make(map[string]string)
	res.Header.VisitAll(func(key, value []byte) {
		resHeaders[string(key)] = string(value)
	})

	return Response{
		Status:  res.StatusCode(),
		Body:    string(res.Body()),
		Headers: resHeaders,
	}, nil
}
<commit_msg>[experiment] Default to unlimited conns/host<commit_after>package http

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/loadimpact/speedboat/sampler"
	"github.com/valyala/fasthttp"
	"golang.org/x/net/context"
	"math"
	neturl "net/url"
	"time"
)

type ContextKey int

const (
	clientKey = ContextKey(iota)
)

var ErrNoClient = errors.New("No client in context")

var mDuration *sampler.Metric
var mErrors *sampler.Metric

func init() {
	mDuration = sampler.Stats("request.duration")
	mErrors = sampler.Counter("request.error")
}

type Args struct {
	Quiet   bool              `json:"quiet"`
	Headers map[string]string `json:"headers"`
}

type Response struct {
	Status  int               `json:"status"`
	Body    string            `json:"body"`
	Headers map[string]string `json:"headers"`
}

func WithDefaultClient(ctx context.Context) context.Context {
	return WithClient(ctx, &fasthttp.Client{
		MaxConnsPerHost: math.MaxInt32,
	})
}

func WithClient(ctx context.Context, c *fasthttp.Client) context.Context {
	return context.WithValue(ctx, clientKey, c)
}

func GetClient(ctx context.Context) *fasthttp.Client {
	return ctx.Value(clientKey).(*fasthttp.Client)
}

func Request(ctx context.Context, method, url, body string, args Args) (Response, error) {
	client := GetClient(ctx)
	if client == nil {
		return Response{}, ErrNoClient
	}

	if method == "GET" && body != "" {
		u, err := neturl.Parse(url)
		if err != nil {
			return Response{}, err
		}

		var params map[string]interface{}
		if err = json.Unmarshal([]byte(body), &params); err != nil {
			return Response{}, err
		}

		q := u.Query()
		for key, val := range params {
			q.Set(key, fmt.Sprint(val))
		}
		u.RawQuery = q.Encode()
		url = u.String()
	}

	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethod(method)
	req.SetRequestURI(url)

	if method != "GET" {
		req.SetBodyString(body)
	}

	for key, value := range args.Headers {
		req.Header.Set(key, value)
	}

	res := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(res)

	startTime := time.Now()
	err := client.Do(req, res)
	duration := time.Since(startTime)

	if !args.Quiet {
		mDuration.WithFields(sampler.Fields{
			"url":    url,
			"method": method,
			"status": res.StatusCode(),
		}).Duration(duration)
	}

	if err != nil {
		if !args.Quiet {
			mErrors.WithFields(sampler.Fields{
				"url":    url,
				"method": method,
				"error":  err,
			}).Int(1)
		}
		return Response{}, err
	}

	resHeaders := make(map[string]string)
	res.Header.VisitAll(func(key, value []byte) {
		resHeaders[string(key)] = string(value)
	})

	return Response{
		Status:  res.StatusCode(),
		Body:    string(res.Body()),
		Headers: resHeaders,
	}, nil
}
<|endoftext|>


<commit_before><commit_msg>Regenerate github fakes<commit_after><|endoftext|>