<commit_before>// package idletiming provides mechanisms for adding idle timeouts to net.Conn
// and net.Listener.
package idletiming

import (
	"errors"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
)

var (
	log = golog.LoggerFor("idletiming")

	// ErrIdled is return when attempting to use a network connection that was
	// closed because of idling.
	ErrIdled = errors.New("Use of idled network connection")
)

// Conn creates a new net.Conn wrapping the given net.Conn that times out after
// the specified period. Once a connection has timed out, any pending reads or
// writes will return io.EOF and the underlying connection will be closed.
//
// idleTimeout specifies how long to wait for inactivity before considering
// connection idle.
//
// If onIdle is specified, it will be called to indicate when the connection has
// idled and been closed.
func Conn(conn net.Conn, idleTimeout time.Duration, onIdle func()) *IdleTimingConn {
	c := &IdleTimingConn{
		conn:             conn,
		idleTimeout:      idleTimeout,
		halfIdleTimeout:  time.Duration(idleTimeout.Nanoseconds() / 2),
		activeCh:         make(chan bool, 1),
		closedCh:         make(chan bool, 1),
		lastActivityTime: int64(time.Now().UnixNano()),
	}

	go func() {
		timer := time.NewTimer(idleTimeout)
		defer timer.Stop()
		for {
			select {
			case <-c.activeCh:
				// We're active, continue
				timer.Reset(idleTimeout)
				atomic.StoreInt64(&c.lastActivityTime, time.Now().UnixNano())
				continue
			case <-timer.C:
				c.Close()
				if onIdle != nil {
					onIdle()
				}
				return
			case <-c.closedCh:
				return
			}
		}
	}()

	return c
}

// IdleTimingConn is a net.Conn that wraps another net.Conn and that times out
// if idle for more than idleTimeout.
type IdleTimingConn struct {
	// Keep it at the top to make sure 64-bit alignment, see
	// https://golang.org/pkg/sync/atomic/#pkg-note-BUG
	lastActivityTime int64
	readDeadline     guardedTime
	writeDeadline    guardedTime

	conn             net.Conn
	idleTimeout      time.Duration
	halfIdleTimeout  time.Duration
	activeCh         chan bool
	closedCh         chan bool
	closeMutex       sync.RWMutex // prevents Close() from interfering with io operations
	closed           bool
	hasReadAfterIdle int32
}

// TimesOutIn returns how much time is left before this connection will time
// out, assuming there is no further activity.
func (c *IdleTimingConn) TimesOutIn() time.Duration {
	return c.TimesOutAt().Sub(time.Now())
}

// TimesOutAt returns the time at which this connection will time out, assuming
// there is no further activity
func (c *IdleTimingConn) TimesOutAt() time.Time {
	return time.Unix(0, atomic.LoadInt64(&c.lastActivityTime)).Add(c.idleTimeout)
}

// Read implements the method from io.Reader
func (c *IdleTimingConn) Read(b []byte) (int, error) {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosedFirstTime(&c.hasReadAfterIdle, io.EOF); err != nil {
		return 0, err
	}

	totalN := 0
	readDeadline := c.readDeadline.Get()

	// Continually read while we can, always setting a deadline that's less than
	// our idleTimeout so that we can update our active status before we hit the
	// idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !readDeadline.IsZero() && !maxDeadline.Before(readDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetReadDeadline(readDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetReadDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Write implements the method from io.Reader
func (c *IdleTimingConn) Write(b []byte) (int, error) {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return 0, err
	}

	totalN := 0
	writeDeadline := c.writeDeadline.Get()

	// Continually write while we can, always setting a deadline that's less
	// than our idleTimeout so that we can update our active status before we
	// hit the idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !writeDeadline.IsZero() && !maxDeadline.Before(writeDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetWriteDeadline(writeDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetWriteDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Close this IdleTimingConn. This will close the underlying net.Conn as well,
// returning the error from calling its Close method.
func (c *IdleTimingConn) Close() error {
	c.closeMutex.Lock()
	defer c.closeMutex.Unlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.closed = true

	select {
	case c.closedCh <- true:
		// close accepted
	default:
		// already closing, ignore
	}
	return c.conn.Close()
}

func (c *IdleTimingConn) LocalAddr() net.Addr {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	return c.conn.LocalAddr()
}

func (c *IdleTimingConn) RemoteAddr() net.Addr {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	return c.conn.RemoteAddr()
}

func (c *IdleTimingConn) SetDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	if err := c.SetReadDeadline(t); err != nil {
		log.Tracef("Unable to set read deadline: %v", err)
	}
	if err := c.SetWriteDeadline(t); err != nil {
		log.Tracef("Unable to set write deadline: %v", err)
	}
	return nil
}

func (c *IdleTimingConn) SetReadDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.readDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) SetWriteDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.writeDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) markActive(n int) bool {
	if n > 0 {
		select {
		case c.activeCh <- true:
			// ok
		default:
			// still waiting to process previous markActive
		}
		return true
	}
	return false
}

func (c *IdleTimingConn) checkClosed() error {
	return c.checkClosedFirstTime(nil, nil)
}

func (c *IdleTimingConn) checkClosedFirstTime(hasDone *int32, firstTimeError error) error {
	if c.closed {
		if hasDone != nil && atomic.CompareAndSwapInt32(hasDone, 0, 1) {
			return firstTimeError
		}
		return ErrIdled
	}
	return nil
}

func isTimeout(err error) bool {
	if netErr, ok := err.(net.Error); ok {
		return netErr.Timeout()
	}
	return false
}

type guardedTime struct {
	sync.RWMutex
	t time.Time
}

func (g *guardedTime) Get() time.Time {
	g.RLock()
	retval := g.t
	g.RUnlock()
	return retval
}

func (g *guardedTime) Set(t time.Time) {
	g.Lock()
	g.t = t
	g.Unlock()
}
<commit_msg>Using mtime to measure elapsed time<commit_after>// package idletiming provides mechanisms for adding idle timeouts to net.Conn
// and net.Listener.
package idletiming

import (
	"errors"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/mtime"
)

var (
	log = golog.LoggerFor("idletiming")

	// ErrIdled is return when attempting to use a network connection that was
	// closed because of idling.
	ErrIdled = errors.New("Use of idled network connection")
)

// Conn creates a new net.Conn wrapping the given net.Conn that times out after
// the specified period. Once a connection has timed out, any pending reads or
// writes will return io.EOF and the underlying connection will be closed.
//
// idleTimeout specifies how long to wait for inactivity before considering
// connection idle.
//
// If onIdle is specified, it will be called to indicate when the connection has
// idled and been closed.
func Conn(conn net.Conn, idleTimeout time.Duration, onIdle func()) *IdleTimingConn {
	c := &IdleTimingConn{
		conn:             conn,
		idleTimeout:      idleTimeout,
		halfIdleTimeout:  time.Duration(idleTimeout.Nanoseconds() / 2),
		activeCh:         make(chan bool, 1),
		closedCh:         make(chan bool, 1),
		lastActivityTime: uint64(mtime.Now()),
	}

	go func() {
		timer := time.NewTimer(idleTimeout)
		defer timer.Stop()
		for {
			select {
			case <-c.activeCh:
				// We're active, continue
				timer.Reset(idleTimeout)
				atomic.StoreUint64(&c.lastActivityTime, uint64(mtime.Now()))
				continue
			case <-timer.C:
				c.Close()
				if onIdle != nil {
					onIdle()
				}
				return
			case <-c.closedCh:
				return
			}
		}
	}()

	return c
}

// IdleTimingConn is a net.Conn that wraps another net.Conn and that times out
// if idle for more than idleTimeout.
type IdleTimingConn struct {
	// Keep it at the top to make sure 64-bit alignment, see
	// https://golang.org/pkg/sync/atomic/#pkg-note-BUG
	lastActivityTime uint64
	readDeadline     guardedTime
	writeDeadline    guardedTime

	conn             net.Conn
	idleTimeout      time.Duration
	halfIdleTimeout  time.Duration
	activeCh         chan bool
	closedCh         chan bool
	closeMutex       sync.RWMutex // prevents Close() from interfering with io operations
	closed           bool
	hasReadAfterIdle int32
}

// TimesOutIn returns how much time is left before this connection will time
// out, assuming there is no further activity.
func (c *IdleTimingConn) TimesOutIn() time.Duration {
	return c.idleTimeout - mtime.Now().Sub(mtime.Instant(atomic.LoadUint64(&c.lastActivityTime)))
}

// Read implements the method from io.Reader
func (c *IdleTimingConn) Read(b []byte) (int, error) {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosedFirstTime(&c.hasReadAfterIdle, io.EOF); err != nil {
		return 0, err
	}

	totalN := 0
	readDeadline := c.readDeadline.Get()

	// Continually read while we can, always setting a deadline that's less than
	// our idleTimeout so that we can update our active status before we hit the
	// idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !readDeadline.IsZero() && !maxDeadline.Before(readDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetReadDeadline(readDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetReadDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Write implements the method from io.Reader
func (c *IdleTimingConn) Write(b []byte) (int, error) {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return 0, err
	}

	totalN := 0
	writeDeadline := c.writeDeadline.Get()

	// Continually write while we can, always setting a deadline that's less
	// than our idleTimeout so that we can update our active status before we
	// hit the idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !writeDeadline.IsZero() && !maxDeadline.Before(writeDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetWriteDeadline(writeDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetWriteDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Close this IdleTimingConn. This will close the underlying net.Conn as well,
// returning the error from calling its Close method.
func (c *IdleTimingConn) Close() error {
	c.closeMutex.Lock()
	defer c.closeMutex.Unlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.closed = true

	select {
	case c.closedCh <- true:
		// close accepted
	default:
		// already closing, ignore
	}
	return c.conn.Close()
}

func (c *IdleTimingConn) LocalAddr() net.Addr {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	return c.conn.LocalAddr()
}

func (c *IdleTimingConn) RemoteAddr() net.Addr {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	return c.conn.RemoteAddr()
}

func (c *IdleTimingConn) SetDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	if err := c.SetReadDeadline(t); err != nil {
		log.Tracef("Unable to set read deadline: %v", err)
	}
	if err := c.SetWriteDeadline(t); err != nil {
		log.Tracef("Unable to set write deadline: %v", err)
	}
	return nil
}

func (c *IdleTimingConn) SetReadDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.readDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) SetWriteDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.writeDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) markActive(n int) bool {
	if n > 0 {
		select {
		case c.activeCh <- true:
			// ok
		default:
			// still waiting to process previous markActive
		}
		return true
	}
	return false
}

func (c *IdleTimingConn) checkClosed() error {
	return c.checkClosedFirstTime(nil, nil)
}

func (c *IdleTimingConn) checkClosedFirstTime(hasDone *int32, firstTimeError error) error {
	if c.closed {
		if hasDone != nil && atomic.CompareAndSwapInt32(hasDone, 0, 1) {
			return firstTimeError
		}
		return ErrIdled
	}
	return nil
}

func isTimeout(err error) bool {
	if netErr, ok := err.(net.Error); ok {
		return netErr.Timeout()
	}
	return false
}

type guardedTime struct {
	sync.RWMutex
	t time.Time
}

func (g *guardedTime) Get() time.Time {
	g.RLock()
	retval := g.t
	g.RUnlock()
	return retval
}

func (g *guardedTime) Set(t time.Time) {
	g.Lock()
	g.t = t
	g.Unlock()
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type podLatencyData struct {
	Name    string
	Latency time.Duration
}

type latencySlice []podLatencyData

func (a latencySlice) Len() int           { return len(a) }
func (a latencySlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a latencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency }

func printLatencies(latencies []podLatencyData, header string) {
	perc50 := latencies[len(latencies)/2].Latency
	perc90 := latencies[(len(latencies)*9)/10].Latency
	perc99 := latencies[(len(latencies)*99)/100].Latency
	Logf("10%% %s: %v", header, latencies[(len(latencies)*9)/10:len(latencies)])
	Logf("perc50: %v, perc90: %v, perc99: %v", perc50, perc90, perc99)
}

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var additionalRCName string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		nsForTesting, err := createTestingNS("density", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())

		// Print latency metrics before the test.
		// TODO: Remove this once we reset metrics before the test.
		_, err = HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)

		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		rc, err = c.ReplicationControllers(ns).Get(additionalRCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, additionalRCName)
			expectNoError(err)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		// TODO: We should reset metrics before the test. Currently previous tests influence latency metrics.
		highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests")
	})

	// Tests with "Skipped" substring in their name will be skipped when running
	// e2e test suite without --ginkgo.focus & --ginkgo.skip flags.
	type Density struct {
		skip bool
		// Controls if e2e latency tests should be run (they are slow)
		runLatencyTest bool
		podsPerMinion  int
		// Controls how often the apiserver is polled for pods
		interval time.Duration
	}

	densityTests := []Density{
		// This test should not be run in a regular jenkins run, because it is not isolated enough
		// (metrics from other tests affects this one).
		// TODO: Reenable once we can measure latency only from a single test.
		// TODO: Expose runLatencyTest as ginkgo flag.
		{podsPerMinion: 3, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 30, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		// More than 30 pods per node is outside our v1.0 goals.
		// We might want to enable those tests in the future.
		{podsPerMinion: 50, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 100, skip: true, runLatencyTest: false, interval: 1 * time.Second},
	}

	for _, testArg := range densityTests {
		name := fmt.Sprintf("should allow starting %d pods per node", testArg.podsPerMinion)
		if testArg.podsPerMinion <= 30 {
			name = "[Performance suite] " + name
		}
		if testArg.skip {
			name = "[Skipped] " + name
		}
		itArg := testArg
		It(name, func() {
			totalPods := itArg.podsPerMinion * minionCount
			RCName = "density" + strconv.Itoa(totalPods) + "-" + uuid
			fileHndl, err := os.Create(fmt.Sprintf(testContext.OutputDir+"/%s/pod_states.csv", uuid))
			expectNoError(err)
			defer fileHndl.Close()

			config := RCConfig{Client: c,
				Image:         "gcr.io/google_containers/pause:go",
				Name:          RCName,
				Namespace:     ns,
				PollInterval:  itArg.interval,
				PodStatusFile: fileHndl,
				Replicas:      totalPods,
			}

			// Create a listener for events.
			events := make([](*api.Event), 0)
			_, controller := framework.NewInformer(
				&cache.ListWatch{
					ListFunc: func() (runtime.Object, error) {
						return c.Events(ns).List(labels.Everything(), fields.Everything())
					},
					WatchFunc: func(rv string) (watch.Interface, error) {
						return c.Events(ns).Watch(labels.Everything(), fields.Everything(), rv)
					},
				},
				&api.Event{},
				0,
				framework.ResourceEventHandlerFuncs{
					AddFunc: func(obj interface{}) {
						events = append(events, obj.(*api.Event))
					},
				},
			)
			stop := make(chan struct{})
			go controller.Run(stop)

			// Start the replication controller.
			startTime := time.Now()
			expectNoError(RunRC(config))
			e2eStartupTime := time.Now().Sub(startTime)
			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)

			By("Waiting for all events to be recorded")
			last := -1
			current := len(events)
			timeout := 10 * time.Minute
			for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				last = current
				current = len(events)
			}
			close(stop)

			if current != last {
				Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes())
			}
			Logf("Found %d events", current)

			// Tune the threshold for allowed failures.
			badEvents := BadEvents(events)
			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))

			if itArg.runLatencyTest {
				Logf("Schedling additional Pods to measure startup latencies")

				createTimes := make(map[string]util.Time, 0)
				scheduleTimes := make(map[string]util.Time, 0)
				runTimes := make(map[string]util.Time, 0)
				watchTimes := make(map[string]util.Time, 0)

				var mutex sync.Mutex
				checkPod := func(p *api.Pod) {
					mutex.Lock()
					defer mutex.Unlock()
					defer GinkgoRecover()

					if p.Status.Phase == api.PodRunning {
						if _, found := watchTimes[p.Name]; !found {
							watchTimes[p.Name] = util.Now()
							createTimes[p.Name] = p.CreationTimestamp
							var startTime util.Time
							for _, cs := range p.Status.ContainerStatuses {
								if cs.State.Running != nil {
									if startTime.Before(cs.State.Running.StartedAt) {
										startTime = cs.State.Running.StartedAt
									}
								}
							}
							if startTime != util.NewTime(time.Time{}) {
								runTimes[p.Name] = startTime
							} else {
								Failf("Pod %v is reported to be running, but none of its containers is", p.Name)
							}
						}
					}
				}

				additionalNameStr := strconv.Itoa(minionCount) + "-" + string(util.NewUUID())
				additionalRCName = "my-hostname-latency" + additionalNameStr
				_, controller := framework.NewInformer(
					&cache.ListWatch{
						ListFunc: func() (runtime.Object, error) {
							return c.Pods(ns).List(labels.SelectorFromSet(labels.Set{"name": additionalRCName}), fields.Everything())
						},
						WatchFunc: func(rv string) (watch.Interface, error) {
							return c.Pods(ns).Watch(labels.SelectorFromSet(labels.Set{"name": additionalRCName}), fields.Everything(), rv)
						},
					},
					&api.Pod{},
					time.Minute*5,
					framework.ResourceEventHandlerFuncs{
						AddFunc: func(obj interface{}) {
							p, ok := obj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
						UpdateFunc: func(oldObj, newObj interface{}) {
							p, ok := newObj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
					},
				)

				stopCh := make(chan struct{})
				go controller.Run(stopCh)

				config = RCConfig{Client: c,
					Image:        "gcr.io/google_containers/pause:go",
					Name:         additionalRCName,
					Namespace:    ns,
					PollInterval: itArg.interval,
					Replicas:     minionCount,
				}
				expectNoError(RunRC(config))

				Logf("Waiting for all Pods begin observed by the watch...")
				for start := time.Now(); len(watchTimes) < minionCount && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				}
				close(stopCh)

				schedEvents, err := c.Events(ns).List(
					labels.Everything(),
					fields.Set{
						"involvedObject.kind":      "Pod",
						"involvedObject.namespace": ns,
						"source":                   "scheduler",
					}.AsSelector())
				expectNoError(err)
				for k := range createTimes {
					for _, event := range schedEvents.Items {
						if event.InvolvedObject.Name == k {
							scheduleTimes[k] = event.FirstTimestamp
							break
						}
					}
				}

				scheduleLag := make([]podLatencyData, 0)
				startupLag := make([]podLatencyData, 0)
				watchLag := make([]podLatencyData, 0)
				schedToWatchLag := make([]podLatencyData, 0)
				e2eLag := make([]podLatencyData, 0)

				for name, create := range createTimes {
					sched, ok := scheduleTimes[name]
					Expect(ok).To(Equal(true))
					run, ok := runTimes[name]
					Expect(ok).To(Equal(true))
					watch, ok := watchTimes[name]
					Expect(ok).To(Equal(true))
					scheduleLag = append(scheduleLag, podLatencyData{name, sched.Time.Sub(create.Time)})
					startupLag = append(startupLag, podLatencyData{name, run.Time.Sub(sched.Time)})
					watchLag = append(watchLag, podLatencyData{name, watch.Time.Sub(run.Time)})
					schedToWatchLag = append(schedToWatchLag, podLatencyData{name, watch.Time.Sub(sched.Time)})
					e2eLag = append(e2eLag, podLatencyData{name, watch.Time.Sub(create.Time)})
				}

				sort.Sort(latencySlice(scheduleLag))
				sort.Sort(latencySlice(startupLag))
				sort.Sort(latencySlice(watchLag))
				sort.Sort(latencySlice(schedToWatchLag))
				sort.Sort(latencySlice(e2eLag))

				printLatencies(scheduleLag, "worst schedule latencies")
				printLatencies(startupLag, "worst run-after-schedule latencies")
				printLatencies(watchLag, "worst watch latencies")
				printLatencies(schedToWatchLag, "worst scheduled-to-end total latencies")
				printLatencies(e2eLag, "worst e2e total latencies")

				Logf("Approx throughput: %v pods/min",
					float64(minionCount)/(e2eLag[len(e2eLag)-1].Latency.Minutes()))
			}
		})
	}
})
<commit_msg>Change density relist period to better reflect the kubelet<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type podLatencyData struct {
	Name    string
	Latency time.Duration
}

type latencySlice []podLatencyData

func (a latencySlice) Len() int           { return len(a) }
func (a latencySlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a latencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency }

func printLatencies(latencies []podLatencyData, header string) {
	perc50 := latencies[len(latencies)/2].Latency
	perc90 := latencies[(len(latencies)*9)/10].Latency
	perc99 := latencies[(len(latencies)*99)/100].Latency
	Logf("10%% %s: %v", header, latencies[(len(latencies)*9)/10:len(latencies)])
	Logf("perc50: %v, perc90: %v, perc99: %v", perc50, perc90, perc99)
}

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var additionalRCName string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		nsForTesting, err := createTestingNS("density", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())

		// Print latency metrics before the test.
		// TODO: Remove this once we reset metrics before the test.
		_, err = HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)

		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		rc, err = c.ReplicationControllers(ns).Get(additionalRCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, additionalRCName)
			expectNoError(err)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		// TODO: We should reset metrics before the test. Currently previous tests influence latency metrics.
		highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests")
	})

	// Tests with "Skipped" substring in their name will be skipped when running
	// e2e test suite without --ginkgo.focus & --ginkgo.skip flags.
	type Density struct {
		skip bool
		// Controls if e2e latency tests should be run (they are slow)
		runLatencyTest bool
		podsPerMinion  int
		// Controls how often the apiserver is polled for pods
		interval time.Duration
	}

	densityTests := []Density{
		// This test should not be run in a regular jenkins run, because it is not isolated enough
		// (metrics from other tests affects this one).
		// TODO: Reenable once we can measure latency only from a single test.
		// TODO: Expose runLatencyTest as ginkgo flag.
		{podsPerMinion: 3, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 30, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		// More than 30 pods per node is outside our v1.0 goals.
		// We might want to enable those tests in the future.
		{podsPerMinion: 50, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 100, skip: true, runLatencyTest: false, interval: 1 * time.Second},
	}

	for _, testArg := range densityTests {
		name := fmt.Sprintf("should allow starting %d pods per node", testArg.podsPerMinion)
		if testArg.podsPerMinion <= 30 {
			name = "[Performance suite] " + name
		}
		if testArg.skip {
			name = "[Skipped] " + name
		}
		itArg := testArg
		It(name, func() {
			totalPods := itArg.podsPerMinion * minionCount
			RCName = "density" + strconv.Itoa(totalPods) + "-" + uuid
			fileHndl, err := os.Create(fmt.Sprintf(testContext.OutputDir+"/%s/pod_states.csv", uuid))
			expectNoError(err)
			defer fileHndl.Close()

			config := RCConfig{Client: c,
				Image:         "gcr.io/google_containers/pause:go",
				Name:          RCName,
				Namespace:     ns,
				PollInterval:  itArg.interval,
				PodStatusFile: fileHndl,
				Replicas:      totalPods,
			}

			// Create a listener for events.
			events := make([](*api.Event), 0)
			_, controller := framework.NewInformer(
				&cache.ListWatch{
					ListFunc: func() (runtime.Object, error) {
						return c.Events(ns).List(labels.Everything(), fields.Everything())
					},
					WatchFunc: func(rv string) (watch.Interface, error) {
						return c.Events(ns).Watch(labels.Everything(), fields.Everything(), rv)
					},
				},
				&api.Event{},
				0,
				framework.ResourceEventHandlerFuncs{
					AddFunc: func(obj interface{}) {
						events = append(events, obj.(*api.Event))
					},
				},
			)
			stop := make(chan struct{})
			go controller.Run(stop)

			// Start the replication controller.
			startTime := time.Now()
			expectNoError(RunRC(config))
			e2eStartupTime := time.Now().Sub(startTime)
			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)

			By("Waiting for all events to be recorded")
			last := -1
			current := len(events)
			timeout := 10 * time.Minute
			for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				last = current
				current = len(events)
			}
			close(stop)

			if current != last {
				Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes())
			}
			Logf("Found %d events", current)

			// Tune the threshold for allowed failures.
			badEvents := BadEvents(events)
			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))

			if itArg.runLatencyTest {
				Logf("Schedling additional Pods to measure startup latencies")

				createTimes := make(map[string]util.Time, 0)
				scheduleTimes := make(map[string]util.Time, 0)
				runTimes := make(map[string]util.Time, 0)
				watchTimes := make(map[string]util.Time, 0)

				var mutex sync.Mutex
				checkPod := func(p *api.Pod) {
					mutex.Lock()
					defer mutex.Unlock()
					defer GinkgoRecover()

					if p.Status.Phase == api.PodRunning {
						if _, found := watchTimes[p.Name]; !found {
							watchTimes[p.Name] = util.Now()
							createTimes[p.Name] = p.CreationTimestamp
							var startTime util.Time
							for _, cs := range p.Status.ContainerStatuses {
								if cs.State.Running != nil {
									if startTime.Before(cs.State.Running.StartedAt) {
										startTime = cs.State.Running.StartedAt
									}
								}
							}
							if startTime != util.NewTime(time.Time{}) {
								runTimes[p.Name] = startTime
							} else {
								Failf("Pod %v is reported to be running, but none of its containers is", p.Name)
							}
						}
					}
				}

				additionalNameStr := strconv.Itoa(minionCount) + "-" + string(util.NewUUID())
				additionalRCName = "my-hostname-latency" + additionalNameStr
				_, controller := framework.NewInformer(
					&cache.ListWatch{
						ListFunc: func() (runtime.Object, error) {
							return c.Pods(ns).List(labels.SelectorFromSet(labels.Set{"name": additionalRCName}), fields.Everything())
						},
						WatchFunc: func(rv string) (watch.Interface, error) {
							return c.Pods(ns).Watch(labels.SelectorFromSet(labels.Set{"name": additionalRCName}), fields.Everything(), rv)
						},
					},
					&api.Pod{},
					0,
					framework.ResourceEventHandlerFuncs{
						AddFunc: func(obj interface{}) {
							p, ok := obj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
						UpdateFunc: func(oldObj, newObj interface{}) {
							p, ok := newObj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
					},
				)

				stopCh := make(chan struct{})
				go controller.Run(stopCh)

				config = RCConfig{Client: c,
					Image:        "gcr.io/google_containers/pause:go",
					Name:         additionalRCName,
					Namespace:    ns,
					PollInterval: itArg.interval,
					Replicas:     minionCount,
				}
				expectNoError(RunRC(config))

				Logf("Waiting for all Pods begin observed by the watch...")
				for start := time.Now(); len(watchTimes) < minionCount && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				}
				close(stopCh)

				schedEvents, err := c.Events(ns).List(
					labels.Everything(),
					fields.Set{
						"involvedObject.kind":      "Pod",
						"involvedObject.namespace": ns,
						"source":                   "scheduler",
					}.AsSelector())
				expectNoError(err)
				for k := range createTimes {
					for _, event := range schedEvents.Items {
						if event.InvolvedObject.Name == k {
							scheduleTimes[k] = event.FirstTimestamp
							break
						}
					}
				}

				scheduleLag := make([]podLatencyData, 0)
				startupLag := make([]podLatencyData, 0)
				watchLag := make([]podLatencyData, 0)
				schedToWatchLag := make([]podLatencyData, 0)
				e2eLag := make([]podLatencyData, 0)

				for name, create := range createTimes {
					sched, ok := scheduleTimes[name]
					Expect(ok).To(Equal(true))
					run, ok := runTimes[name]
					Expect(ok).To(Equal(true))
					watch, ok := watchTimes[name]
					Expect(ok).To(Equal(true))
					scheduleLag = append(scheduleLag, podLatencyData{name, sched.Time.Sub(create.Time)})
					startupLag = append(startupLag, podLatencyData{name, run.Time.Sub(sched.Time)})
					watchLag = append(watchLag, podLatencyData{name, watch.Time.Sub(run.Time)})
					schedToWatchLag = append(schedToWatchLag, podLatencyData{name, watch.Time.Sub(sched.Time)})
					e2eLag = append(e2eLag, podLatencyData{name, watch.Time.Sub(create.Time)})
				}

				sort.Sort(latencySlice(scheduleLag))
				sort.Sort(latencySlice(startupLag))
				sort.Sort(latencySlice(watchLag))
				sort.Sort(latencySlice(schedToWatchLag))
				sort.Sort(latencySlice(e2eLag))

				printLatencies(scheduleLag, "worst schedule latencies")
				printLatencies(startupLag, "worst run-after-schedule latencies")
				printLatencies(watchLag, "worst watch latencies")
				printLatencies(schedToWatchLag, "worst scheduled-to-end total latencies")
				printLatencies(e2eLag, "worst e2e total latencies")

				Logf("Approx throughput: %v pods/min",
					float64(minionCount)/(e2eLag[len(e2eLag)-1].Latency.Minutes()))
			}
		})
	}
})
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build cgo

// Even though this file requires no C, it is used to provide a
// listGroup stub because all the other Solaris calls work.  Otherwise,
// this stub will conflict with the lookup_stubs.go fallback.

package user

import "fmt"

func listGroups(u *User) ([]string, error) {
	return nil, fmt.Errorf("user: list groups for %s: not supported on Solaris", u.Username)
}
<commit_msg>os/user: fix osusergo build on Solaris<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build cgo,!osusergo

// Even though this file requires no C, it is used to provide a
// listGroup stub because all the other Solaris calls work.  Otherwise,
// this stub will conflict with the lookup_stubs.go fallback.

package user

import "fmt"

func listGroups(u *User) ([]string, error) {
	return nil, fmt.Errorf("user: list groups for %s: not supported on Solaris", u.Username)
}
<|endoftext|>


<commit_before>package http

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/BurntSushi/locker"
	"github.com/boltdb/bolt"
	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/cache/metadata"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/tracing"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type Opt struct {
	CacheAccessor cache.Accessor
	MetadataStore *metadata.Store
	Transport     http.RoundTripper
}

type httpSource struct {
	md     *metadata.Store
	cache  cache.Accessor
	locker *locker.Locker
	client *http.Client
}

func NewSource(opt Opt) (source.Source, error) {
	transport := opt.Transport
	if transport == nil {
		transport = tracing.DefaultTransport
	}
	hs := &httpSource{
		md:     opt.MetadataStore,
		cache:  opt.CacheAccessor,
		locker: locker.NewLocker(),
		client: &http.Client{
			Transport: transport,
		},
	}
	return hs, nil
}

func (hs *httpSource) ID() string {
	return source.HttpsScheme
}

type httpSourceHandler struct {
	*httpSource
	src      source.HttpIdentifier
	refID    string
	cacheKey digest.Digest
}

func (hs *httpSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
	httpIdentifier, ok := id.(*source.HttpIdentifier)
	if !ok {
		return nil, errors.Errorf("invalid http identifier %v", id)
	}

	return &httpSourceHandler{
		src:        *httpIdentifier,
		httpSource: hs,
	}, nil
}

// urlHash is internal hash the etag is stored by that doesn't leak outside
// this package.
func (hs *httpSourceHandler) urlHash() (digest.Digest, error) {
	dt, err := json.Marshal(struct {
		Filename       string
		Perm, UID, GID int
	}{
		Filename: getFileName(hs.src.URL, hs.src.Filename, nil),
		Perm:     hs.src.Perm,
		UID:      hs.src.UID,
		GID:      hs.src.GID,
	})
	if err != nil {
		return "", err
	}
	return digest.FromBytes(dt), nil
}

func (hs *httpSourceHandler) formatCacheKey(filename string, dgst digest.Digest) digest.Digest {
	dt, err := json.Marshal(struct {
		Filename       string
		Perm, UID, GID int
		Checksum       digest.Digest
	}{
		Filename: filename,
		Perm:     hs.src.Perm,
		UID:      hs.src.UID,
		GID:      hs.src.GID,
		Checksum: dgst,
	})
	if err != nil {
		return dgst
	}
	return digest.FromBytes(dt)
}

func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, bool, error) {
	if hs.src.Checksum != "" {
		hs.cacheKey = hs.src.Checksum
		return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, nil), hs.src.Checksum).String(), true, nil
	}

	uh, err := hs.urlHash()
	if err != nil {
		return "", false, nil
	}

	// look up metadata(previously stored headers) for that URL
	sis, err := hs.md.Search(uh.String())
	if err != nil {
		return "", false, errors.Wrapf(err, "failed to search metadata for %s", uh)
	}

	req, err := http.NewRequest("GET", hs.src.URL, nil)
	if err != nil {
		return "", false, err
	}
	req = req.WithContext(ctx)
	m := map[string]*metadata.StorageItem{}

	if len(sis) > 0 {
		for _, si := range sis {
			// if metaDigest := getMetaDigest(si); metaDigest == hs.formatCacheKey("") {
			if etag := getETag(si); etag != "" {
				if dgst := getChecksum(si); dgst != "" {
					m[etag] = si
					req.Header.Add("If-None-Match", etag)
				}
			}
			// }
		}
	}

	resp, err := hs.client.Do(req)
	if err != nil {
		return "", false, err
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		return "", false, errors.Errorf("invalid response status %d", resp.StatusCode)
	}
	if resp.StatusCode == http.StatusNotModified {
		respETag := resp.Header.Get("ETag")
		si, ok := m[respETag]
		if !ok {
			return "", false, errors.Errorf("invalid not-modified ETag: %v", respETag)
		}
		hs.refID = si.ID()
		dgst := getChecksum(si)
		if dgst == "" {
			return "", false, errors.Errorf("invalid metadata change")
		}
		resp.Body.Close()
		return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst).String(), true, nil
	}

	ref, dgst, err := hs.save(ctx, resp)
	if err != nil {
		return "", false, err
	}
	ref.Release(context.TODO())

	hs.cacheKey = dgst

	return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst).String(), true, nil
}

func (hs *httpSourceHandler) save(ctx context.Context, resp *http.Response) (ref cache.ImmutableRef, dgst digest.Digest, retErr error) {
	newRef, err := hs.cache.New(ctx, nil, cache.CachePolicyRetain, cache.WithDescription(fmt.Sprintf("http url %s", hs.src.URL)))
	if err != nil {
		return nil, "", err
	}

	releaseRef := func() {
		newRef.Release(context.TODO())
	}

	defer func() {
		if retErr != nil && newRef != nil {
			releaseRef()
		}
	}()

	mount, err := newRef.Mount(ctx, false)
	if err != nil {
		return nil, "", err
	}

	lm := snapshot.LocalMounter(mount)
	dir, err := lm.Mount()
	if err != nil {
		return nil, "", err
	}

	defer func() {
		if retErr != nil && lm != nil {
			lm.Unmount()
		}
	}()
	perm := 0600
	if hs.src.Perm != 0 {
		perm = hs.src.Perm
	}
	fp := filepath.Join(dir, getFileName(hs.src.URL, hs.src.Filename, resp))
	logrus.Debugf("write to  %v", fp)

	f, err := os.OpenFile(fp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(perm))
	if err != nil {
		return nil, "", err
	}
	defer func() {
		if f != nil {
			f.Close()
		}
	}()

	h := sha256.New()

	if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil {
		return nil, "", err
	}

	if err := f.Close(); err != nil {
		return nil, "", err
	}
	f = nil

	if hs.src.UID != 0 || hs.src.GID != 0 {
		if err := os.Chown(fp, hs.src.UID, hs.src.GID); err != nil {
			return nil, "", err
		}
	}

	mTime := time.Unix(0, 0)
	lastMod := resp.Header.Get("Last-Modified")
	if lastMod != "" {
		if parsedMTime, err := http.ParseTime(lastMod); err == nil {
			mTime = parsedMTime
		}
	}

	if err := os.Chtimes(fp, mTime, mTime); err != nil {
		return nil, "", err
	}

	lm.Unmount()
	lm = nil

	ref, err = newRef.Commit(ctx)
	if err != nil {
		return nil, "", err
	}
	newRef = nil

	hs.refID = ref.ID()
	dgst = digest.NewDigest(digest.SHA256, h)

	if respETag := resp.Header.Get("ETag"); respETag != "" {
		setETag(ref.Metadata(), respETag)
		uh, err := hs.urlHash()
		if err != nil {
			return nil, "", err
		}
		setChecksum(ref.Metadata(), uh.String(), dgst)
		if err := ref.Metadata().Commit(); err != nil {
			return nil, "", err
		}
	}

	return ref, dgst, nil
}

func (hs *httpSourceHandler) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
	if hs.refID != "" {
		ref, err := hs.cache.Get(ctx, hs.refID)
		if err == nil {
			return ref, nil
		}
	}

	req, err := http.NewRequest("GET", hs.src.URL, nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)

	resp, err := hs.client.Do(req)
	if err != nil {
		return nil, err
	}

	ref, dgst, err := hs.save(ctx, resp)
	if err != nil {
		return nil, err
	}
	if dgst != hs.cacheKey {
		ref.Release(context.TODO())
		return nil, errors.Errorf("digest mismatch %s: %s", dgst, hs.cacheKey)
	}

	return ref, nil
}

const keyETag = "etag"
const keyChecksum = "http.checksum"

func setETag(si *metadata.StorageItem, s string) error {
	v, err := metadata.NewValue(s)
	if err != nil {
		return errors.Wrap(err, "failed to create etag value")
	}
	si.Queue(func(b *bolt.Bucket) error {
		return si.SetValue(b, keyETag, v)
	})
	return nil
}

func getETag(si *metadata.StorageItem) string {
	v := si.Get(keyETag)
	if v == nil {
		return ""
	}
	var etag string
	if err := v.Unmarshal(&etag); err != nil {
		return ""
	}
	return etag
}

func setChecksum(si *metadata.StorageItem, url string, d digest.Digest) error {
	v, err := metadata.NewValue(d)
	if err != nil {
		return errors.Wrap(err, "failed to create checksum value")
	}
	v.Index = url
	si.Queue(func(b *bolt.Bucket) error {
		return si.SetValue(b, keyChecksum, v)
	})
	return nil
}

func getChecksum(si *metadata.StorageItem) digest.Digest {
	v := si.Get(keyChecksum)
	if v == nil {
		return ""
	}
	var dgstStr string
	if err := v.Unmarshal(&dgstStr); err != nil {
		return ""
	}
	dgst, err := digest.Parse(dgstStr)
	if err != nil {
		return ""
	}
	return dgst
}

func getFileName(urlStr, manualFilename string, resp *http.Response) string {
	if manualFilename != "" {
		return manualFilename
	}
	if resp != nil {
		if contentDisposition := resp.Header.Get("Content-Disposition"); contentDisposition != "" {
			if _, params, err := mime.ParseMediaType(contentDisposition); err == nil {
				if params["filename"] != "" && !strings.HasSuffix(params["filename"], "/") {
					if filename := filepath.Base(filepath.FromSlash(params["filename"])); filename != "" {
						return filename
					}
				}
			}
		}
	}
	u, err := url.Parse(urlStr)
	if err == nil {
		if base := path.Base(u.Path); base != "." && base != "/" {
			return base
		}
	}
	return "download"
}
<commit_msg>http: include modtime in cache hash<commit_after>package http

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/BurntSushi/locker"
	"github.com/boltdb/bolt"
	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/cache/metadata"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/tracing"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

type Opt struct {
	CacheAccessor cache.Accessor
	MetadataStore *metadata.Store
	Transport     http.RoundTripper
}

type httpSource struct {
	md     *metadata.Store
	cache  cache.Accessor
	locker *locker.Locker
	client *http.Client
}

func NewSource(opt Opt) (source.Source, error) {
	transport := opt.Transport
	if transport == nil {
		transport = tracing.DefaultTransport
	}
	hs := &httpSource{
		md:     opt.MetadataStore,
		cache:  opt.CacheAccessor,
		locker: locker.NewLocker(),
		client: &http.Client{
			Transport: transport,
		},
	}
	return hs, nil
}

func (hs *httpSource) ID() string {
	return source.HttpsScheme
}

type httpSourceHandler struct {
	*httpSource
	src      source.HttpIdentifier
	refID    string
	cacheKey digest.Digest
}

func (hs *httpSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
	httpIdentifier, ok := id.(*source.HttpIdentifier)
	if !ok {
		return nil, errors.Errorf("invalid http identifier %v", id)
	}

	return &httpSourceHandler{
		src:        *httpIdentifier,
		httpSource: hs,
	}, nil
}

// urlHash is internal hash the etag is stored by that doesn't leak outside
// this package.
func (hs *httpSourceHandler) urlHash() (digest.Digest, error) {
	dt, err := json.Marshal(struct {
		Filename       string
		Perm, UID, GID int
	}{
		Filename: getFileName(hs.src.URL, hs.src.Filename, nil),
		Perm:     hs.src.Perm,
		UID:      hs.src.UID,
		GID:      hs.src.GID,
	})
	if err != nil {
		return "", err
	}
	return digest.FromBytes(dt), nil
}

func (hs *httpSourceHandler) formatCacheKey(filename string, dgst digest.Digest, lastModTime string) digest.Digest {
	dt, err := json.Marshal(struct {
		Filename       string
		Perm, UID, GID int
		Checksum       digest.Digest
		LastModTime    string `json:",omitempty"`
	}{
		Filename:    filename,
		Perm:        hs.src.Perm,
		UID:         hs.src.UID,
		GID:         hs.src.GID,
		Checksum:    dgst,
		LastModTime: lastModTime,
	})
	if err != nil {
		return dgst
	}
	return digest.FromBytes(dt)
}

func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, bool, error) {
	if hs.src.Checksum != "" {
		hs.cacheKey = hs.src.Checksum
		return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, nil), hs.src.Checksum, "").String(), true, nil
	}

	uh, err := hs.urlHash()
	if err != nil {
		return "", false, nil
	}

	// look up metadata(previously stored headers) for that URL
	sis, err := hs.md.Search(uh.String())
	if err != nil {
		return "", false, errors.Wrapf(err, "failed to search metadata for %s", uh)
	}

	req, err := http.NewRequest("GET", hs.src.URL, nil)
	if err != nil {
		return "", false, err
	}
	req = req.WithContext(ctx)
	m := map[string]*metadata.StorageItem{}

	if len(sis) > 0 {
		for _, si := range sis {
			// if metaDigest := getMetaDigest(si); metaDigest == hs.formatCacheKey("") {
			if etag := getETag(si); etag != "" {
				if dgst := getChecksum(si); dgst != "" {
					m[etag] = si
					req.Header.Add("If-None-Match", etag)
				}
			}
			// }
		}
	}

	resp, err := hs.client.Do(req)
	if err != nil {
		return "", false, err
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		return "", false, errors.Errorf("invalid response status %d", resp.StatusCode)
	}
	if resp.StatusCode == http.StatusNotModified {
		respETag := resp.Header.Get("ETag")
		si, ok := m[respETag]
		if !ok {
			return "", false, errors.Errorf("invalid not-modified ETag: %v", respETag)
		}
		hs.refID = si.ID()
		dgst := getChecksum(si)
		if dgst == "" {
			return "", false, errors.Errorf("invalid metadata change")
		}
		modTime := getModTime(si)
		resp.Body.Close()
		return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, modTime).String(), true, nil
	}

	ref, dgst, err := hs.save(ctx, resp)
	if err != nil {
		return "", false, err
	}
	ref.Release(context.TODO())

	hs.cacheKey = dgst

	return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, resp.Header.Get("Last-Modified")).String(), true, nil
}

func (hs *httpSourceHandler) save(ctx context.Context, resp *http.Response) (ref cache.ImmutableRef, dgst digest.Digest, retErr error) {
	newRef, err := hs.cache.New(ctx, nil, cache.CachePolicyRetain, cache.WithDescription(fmt.Sprintf("http url %s", hs.src.URL)))
	if err != nil {
		return nil, "", err
	}

	releaseRef := func() {
		newRef.Release(context.TODO())
	}

	defer func() {
		if retErr != nil && newRef != nil {
			releaseRef()
		}
	}()

	mount, err := newRef.Mount(ctx, false)
	if err != nil {
		return nil, "", err
	}

	lm := snapshot.LocalMounter(mount)
	dir, err := lm.Mount()
	if err != nil {
		return nil, "", err
	}

	defer func() {
		if retErr != nil && lm != nil {
			lm.Unmount()
		}
	}()
	perm := 0600
	if hs.src.Perm != 0 {
		perm = hs.src.Perm
	}
	fp := filepath.Join(dir, getFileName(hs.src.URL, hs.src.Filename, resp))

	f, err := os.OpenFile(fp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(perm))
	if err != nil {
		return nil, "", err
	}
	defer func() {
		if f != nil {
			f.Close()
		}
	}()

	h := sha256.New()

	if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil {
		return nil, "", err
	}

	if err := f.Close(); err != nil {
		return nil, "", err
	}
	f = nil

	if hs.src.UID != 0 || hs.src.GID != 0 {
		if err := os.Chown(fp, hs.src.UID, hs.src.GID); err != nil {
			return nil, "", err
		}
	}

	mTime := time.Unix(0, 0)
	lastMod := resp.Header.Get("Last-Modified")
	if lastMod != "" {
		if parsedMTime, err := http.ParseTime(lastMod); err == nil {
			mTime = parsedMTime
		}
	}

	if err := os.Chtimes(fp, mTime, mTime); err != nil {
		return nil, "", err
	}

	lm.Unmount()
	lm = nil

	ref, err = newRef.Commit(ctx)
	if err != nil {
		return nil, "", err
	}
	newRef = nil

	hs.refID = ref.ID()
	dgst = digest.NewDigest(digest.SHA256, h)

	if respETag := resp.Header.Get("ETag"); respETag != "" {
		setETag(ref.Metadata(), respETag)
		uh, err := hs.urlHash()
		if err != nil {
			return nil, "", err
		}
		setChecksum(ref.Metadata(), uh.String(), dgst)
		if err := ref.Metadata().Commit(); err != nil {
			return nil, "", err
		}
	}

	if modTime := resp.Header.Get("Last-Modified"); modTime != "" {
		setModTime(ref.Metadata(), modTime)
	}

	return ref, dgst, nil
}

func (hs *httpSourceHandler) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
	if hs.refID != "" {
		ref, err := hs.cache.Get(ctx, hs.refID)
		if err == nil {
			return ref, nil
		}
	}

	req, err := http.NewRequest("GET", hs.src.URL, nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)

	resp, err := hs.client.Do(req)
	if err != nil {
		return nil, err
	}

	ref, dgst, err := hs.save(ctx, resp)
	if err != nil {
		return nil, err
	}
	if dgst != hs.cacheKey {
		ref.Release(context.TODO())
		return nil, errors.Errorf("digest mismatch %s: %s", dgst, hs.cacheKey)
	}

	return ref, nil
}

const keyETag = "etag"
const keyChecksum = "http.checksum"
const keyModTime = "http.modtime"

func setETag(si *metadata.StorageItem, s string) error {
	v, err := metadata.NewValue(s)
	if err != nil {
		return errors.Wrap(err, "failed to create etag value")
	}
	si.Queue(func(b *bolt.Bucket) error {
		return si.SetValue(b, keyETag, v)
	})
	return nil
}

func getETag(si *metadata.StorageItem) string {
	v := si.Get(keyETag)
	if v == nil {
		return ""
	}
	var etag string
	if err := v.Unmarshal(&etag); err != nil {
		return ""
	}
	return etag
}

func setModTime(si *metadata.StorageItem, s string) error {
	v, err := metadata.NewValue(s)
	if err != nil {
		return errors.Wrap(err, "failed to create modtime value")
	}
	si.Queue(func(b *bolt.Bucket) error {
		return si.SetValue(b, keyModTime, v)
	})
	return nil
}

func getModTime(si *metadata.StorageItem) string {
	v := si.Get(keyModTime)
	if v == nil {
		return ""
	}
	var modTime string
	if err := v.Unmarshal(&modTime); err != nil {
		return ""
	}
	return modTime
}

func setChecksum(si *metadata.StorageItem, url string, d digest.Digest) error {
	v, err := metadata.NewValue(d)
	if err != nil {
		return errors.Wrap(err, "failed to create checksum value")
	}
	v.Index = url
	si.Queue(func(b *bolt.Bucket) error {
		return si.SetValue(b, keyChecksum, v)
	})
	return nil
}

func getChecksum(si *metadata.StorageItem) digest.Digest {
	v := si.Get(keyChecksum)
	if v == nil {
		return ""
	}
	var dgstStr string
	if err := v.Unmarshal(&dgstStr); err != nil {
		return ""
	}
	dgst, err := digest.Parse(dgstStr)
	if err != nil {
		return ""
	}
	return dgst
}

func getFileName(urlStr, manualFilename string, resp *http.Response) string {
	if manualFilename != "" {
		return manualFilename
	}
	if resp != nil {
		if contentDisposition := resp.Header.Get("Content-Disposition"); contentDisposition != "" {
			if _, params, err := mime.ParseMediaType(contentDisposition); err == nil {
				if params["filename"] != "" && !strings.HasSuffix(params["filename"], "/") {
					if filename := filepath.Base(filepath.FromSlash(params["filename"])); filename != "" {
						return filename
					}
				}
			}
		}
	}
	u, err := url.Parse(urlStr)
	if err == nil {
		if base := path.Base(u.Path); base != "." && base != "/" {
			return base
		}
	}
	return "download"
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"errors"
	"log"

	"github.com/oursky/ourd/auth"
	"github.com/oursky/ourd/oddb"
	"github.com/oursky/ourd/router"
)

// RecordHandler declares the interface of a handler that works with records
type RecordHandler func(*recordPayload, *router.Response, oddb.Database)

// RecordService provides a collection of handlers to
// handle oddb.Record related operations on an oddb.Database.
type RecordService struct {
	auth.TokenStore
}

// injectRecordHandler returns a router.Handler that has a proper
// public / private database injected into RecordHandler according to
// the payload
func (s RecordService) injectRecordHandler(recordHandler RecordHandler) router.Handler {
	return func(rpayload *router.Payload, response *router.Response) {
		payload := newRecordPayload(rpayload)

		if !payload.IsValidDB() {
			response.Result = NewError(MissingDatabaseIDErr, "Invalid Database ID")
			return
		}

		var db oddb.Database
		token := auth.Token{}
		if payload.IsPublicDB() {
			if !payload.IsReadOnly() {
				if err := s.TokenStore.Get(payload.AccessToken(), &token); err != nil {
					response.Result = NewError(InvalidAccessTokenErr, "Invalid access token")
					return
				}
			}
			db = payload.DBConn.PublicDB()
		} else { // if a request doesn't ask for public DB, then it is private DB
			if err := s.TokenStore.Get(payload.AccessToken(), &token); err != nil {
				response.Result = NewError(InvalidAccessTokenErr, "Invalid access token")
				return
			}

			db = payload.DBConn.PrivateDB(token.UserInfoID)
		}

		recordHandler(&payload, response, db)
	}
}

// RecordFetchHandler returns a router.Handler that fetches a record.
func (s RecordService) RecordFetchHandler() router.Handler {
	return s.injectRecordHandler(RecordFetchHandler)
}

// RecordSaveHandler returns a router.Handler that saves a record.
func (s RecordService) RecordSaveHandler() router.Handler {
	return s.injectRecordHandler(RecordSaveHandler)
}

// RecordDeleteHandler returns a router.Handler that deletes a record.
func (s RecordService) RecordDeleteHandler() router.Handler {
	return s.injectRecordHandler(RecordDeleteHandler)
}

// RecordQueryHandler returns a router.Handler that queries records.
func (s RecordService) RecordQueryHandler() router.Handler {
	return s.injectRecordHandler(RecordQueryHandler)
}

// recordPayload is the input parameter in RecordHandler
type recordPayload struct {
	*router.Payload
	DatabaseID string
}

func newRecordPayload(payload *router.Payload) recordPayload {
	databaseID, _ := payload.Data["database_id"].(string)
	return recordPayload{
		Payload:    payload,
		DatabaseID: databaseID,
	}
}

func (p recordPayload) IsValidDB() bool {
	return p.DatabaseID == "_public" || p.DatabaseID == "_private"
}

func (p recordPayload) IsPublicDB() bool {
	return p.DatabaseID == "_public"
}

func (p recordPayload) IsReadOnly() bool {
	action := p.RouteAction()
	return action == "record:fetch" || action == "record:query"
}

// transportRecord override JSON serialization and deserialization of
// oddb.Record
type transportRecord oddb.Record

func (r transportRecord) MarshalJSON() ([]byte, error) {
	// NOTE(limouren): if there is a better way to shallow copy a map,
	// do let me know
	object := map[string]interface{}{}
	for k, v := range r.Data {
		object[k] = v
	}
	object["_id"] = r.Key
	object["_type"] = r.Type

	return json.Marshal(object)
}

func (r *transportRecord) UnmarshalJSON(data []byte) error {
	object := map[string]interface{}{}
	err := json.Unmarshal(data, &object)

	if err != nil {
		return err
	}

	return r.InitFromMap(object)
}

func (r *transportRecord) InitFromMap(m map[string]interface{}) error {
	id, ok := m["_id"].(string)
	if !ok {
		return errors.New(`record/json: required field "_id" not found`)
	}
	r.Key = id
	delete(m, "_id")

	t, ok := m["_type"].(string)
	if !ok {
		return errors.New(`record/json: required field "_type" not found`)
	}
	r.Type = t
	delete(m, "_type")

	r.Data = m

	return nil
}

/*
RecordSaveHandler is dummy implementation on save/modify Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "record:save",
    "access_token": "validToken",
    "database_id": "private"
}
EOF
*/
func RecordSaveHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	recordMaps, ok := payload.Data["records"].([]map[string]interface{})
	if !ok {
		response.Result = NewError(RequestInvalidErr, "invalid request: expected list of records")
		return
	}

	length := len(recordMaps)

	records := make([]transportRecord, length, length)
	results := make([]interface{}, length, length)
	for i := range records {
		if err := records[i].InitFromMap(recordMaps[i]); err != nil {
			results[i] = NewError(RequestInvalidErr, "invalid request: "+err.Error())
		}
	}

	for i := range records {
		_, fail := results[i].(error)
		if !fail {
			if err := db.Save((*oddb.Record)(&records[i])); err != nil {
				results[i] = NewError(PersistentStorageErr, "persistent error: failed to save record")
			} else {
				results[i] = records[i]
			}
		}
	}

	response.Result = results
}

/*
RecordFetchHandler is dummy implementation on fetching Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "record:fetch",
    "access_token": "validToken",
    "database_id": "private",
    "ids": ["1004", "1005"]
}
EOF
*/
func RecordFetchHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	var (
		records []oddb.Record
	)
	records = append(records, oddb.Record{
		Type: "abc",
		Key:  "abc:uuid",
	})
	log.Println("RecordFetchHandler")
	response.Result = records
	return
}

/*
RecordQueryHandler is dummy implementation on fetching Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "record:query",
    "access_token": "validToken",
    "database_id": "private"
}
EOF
*/
func RecordQueryHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	log.Println("RecordQueryHandler")
	return
}

/*
RecordDeleteHandler is dummy implementation on delete Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "redord:delete",
    "access_token": "validToken",
    "database_id": "private"
}
EOF
*/
func RecordDeleteHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	log.Println("RecordDeleteHandler")
	return
}
<commit_msg>Implement endpoint of querying records, #26<commit_after>package handler

import (
	"encoding/json"
	"errors"
	"io"
	"log"

	"github.com/oursky/ourd/auth"
	"github.com/oursky/ourd/oddb"
	"github.com/oursky/ourd/router"
)

// RecordHandler declares the interface of a handler that works with records
type RecordHandler func(*recordPayload, *router.Response, oddb.Database)

// RecordService provides a collection of handlers to
// handle oddb.Record related operations on an oddb.Database.
type RecordService struct {
	auth.TokenStore
}

// injectRecordHandler returns a router.Handler that has a proper
// public / private database injected into RecordHandler according to
// the payload
func (s RecordService) injectRecordHandler(recordHandler RecordHandler) router.Handler {
	return func(rpayload *router.Payload, response *router.Response) {
		payload := newRecordPayload(rpayload)

		if !payload.IsValidDB() {
			response.Result = NewError(MissingDatabaseIDErr, "Invalid Database ID")
			return
		}

		var db oddb.Database
		token := auth.Token{}
		if payload.IsPublicDB() {
			if !payload.IsReadOnly() {
				if err := s.TokenStore.Get(payload.AccessToken(), &token); err != nil {
					response.Result = NewError(InvalidAccessTokenErr, "Invalid access token")
					return
				}
			}
			db = payload.DBConn.PublicDB()
		} else { // if a request doesn't ask for public DB, then it is private DB
			if err := s.TokenStore.Get(payload.AccessToken(), &token); err != nil {
				response.Result = NewError(InvalidAccessTokenErr, "Invalid access token")
				return
			}

			db = payload.DBConn.PrivateDB(token.UserInfoID)
		}

		recordHandler(&payload, response, db)
	}
}

// RecordFetchHandler returns a router.Handler that fetches a record.
func (s RecordService) RecordFetchHandler() router.Handler {
	return s.injectRecordHandler(RecordFetchHandler)
}

// RecordSaveHandler returns a router.Handler that saves a record.
func (s RecordService) RecordSaveHandler() router.Handler {
	return s.injectRecordHandler(RecordSaveHandler)
}

// RecordDeleteHandler returns a router.Handler that deletes a record.
func (s RecordService) RecordDeleteHandler() router.Handler {
	return s.injectRecordHandler(RecordDeleteHandler)
}

// RecordQueryHandler returns a router.Handler that queries records.
func (s RecordService) RecordQueryHandler() router.Handler {
	return s.injectRecordHandler(RecordQueryHandler)
}

// recordPayload is the input parameter in RecordHandler
type recordPayload struct {
	*router.Payload
	DatabaseID string
}

func newRecordPayload(payload *router.Payload) recordPayload {
	databaseID, _ := payload.Data["database_id"].(string)
	return recordPayload{
		Payload:    payload,
		DatabaseID: databaseID,
	}
}

func (p recordPayload) IsValidDB() bool {
	return p.DatabaseID == "_public" || p.DatabaseID == "_private"
}

func (p recordPayload) IsPublicDB() bool {
	return p.DatabaseID == "_public"
}

func (p recordPayload) IsReadOnly() bool {
	action := p.RouteAction()
	return action == "record:fetch" || action == "record:query"
}

// transportRecord override JSON serialization and deserialization of
// oddb.Record
type transportRecord oddb.Record

func (r transportRecord) MarshalJSON() ([]byte, error) {
	// NOTE(limouren): if there is a better way to shallow copy a map,
	// do let me know
	object := map[string]interface{}{}
	for k, v := range r.Data {
		object[k] = v
	}
	object["_id"] = r.Key
	object["_type"] = r.Type

	return json.Marshal(object)
}

func (r *transportRecord) UnmarshalJSON(data []byte) error {
	object := map[string]interface{}{}
	err := json.Unmarshal(data, &object)

	if err != nil {
		return err
	}

	return r.InitFromMap(object)
}

func (r *transportRecord) InitFromMap(m map[string]interface{}) error {
	id, ok := m["_id"].(string)
	if !ok {
		return errors.New(`record/json: required field "_id" not found`)
	}
	r.Key = id
	delete(m, "_id")

	t, ok := m["_type"].(string)
	if !ok {
		return errors.New(`record/json: required field "_type" not found`)
	}
	r.Type = t
	delete(m, "_type")

	r.Data = m

	return nil
}

/*
RecordSaveHandler is dummy implementation on save/modify Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "record:save",
    "access_token": "validToken",
    "database_id": "private"
}
EOF
*/
func RecordSaveHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	recordMaps, ok := payload.Data["records"].([]map[string]interface{})
	if !ok {
		response.Result = NewError(RequestInvalidErr, "invalid request: expected list of records")
		return
	}

	length := len(recordMaps)

	records := make([]transportRecord, length, length)
	results := make([]interface{}, length, length)
	for i := range records {
		if err := records[i].InitFromMap(recordMaps[i]); err != nil {
			results[i] = NewError(RequestInvalidErr, "invalid request: "+err.Error())
		}
	}

	for i := range records {
		_, fail := results[i].(error)
		if !fail {
			if err := db.Save((*oddb.Record)(&records[i])); err != nil {
				results[i] = NewError(PersistentStorageErr, "persistent error: failed to save record")
			} else {
				results[i] = records[i]
			}
		}
	}

	response.Result = results
}

/*
RecordFetchHandler is dummy implementation on fetching Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "record:fetch",
    "access_token": "validToken",
    "database_id": "private",
    "ids": ["1004", "1005"]
}
EOF
*/
func RecordFetchHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	var (
		records []oddb.Record
	)
	records = append(records, oddb.Record{
		Type: "abc",
		Key:  "abc:uuid",
	})
	log.Println("RecordFetchHandler")
	response.Result = records
	return
}

/*
RecordQueryHandler is dummy implementation on fetching Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "record:query",
    "access_token": "validToken",
    "database_id": "private"
}
EOF
*/
func RecordQueryHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	recordType, _ := payload.Data["record_type"].(string)
	if recordType == "" {
		response.Result = NewError(RequestInvalidErr, "recordType cannot be empty")
		return
	}

	results, err := db.Query("", recordType)
	if err != nil {
		response.Result = NewError(UnknownErr, "failed to open database")
		return
	}
	defer results.Close()

	records := []transportRecord{}
	record := oddb.Record{}

	// needs a better abstraction here
	err = results.Next(&record)
	for err != nil {
		records = append(records, transportRecord(record))
		err = results.Next(&record)
	}

	// query failed
	if err != io.EOF {
		response.Result = NewError(UnknownErr, "failed to query records")
		return
	}

	response.Result = records
}

/*
RecordDeleteHandler is dummy implementation on delete Records
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/ <<EOF
{
    "action": "redord:delete",
    "access_token": "validToken",
    "database_id": "private"
}
EOF
*/
func RecordDeleteHandler(payload *recordPayload, response *router.Response, db oddb.Database) {
	log.Println("RecordDeleteHandler")
	return
}
<|endoftext|>


<commit_before>package handlers

import (
	"container/heap"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"golang.org/x/tools/go/vcs"

	"github.com/boltdb/bolt"
)

const (
	// DBPath is the relative (or absolute) path to the bolt database file
	DBPath string = "goreportcard.db"

	// RepoBucket is the bucket in which repos will be cached in the bolt DB
	RepoBucket string = "repos"

	// MetaBucket is the bucket containing the names of the projects with the
	// top 100 high scores, and other meta information
	MetaBucket string = "meta"
)

// trimScheme removes a scheme (e.g. https://) from the URL for more
// convenient pasting from browsers.
func trimScheme(repo string) string {
	schemeSep := "://"
	schemeSepIdx := strings.Index(repo, schemeSep)
	if schemeSepIdx > -1 {
		return repo[schemeSepIdx+len(schemeSep):]
	}

	return repo
}

// CheckHandler handles the request for checking a repo
func CheckHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	repo := trimScheme(strings.ToLower(r.FormValue("repo")))

	repoRoot, err := vcs.RepoRootForImportPath(repo, true)
	if err != nil || repoRoot.Root == "" || repoRoot.Repo == "" {
		log.Println("Failed to create repoRoot:", repoRoot, err)
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`Please enter a valid 'go get'-able package name`))
		return
	}

	log.Printf("Checking repo %q...", repo)

	forceRefresh := r.Method != "GET" // if this is a GET request, try to fetch from cached version in boltdb first
	resp, err := newChecksResp(repo, forceRefresh)
	if err != nil {
		log.Println("ERROR: from newChecksResp:", err)
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`Could not go get the repository.`))
		return
	}

	respBytes, err := json.Marshal(resp)
	if err != nil {
		log.Println("ERROR: could not marshal json:", err)
		http.Error(w, err.Error(), 500)
		return
	}

	// write to boltdb
	db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Println("Failed to open bolt database: ", err)
		return
	}
	defer db.Close()

	// is this a new repo? if so, increase the count in the high scores bucket later
	isNewRepo := false
	var oldRepoBytes []byte
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(RepoBucket))
		if b == nil {
			return fmt.Errorf("repo bucket not found")
		}
		oldRepoBytes = b.Get([]byte(repo))
		return nil
	})
	if err != nil {
		log.Println(err)
	}

	// get the old score and store it for stats updating
	var oldScore *float64
	if isNewRepo = oldRepoBytes == nil; !isNewRepo {
		oldRepo := checksResp{}
		err = json.Unmarshal(oldRepoBytes, &oldRepo)
		if err != nil {
			log.Println("ERROR: could not unmarshal json:", err)
			http.Error(w, err.Error(), 500)
			return
		}
		oldScore = &oldRepo.Average
	}

	// if this is a new repo, or the user force-refreshed, update the cache
	if isNewRepo || forceRefresh {
		err = db.Update(func(tx *bolt.Tx) error {
			log.Printf("Saving repo %q to cache...", repo)

			b := tx.Bucket([]byte(RepoBucket))
			if b == nil {
				return fmt.Errorf("repo bucket not found")
			}

			// save repo to cache
			err = b.Put([]byte(repo), respBytes)
			if err != nil {
				return err
			}

			// fetch meta-bucket
			mb := tx.Bucket([]byte(MetaBucket))
			if mb == nil {
				return fmt.Errorf("high score bucket not found")
			}

			// update total repos count
			if isNewRepo {
				err = updateReposCount(mb, resp, repo)
				if err != nil {
					return err
				}
			}

			err = updateHighScores(mb, resp, repo)
			if err != nil {
				return err
			}

			return updateStats(mb, resp, repo, oldScore)
		})

		if err != nil {
			log.Println("Bolt writing error:", err)
		}

	}

	err = db.Update(func(tx *bolt.Tx) error {
		// fetch meta-bucket
		mb := tx.Bucket([]byte(MetaBucket))
		if mb == nil {
			return fmt.Errorf("meta bucket not found")
		}

		return updateRecentlyViewed(mb, repo)
	})

	b, err := json.Marshal(map[string]string{"redirect": "/report/" + repo})
	if err != nil {
		log.Println("JSON marshal error:", err)
	}
	w.WriteHeader(http.StatusOK)
	w.Write(b)
	return
}

func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error {
	// check if we need to update the high score list
	if resp.Files < 100 {
		// only repos with >= 100 files are considered for the high score list
		return nil
	}

	// start updating high score list
	scoreBytes := mb.Get([]byte("scores"))
	if scoreBytes == nil {
		scoreBytes, _ = json.Marshal([]scoreHeap{})
	}
	scores := &scoreHeap{}
	json.Unmarshal(scoreBytes, scores)

	heap.Init(scores)
	if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 && len(*scores) == 50 {
		// lowest score on list is higher than this repo's score, so no need to add, unless
		// we do not have 50 high scores yet
		return nil
	}
	// if this repo is already in the list, remove the original entry:
	for i := range *scores {
		if (*scores)[i].Repo == repo {
			heap.Remove(scores, i)
			break
		}
	}
	// now we can safely push it onto the heap
	heap.Push(scores, scoreItem{
		Repo:  repo,
		Score: resp.Average * 100.0,
		Files: resp.Files,
	})
	if len(*scores) > 50 {
		// trim heap if it's grown to over 50
		*scores = (*scores)[1:51]
	}
	scoreBytes, err := json.Marshal(&scores)
	if err != nil {
		return err
	}
	err = mb.Put([]byte("scores"), scoreBytes)
	if err != nil {
		return err
	}

	return nil
}

func updateStats(mb *bolt.Bucket, resp checksResp, repo string, oldScore *float64) error {
	scores := make([]int, 101, 101)
	statsBytes := mb.Get([]byte("stats"))
	if statsBytes == nil {
		statsBytes, _ = json.Marshal(scores)
	}
	err := json.Unmarshal(statsBytes, &scores)
	if err != nil {
		return err
	}
	scores[int(resp.Average*100)]++
	if oldScore != nil {
		scores[int(*oldScore*100)]--
	}
	newStats, err := json.Marshal(scores)
	if err != nil {
		return err
	}
	err = mb.Put([]byte("stats"), newStats)
	if err != nil {
		return err
	}
	return nil
}

func updateReposCount(mb *bolt.Bucket, resp checksResp, repo string) (err error) {
	log.Printf("New repo %q, adding to repo count...", repo)
	totalInt := 0
	total := mb.Get([]byte("total_repos"))
	if total != nil {
		err = json.Unmarshal(total, &totalInt)
		if err != nil {
			return fmt.Errorf("could not unmarshal total repos count: %v", err)
		}
	}
	totalInt++ // increase repo count
	total, err = json.Marshal(totalInt)
	if err != nil {
		return fmt.Errorf("could not marshal total repos count: %v", err)
	}
	mb.Put([]byte("total_repos"), total)
	log.Println("Repo count is now", totalInt)
	return nil
}

type recentItem struct {
	Repo string
}

func updateRecentlyViewed(mb *bolt.Bucket, repo string) error {
	b := mb.Get([]byte("recent"))
	if b == nil {
		b, _ = json.Marshal([]recentItem{})
	}
	recent := []recentItem{}
	json.Unmarshal(b, &recent)

	// add it to the slice, if it is not in there already
	for i := range recent {
		if recent[i].Repo == repo {
			return nil
		}
	}

	recent = append(recent, recentItem{Repo: repo})
	if len(recent) > 5 {
		// trim recent if it's grown to over 5
		recent = (recent)[1:6]
	}
	b, err := json.Marshal(&recent)
	if err != nil {
		return err
	}
	err = mb.Put([]byte("recent"), b)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>#45 compare repo name during high scores insertion rather than form value<commit_after>package handlers

import (
	"container/heap"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"golang.org/x/tools/go/vcs"

	"github.com/boltdb/bolt"
)

const (
	// DBPath is the relative (or absolute) path to the bolt database file
	DBPath string = "goreportcard.db"

	// RepoBucket is the bucket in which repos will be cached in the bolt DB
	RepoBucket string = "repos"

	// MetaBucket is the bucket containing the names of the projects with the
	// top 100 high scores, and other meta information
	MetaBucket string = "meta"
)

// trimScheme removes a scheme (e.g. https://) from the URL for more
// convenient pasting from browsers.
func trimScheme(repo string) string {
	schemeSep := "://"
	schemeSepIdx := strings.Index(repo, schemeSep)
	if schemeSepIdx > -1 {
		return repo[schemeSepIdx+len(schemeSep):]
	}

	return repo
}

// CheckHandler handles the request for checking a repo
func CheckHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	repo := trimScheme(r.FormValue("repo"))

	repoRoot, err := vcs.RepoRootForImportPath(repo, true)
	if err != nil || repoRoot.Root == "" || repoRoot.Repo == "" {
		log.Println("Failed to create repoRoot:", repoRoot, err)
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`Please enter a valid 'go get'-able package name`))
		return
	}

	log.Printf("Checking repo %q...", repo)

	forceRefresh := r.Method != "GET" // if this is a GET request, try to fetch from cached version in boltdb first
	resp, err := newChecksResp(repo, forceRefresh)
	if err != nil {
		log.Println("ERROR: from newChecksResp:", err)
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`Could not go get the repository.`))
		return
	}

	respBytes, err := json.Marshal(resp)
	if err != nil {
		log.Println("ERROR: could not marshal json:", err)
		http.Error(w, err.Error(), 500)
		return
	}

	// write to boltdb
	db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Println("Failed to open bolt database: ", err)
		return
	}
	defer db.Close()

	// is this a new repo? if so, increase the count in the high scores bucket later
	isNewRepo := false
	var oldRepoBytes []byte
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(RepoBucket))
		if b == nil {
			return fmt.Errorf("repo bucket not found")
		}
		oldRepoBytes = b.Get([]byte(repo))
		return nil
	})
	if err != nil {
		log.Println(err)
	}

	// get the old score and store it for stats updating
	var oldScore *float64
	if isNewRepo = oldRepoBytes == nil; !isNewRepo {
		oldRepo := checksResp{}
		err = json.Unmarshal(oldRepoBytes, &oldRepo)
		if err != nil {
			log.Println("ERROR: could not unmarshal json:", err)
			http.Error(w, err.Error(), 500)
			return
		}
		oldScore = &oldRepo.Average
	}

	// if this is a new repo, or the user force-refreshed, update the cache
	if isNewRepo || forceRefresh {
		err = db.Update(func(tx *bolt.Tx) error {
			log.Printf("Saving repo %q to cache...", repo)

			b := tx.Bucket([]byte(RepoBucket))
			if b == nil {
				return fmt.Errorf("repo bucket not found")
			}

			// save repo to cache
			err = b.Put([]byte(repo), respBytes)
			if err != nil {
				return err
			}

			// fetch meta-bucket
			mb := tx.Bucket([]byte(MetaBucket))
			if mb == nil {
				return fmt.Errorf("high score bucket not found")
			}

			// update total repos count
			if isNewRepo {
				err = updateReposCount(mb, resp, repo)
				if err != nil {
					return err
				}
			}

			err = updateHighScores(mb, resp, repo)
			if err != nil {
				return err
			}

			return updateStats(mb, resp, repo, oldScore)
		})

		if err != nil {
			log.Println("Bolt writing error:", err)
		}

	}

	err = db.Update(func(tx *bolt.Tx) error {
		// fetch meta-bucket
		mb := tx.Bucket([]byte(MetaBucket))
		if mb == nil {
			return fmt.Errorf("meta bucket not found")
		}

		return updateRecentlyViewed(mb, repo)
	})

	b, err := json.Marshal(map[string]string{"redirect": "/report/" + repo})
	if err != nil {
		log.Println("JSON marshal error:", err)
	}
	w.WriteHeader(http.StatusOK)
	w.Write(b)
	return
}

func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error {
	// check if we need to update the high score list
	if resp.Files < 100 {
		// only repos with >= 100 files are considered for the high score list
		return nil
	}

	// start updating high score list
	scoreBytes := mb.Get([]byte("scores"))
	if scoreBytes == nil {
		scoreBytes, _ = json.Marshal([]scoreHeap{})
	}
	scores := &scoreHeap{}
	json.Unmarshal(scoreBytes, scores)

	heap.Init(scores)
	if len(*scores) > 0 && (*scores)[0].Score > resp.Average*100.0 && len(*scores) == 50 {
		// lowest score on list is higher than this repo's score, so no need to add, unless
		// we do not have 50 high scores yet
		return nil
	}
	// if this repo is already in the list, remove the original entry:
	for i := range *scores {
		if strings.ToLower((*scores)[i].Repo) == strings.ToLower(repo) {
			heap.Remove(scores, i)
			break
		}
	}
	// now we can safely push it onto the heap
	heap.Push(scores, scoreItem{
		Repo:  repo,
		Score: resp.Average * 100.0,
		Files: resp.Files,
	})
	if len(*scores) > 50 {
		// trim heap if it's grown to over 50
		*scores = (*scores)[1:51]
	}
	scoreBytes, err := json.Marshal(&scores)
	if err != nil {
		return err
	}
	err = mb.Put([]byte("scores"), scoreBytes)
	if err != nil {
		return err
	}

	return nil
}

func updateStats(mb *bolt.Bucket, resp checksResp, repo string, oldScore *float64) error {
	scores := make([]int, 101, 101)
	statsBytes := mb.Get([]byte("stats"))
	if statsBytes == nil {
		statsBytes, _ = json.Marshal(scores)
	}
	err := json.Unmarshal(statsBytes, &scores)
	if err != nil {
		return err
	}
	scores[int(resp.Average*100)]++
	if oldScore != nil {
		scores[int(*oldScore*100)]--
	}
	newStats, err := json.Marshal(scores)
	if err != nil {
		return err
	}
	err = mb.Put([]byte("stats"), newStats)
	if err != nil {
		return err
	}
	return nil
}

func updateReposCount(mb *bolt.Bucket, resp checksResp, repo string) (err error) {
	log.Printf("New repo %q, adding to repo count...", repo)
	totalInt := 0
	total := mb.Get([]byte("total_repos"))
	if total != nil {
		err = json.Unmarshal(total, &totalInt)
		if err != nil {
			return fmt.Errorf("could not unmarshal total repos count: %v", err)
		}
	}
	totalInt++ // increase repo count
	total, err = json.Marshal(totalInt)
	if err != nil {
		return fmt.Errorf("could not marshal total repos count: %v", err)
	}
	mb.Put([]byte("total_repos"), total)
	log.Println("Repo count is now", totalInt)
	return nil
}

type recentItem struct {
	Repo string
}

func updateRecentlyViewed(mb *bolt.Bucket, repo string) error {
	b := mb.Get([]byte("recent"))
	if b == nil {
		b, _ = json.Marshal([]recentItem{})
	}
	recent := []recentItem{}
	json.Unmarshal(b, &recent)

	// add it to the slice, if it is not in there already
	for i := range recent {
		if recent[i].Repo == repo {
			return nil
		}
	}

	recent = append(recent, recentItem{Repo: repo})
	if len(recent) > 5 {
		// trim recent if it's grown to over 5
		recent = (recent)[1:6]
	}
	b, err := json.Marshal(&recent)
	if err != nil {
		return err
	}
	err = mb.Put([]byte("recent"), b)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>