<commit_before>/*
Package detour provides a net.Conn interface to dial another dialer
if direct connection is considered blocked
It maintains three states in a connection: initial, direct and detoured
along with a temporary whitelist across connections.
it also add a blocked site to whitelist

The action taken and state transistion in each phase is as follows:
+-------------------------+-----------+-------------+-------------+-------------+-------------+
|                         | no error  |   timeout*  | conn reset/ | content     | other error |
|                         |           |             | dns hijack  | hijack      |             |
+-------------------------+-----------+-------------+-------------+-------------+-------------+
| dial (intial)           | noop      | detour      | detour      | n/a         | noop        |
| first read (intial)     | direct    | detour(buf) | detour(buf) | detour(buf) | noop        |
|                         |           | add to tl   | add to tl   | add to tl   |             |
| follow-up read (direct) | direct    | add to tl   | add to tl   | add to tl   | noop        |
| follow-up read (detour) | noop      | rm from tl  | rm from tl  | rm from tl  | rm from tl  |
| close (direct)          | noop      | n/a         | n/a         | n/a         | n/a         |
| close (detour)          | add to wl | n/a         | n/a         | n/a         | n/a         |
+-------------------------+-----------+-------------+-------------+-------------+-------------+
| next dial/read(in tl)***| noop      | rm from tl  | rm from tl  | rm from tl  | rm from tl  |
| next close(in tl)       | add to wl | n/a         | n/a         | n/a         | n/a         |
+-------------------------+-----------+-------------+-------------+-------------+-------------+
(buf) = resend buffer
tl = temporary whitelist
wl = permanent whitelist
* Operation will time out in TimeoutToDetour in initial state,
and at system default or caller supplied deadline for other states;
but in detoured state, it is considered as timeout if an operation takes longer than TimeoutToDetour.
** DNS hijack is only checked at dial time.
*** Connection is always detoured if the site is in tl or wl.
*/
package detour

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
)

// if dial or read exceeded this timeout, we consider switch to detour
// The value depends on OS and browser and defaults to 3s
// For Windows XP, find TcpMaxConnectRetransmissions in
// http://support2.microsoft.com/default.aspx?scid=kb;en-us;314053
var TimeoutToDetour = 3 * time.Second

var (
	log = golog.LoggerFor("detour")

	// instance of Detector
	blockDetector atomic.Value
)

func init() {
	blockDetector.Store(detectorByCountry(""))
}

type dialFunc func(network, addr string) (net.Conn, error)

type detourConn struct {
	muConn sync.RWMutex
	// the actual connection, will change so protect it
	// can't user atomic.Value as the concrete type may vary
	conn net.Conn

	// don't access directly, use inState() and setState() instead
	state uint32

	// the function to dial detour if the site to connect seems blocked
	dialDetour dialFunc

	muBuf sync.Mutex
	// keep track of bytes sent through direct connection
	// so we can resend them when detour
	buf bytes.Buffer

	network, addr string
	readDeadline  time.Time
	writeDeadline time.Time
}

const (
	stateInitial = iota
	stateDirect
	stateDetour
	stateWhitelistCandidate
	stateWhitelist
)

var statesDesc = []string{
	"INITIALLY",
	"DIRECTLY",
	"DETOURED",
	"WHITELIST CANDIDATE",
	"WHITELISTED",
}

func SetCountry(country string) {
	blockDetector.Store(detectorByCountry(country))
}

func Dialer(dialer dialFunc) dialFunc {
	return func(network, addr string) (conn net.Conn, err error) {
		dc := &detourConn{dialDetour: dialer, network: network, addr: addr}
		if !whitelisted(addr) {
			detector := blockDetector.Load().(*Detector)
			dc.setState(stateInitial)
			dc.conn, err = net.DialTimeout(network, addr, TimeoutToDetour)
			if err == nil {
				if !detector.CheckConn(dc.conn) {
					log.Tracef("Dial %s to %s succeeded", dc.stateDesc(), addr)
					return dc, nil
				}
				log.Debugf("Dial %s to %s, dns hijacked, try detour", dc.stateDesc(), addr)
			} else if detector.CheckError(err) {
				log.Debugf("Dial %s to %s failed, try detour: %s", dc.stateDesc(), addr, err)
			} else {
				log.Errorf("Dial %s to %s failed: %s", dc.stateDesc(), addr, err)
				return dc, err
			}
		}
		dc.conn, err = dc.dialDetour(network, addr)
		if err != nil {
			log.Errorf("Dial %s to %s failed", dc.stateDesc(), addr)
			return nil, err
		}
		dc.setState(stateDetour)
		if !whitelisted(addr) {
			log.Tracef("Dial %s to %s succeeded, add to temporary list", dc.stateDesc(), addr)
			addToWl(dc.addr, false)
		}
		return dc, err
	}
}

// Read() implements the function from net.Conn
func (dc *detourConn) Read(b []byte) (n int, err error) {
	conn := dc.getConn()
	detector := blockDetector.Load().(*Detector)
	if !dc.inState(stateInitial) {
		if n, err = conn.Read(b); err != nil {
			if err == io.EOF {
				log.Tracef("Read %d bytes from %s %s, EOF", n, dc.addr, dc.stateDesc())
				return
			}
			log.Tracef("Read from %s %s failed: %s", dc.addr, dc.stateDesc(), err)
			if dc.inState(stateDirect) && detector.CheckError(err) {
				log.Tracef("Seems %s still blocked, add to whitelist so will try detour next time", dc.addr)
				addToWl(dc.addr, false)
			} else if dc.inState(stateDetour) && wlTemporarily(dc.addr) {
				log.Tracef("Detoured route is still not reliable for %s, not whitelist it", dc.addr)
				removeFromWl(dc.addr)
			}
			return
		}
		if dc.inState(stateDirect) && detector.CheckResp(b) {
			log.Tracef("Seems %s still hijacked, add to whitelist so will try detour next time", dc.addr)
			addToWl(dc.addr, false)
		}
		log.Tracef("Read %d bytes from %s %s", n, dc.addr, dc.stateDesc())
		return n, err
	}
	// state will always be settled after first read, safe to clear buffer at end of it
	defer dc.resetBuffer()
	start := time.Now()
	dl := start.Add(TimeoutToDetour)
	if !dc.readDeadline.IsZero() && dc.readDeadline.Sub(start) < 2*TimeoutToDetour {
		log.Tracef("no time left to test %s, read %s", dc.addr, stateDirect)
		dc.setState(stateDirect)
		return conn.Read(b)
	}

	conn.SetReadDeadline(dl)
	n, err = conn.Read(b)
	conn.SetReadDeadline(dc.readDeadline)
	if err != nil && err != io.EOF {
		ne := fmt.Errorf("Error while read from %s %s, takes %s: %s", dc.addr, dc.stateDesc(), time.Now().Sub(start), err)
		log.Debug(ne)
		if detector.CheckError(err) {
			return dc.detour(b)
		}
		return n, ne
	}
	if err == io.EOF {
		log.Tracef("Read %d bytes from %s %s, EOF", n, dc.addr, dc.stateDesc())
		return
	}
	if detector.CheckResp(b) {
		log.Tracef("Read %d bytes from %s %s, content is hijacked, detour", n, dc.addr, dc.stateDesc())
		return dc.detour(b)
	}
	log.Tracef("Read %d bytes from %s %s, set state to DIRECT", n, dc.addr, dc.stateDesc())
	dc.setState(stateDirect)
	return n, err
}

// Write() implements the function from net.Conn
func (dc *detourConn) Write(b []byte) (n int, err error) {
	if dc.inState(stateInitial) {
		if n, err = dc.writeToBuffer(b); err != nil {
			return n, fmt.Errorf("Unable to write to local buffer: %s", err)
		}
	}
	if n, err = dc.getConn().Write(b); err != nil {
		log.Debugf("Error while write %d bytes to %s %s: %s", len(b), dc.addr, dc.stateDesc(), err)
		return
	}
	log.Tracef("Wrote %d bytes to %s %s", len(b), dc.addr, dc.stateDesc())
	return
}

// Close() implements the function from net.Conn
func (dc *detourConn) Close() error {
	log.Tracef("Closing %s connection to %s", dc.stateDesc(), dc.addr)
	if dc.inState(stateDetour) && wlTemporarily(dc.addr) {
		log.Tracef("no error found till closing, add %s to permanent whitelist", dc.addr)
		addToWl(dc.addr, true)
	}
	return dc.getConn().Close()
}

func (dc *detourConn) LocalAddr() net.Addr {
	return dc.getConn().LocalAddr()
}

func (dc *detourConn) RemoteAddr() net.Addr {
	return dc.getConn().RemoteAddr()
}

func (dc *detourConn) SetDeadline(t time.Time) error {
	dc.SetReadDeadline(t)
	dc.SetWriteDeadline(t)
	return nil
}

func (dc *detourConn) SetReadDeadline(t time.Time) error {
	dc.readDeadline = t
	dc.conn.SetReadDeadline(t)
	return nil
}

func (dc *detourConn) SetWriteDeadline(t time.Time) error {
	dc.writeDeadline = t
	dc.conn.SetWriteDeadline(t)
	return nil
}

func (dc *detourConn) writeToBuffer(b []byte) (n int, err error) {
	dc.muBuf.Lock()
	n, err = dc.buf.Write(b)
	dc.muBuf.Unlock()
	return
}

func (dc *detourConn) resetBuffer() {
	dc.muBuf.Lock()
	dc.buf.Reset()
	dc.muBuf.Unlock()
}

func (dc *detourConn) detour(b []byte) (n int, err error) {
	if err = dc.setupDetour(); err != nil {
		log.Errorf("Error to setup detour: %s", err)
		return
	}
	if _, err = dc.resend(); err != nil {
		err = fmt.Errorf("Error resend buffer to %s: %s", dc.addr, err)
		log.Error(err)
		return
	}
	// should getConn() again as it has changed
	if n, err = dc.getConn().Read(b); err != nil {
		log.Debugf("Read from %s %s still failed: %s", dc.addr, dc.stateDesc(), err)
		return
	}
	dc.setState(stateDetour)
	addToWl(dc.addr, false)
	log.Tracef("Read %d bytes from %s through detour, set state to %s", n, dc.addr, dc.stateDesc())
	return
}

func (dc *detourConn) resend() (int, error) {
	dc.muBuf.Lock()
	b := dc.buf.Bytes()
	dc.muBuf.Unlock()
	if len(b) > 0 {
		log.Tracef("Resending %d buffered bytes to %s", len(b), dc.addr)
		n, err := dc.getConn().Write(b)
		return n, err
	}
	return 0, nil
}

func (dc *detourConn) setupDetour() error {
	c, err := dc.dialDetour("tcp", dc.addr)
	if err != nil {
		return err
	}
	log.Tracef("Dialed a new detour connection to %s", dc.addr)
	dc.setConn(c)
	return nil
}

func (dc *detourConn) getConn() (c net.Conn) {
	dc.muConn.RLock()
	defer dc.muConn.RUnlock()
	return dc.conn
}

func (dc *detourConn) setConn(c net.Conn) {
	dc.muConn.Lock()
	oldConn := dc.conn
	dc.conn = c
	dc.muConn.Unlock()
	dc.conn.SetReadDeadline(dc.readDeadline)
	dc.conn.SetWriteDeadline(dc.writeDeadline)
	log.Tracef("Replaced connection to %s from direct to detour and closing old one", dc.addr)
	oldConn.Close()
}

func (dc *detourConn) stateDesc() string {
	return statesDesc[atomic.LoadUint32(&dc.state)]
}

func (dc *detourConn) inState(s uint32) bool {
	return atomic.LoadUint32(&dc.state) == s
}

func (dc *detourConn) setState(s uint32) {
	atomic.StoreUint32(&dc.state, s)
}
<commit_msg>add trace<commit_after>/*
Package detour provides a net.Conn interface to dial another dialer
if direct connection is considered blocked
It maintains three states in a connection: initial, direct and detoured
along with a temporary whitelist across connections.
it also add a blocked site to whitelist

The action taken and state transistion in each phase is as follows:
+-------------------------+-----------+-------------+-------------+-------------+-------------+
|                         | no error  |   timeout*  | conn reset/ | content     | other error |
|                         |           |             | dns hijack  | hijack      |             |
+-------------------------+-----------+-------------+-------------+-------------+-------------+
| dial (intial)           | noop      | detour      | detour      | n/a         | noop        |
| first read (intial)     | direct    | detour(buf) | detour(buf) | detour(buf) | noop        |
|                         |           | add to tl   | add to tl   | add to tl   |             |
| follow-up read (direct) | direct    | add to tl   | add to tl   | add to tl   | noop        |
| follow-up read (detour) | noop      | rm from tl  | rm from tl  | rm from tl  | rm from tl  |
| close (direct)          | noop      | n/a         | n/a         | n/a         | n/a         |
| close (detour)          | add to wl | n/a         | n/a         | n/a         | n/a         |
+-------------------------+-----------+-------------+-------------+-------------+-------------+
| next dial/read(in tl)***| noop      | rm from tl  | rm from tl  | rm from tl  | rm from tl  |
| next close(in tl)       | add to wl | n/a         | n/a         | n/a         | n/a         |
+-------------------------+-----------+-------------+-------------+-------------+-------------+
(buf) = resend buffer
tl = temporary whitelist
wl = permanent whitelist
* Operation will time out in TimeoutToDetour in initial state,
and at system default or caller supplied deadline for other states;
but in detoured state, it is considered as timeout if an operation takes longer than TimeoutToDetour.
** DNS hijack is only checked at dial time.
*** Connection is always detoured if the site is in tl or wl.
*/
package detour

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
)

// if dial or read exceeded this timeout, we consider switch to detour
// The value depends on OS and browser and defaults to 3s
// For Windows XP, find TcpMaxConnectRetransmissions in
// http://support2.microsoft.com/default.aspx?scid=kb;en-us;314053
var TimeoutToDetour = 3 * time.Second

var (
	log = golog.LoggerFor("detour")

	// instance of Detector
	blockDetector atomic.Value
)

func init() {
	blockDetector.Store(detectorByCountry(""))
}

type dialFunc func(network, addr string) (net.Conn, error)

type detourConn struct {
	muConn sync.RWMutex
	// the actual connection, will change so protect it
	// can't user atomic.Value as the concrete type may vary
	conn net.Conn

	// don't access directly, use inState() and setState() instead
	state uint32

	// the function to dial detour if the site to connect seems blocked
	dialDetour dialFunc

	muBuf sync.Mutex
	// keep track of bytes sent through direct connection
	// so we can resend them when detour
	buf bytes.Buffer

	network, addr string
	readDeadline  time.Time
	writeDeadline time.Time
}

const (
	stateInitial = iota
	stateDirect
	stateDetour
	stateWhitelistCandidate
	stateWhitelist
)

var statesDesc = []string{
	"INITIALLY",
	"DIRECTLY",
	"DETOURED",
	"WHITELIST CANDIDATE",
	"WHITELISTED",
}

func SetCountry(country string) {
	blockDetector.Store(detectorByCountry(country))
}

func Dialer(dialer dialFunc) dialFunc {
	return func(network, addr string) (conn net.Conn, err error) {
		dc := &detourConn{dialDetour: dialer, network: network, addr: addr}
		if !whitelisted(addr) {
			detector := blockDetector.Load().(*Detector)
			dc.setState(stateInitial)
			dc.conn, err = net.DialTimeout(network, addr, TimeoutToDetour)
			if err == nil {
				if !detector.CheckConn(dc.conn) {
					log.Tracef("Dial %s to %s succeeded", dc.stateDesc(), addr)
					return dc, nil
				}
				log.Debugf("Dial %s to %s, dns hijacked, try detour", dc.stateDesc(), addr)
			} else if detector.CheckError(err) {
				log.Debugf("Dial %s to %s failed, try detour: %s", dc.stateDesc(), addr, err)
			} else {
				log.Errorf("Dial %s to %s failed: %s", dc.stateDesc(), addr, err)
				return dc, err
			}
		}
		dc.conn, err = dc.dialDetour(network, addr)
		if err != nil {
			log.Errorf("Dial %s to %s failed", dc.stateDesc(), addr)
			return nil, err
		}
		dc.setState(stateDetour)
		log.Tracef("Dial %s to %s succeeded", dc.stateDesc(), addr)
		if !whitelisted(addr) {
			log.Tracef("Add %s to whitelist", dc.stateDesc(), addr)
			addToWl(dc.addr, false)
		}
		return dc, err
	}
}

// Read() implements the function from net.Conn
func (dc *detourConn) Read(b []byte) (n int, err error) {
	conn := dc.getConn()
	detector := blockDetector.Load().(*Detector)
	if !dc.inState(stateInitial) {
		if n, err = conn.Read(b); err != nil {
			if err == io.EOF {
				log.Tracef("Read %d bytes from %s %s, EOF", n, dc.addr, dc.stateDesc())
				return
			}
			log.Tracef("Read from %s %s failed: %s", dc.addr, dc.stateDesc(), err)
			if dc.inState(stateDirect) && detector.CheckError(err) {
				log.Tracef("Seems %s still blocked, add to whitelist so will try detour next time", dc.addr)
				addToWl(dc.addr, false)
			} else if dc.inState(stateDetour) && wlTemporarily(dc.addr) {
				log.Tracef("Detoured route is still not reliable for %s, not whitelist it", dc.addr)
				removeFromWl(dc.addr)
			}
			return
		}
		if dc.inState(stateDirect) && detector.CheckResp(b) {
			log.Tracef("Seems %s still hijacked, add to whitelist so will try detour next time", dc.addr)
			addToWl(dc.addr, false)
		}
		log.Tracef("Read %d bytes from %s %s", n, dc.addr, dc.stateDesc())
		return n, err
	}
	// state will always be settled after first read, safe to clear buffer at end of it
	defer dc.resetBuffer()
	start := time.Now()
	dl := start.Add(TimeoutToDetour)
	if !dc.readDeadline.IsZero() && dc.readDeadline.Sub(start) < 2*TimeoutToDetour {
		log.Tracef("no time left to test %s, read %s", dc.addr, stateDirect)
		dc.setState(stateDirect)
		return conn.Read(b)
	}

	conn.SetReadDeadline(dl)
	n, err = conn.Read(b)
	conn.SetReadDeadline(dc.readDeadline)
	if err != nil && err != io.EOF {
		ne := fmt.Errorf("Error while read from %s %s, takes %s: %s", dc.addr, dc.stateDesc(), time.Now().Sub(start), err)
		log.Debug(ne)
		if detector.CheckError(err) {
			return dc.detour(b)
		}
		return n, ne
	}
	if err == io.EOF {
		log.Tracef("Read %d bytes from %s %s, EOF", n, dc.addr, dc.stateDesc())
		return
	}
	if detector.CheckResp(b) {
		log.Tracef("Read %d bytes from %s %s, content is hijacked, detour", n, dc.addr, dc.stateDesc())
		return dc.detour(b)
	}
	log.Tracef("Read %d bytes from %s %s, set state to DIRECT", n, dc.addr, dc.stateDesc())
	dc.setState(stateDirect)
	return n, err
}

// Write() implements the function from net.Conn
func (dc *detourConn) Write(b []byte) (n int, err error) {
	if dc.inState(stateInitial) {
		if n, err = dc.writeToBuffer(b); err != nil {
			return n, fmt.Errorf("Unable to write to local buffer: %s", err)
		}
	}
	if n, err = dc.getConn().Write(b); err != nil {
		log.Debugf("Error while write %d bytes to %s %s: %s", len(b), dc.addr, dc.stateDesc(), err)
		return
	}
	log.Tracef("Wrote %d bytes to %s %s", len(b), dc.addr, dc.stateDesc())
	return
}

// Close() implements the function from net.Conn
func (dc *detourConn) Close() error {
	log.Tracef("Closing %s connection to %s", dc.stateDesc(), dc.addr)
	if dc.inState(stateDetour) && wlTemporarily(dc.addr) {
		log.Tracef("no error found till closing, add %s to permanent whitelist", dc.addr)
		addToWl(dc.addr, true)
	}
	return dc.getConn().Close()
}

func (dc *detourConn) LocalAddr() net.Addr {
	return dc.getConn().LocalAddr()
}

func (dc *detourConn) RemoteAddr() net.Addr {
	return dc.getConn().RemoteAddr()
}

func (dc *detourConn) SetDeadline(t time.Time) error {
	dc.SetReadDeadline(t)
	dc.SetWriteDeadline(t)
	return nil
}

func (dc *detourConn) SetReadDeadline(t time.Time) error {
	dc.readDeadline = t
	dc.conn.SetReadDeadline(t)
	return nil
}

func (dc *detourConn) SetWriteDeadline(t time.Time) error {
	dc.writeDeadline = t
	dc.conn.SetWriteDeadline(t)
	return nil
}

func (dc *detourConn) writeToBuffer(b []byte) (n int, err error) {
	dc.muBuf.Lock()
	n, err = dc.buf.Write(b)
	dc.muBuf.Unlock()
	return
}

func (dc *detourConn) resetBuffer() {
	dc.muBuf.Lock()
	dc.buf.Reset()
	dc.muBuf.Unlock()
}

func (dc *detourConn) detour(b []byte) (n int, err error) {
	if err = dc.setupDetour(); err != nil {
		log.Errorf("Error to setup detour: %s", err)
		return
	}
	if _, err = dc.resend(); err != nil {
		err = fmt.Errorf("Error resend buffer to %s: %s", dc.addr, err)
		log.Error(err)
		return
	}
	// should getConn() again as it has changed
	if n, err = dc.getConn().Read(b); err != nil {
		log.Debugf("Read from %s %s still failed: %s", dc.addr, dc.stateDesc(), err)
		return
	}
	dc.setState(stateDetour)
	addToWl(dc.addr, false)
	log.Tracef("Read %d bytes from %s through detour, set state to %s", n, dc.addr, dc.stateDesc())
	return
}

func (dc *detourConn) resend() (int, error) {
	dc.muBuf.Lock()
	b := dc.buf.Bytes()
	dc.muBuf.Unlock()
	if len(b) > 0 {
		log.Tracef("Resending %d buffered bytes to %s", len(b), dc.addr)
		n, err := dc.getConn().Write(b)
		return n, err
	}
	return 0, nil
}

func (dc *detourConn) setupDetour() error {
	c, err := dc.dialDetour("tcp", dc.addr)
	if err != nil {
		return err
	}
	log.Tracef("Dialed a new detour connection to %s", dc.addr)
	dc.setConn(c)
	return nil
}

func (dc *detourConn) getConn() (c net.Conn) {
	dc.muConn.RLock()
	defer dc.muConn.RUnlock()
	return dc.conn
}

func (dc *detourConn) setConn(c net.Conn) {
	dc.muConn.Lock()
	oldConn := dc.conn
	dc.conn = c
	dc.muConn.Unlock()
	dc.conn.SetReadDeadline(dc.readDeadline)
	dc.conn.SetWriteDeadline(dc.writeDeadline)
	log.Tracef("Replaced connection to %s from direct to detour and closing old one", dc.addr)
	oldConn.Close()
}

func (dc *detourConn) stateDesc() string {
	return statesDesc[atomic.LoadUint32(&dc.state)]
}

func (dc *detourConn) inState(s uint32) bool {
	return atomic.LoadUint32(&dc.state) == s
}

func (dc *detourConn) setState(s uint32) {
	atomic.StoreUint32(&dc.state, s)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poll

import (
	"internal/syscall/unix"
	"sync/atomic"
	"syscall"
)

var copyFileRangeSupported int32 = 1 // accessed atomically

const maxCopyFileRangeRound = 1 << 30

// CopyFileRange copies at most remain bytes of data from src to dst, using
// the copy_file_range system call. dst and src must refer to regular files.
func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) {
	if atomic.LoadInt32(&copyFileRangeSupported) == 0 {
		return 0, false, nil
	}
	for remain > 0 {
		max := remain
		if max > maxCopyFileRangeRound {
			max = maxCopyFileRangeRound
		}
		n, err := copyFileRange(dst, src, int(max))
		switch err {
		case syscall.ENOSYS:
			// copy_file_range(2) was introduced in Linux 4.5.
			// Go supports Linux >= 2.6.33, so the system call
			// may not be present.
			//
			// If we see ENOSYS, we have certainly not transfered
			// any data, so we can tell the caller that we
			// couldn't handle the transfer and let them fall
			// back to more generic code.
			//
			// Seeing ENOSYS also means that we will not try to
			// use copy_file_range(2) again.
			atomic.StoreInt32(&copyFileRangeSupported, 0)
			return 0, false, nil
		case syscall.EXDEV, syscall.EINVAL, syscall.EOPNOTSUPP, syscall.EPERM:
			// Prior to Linux 5.3, it was not possible to
			// copy_file_range across file systems. Similarly to
			// the ENOSYS case above, if we see EXDEV, we have
			// not transfered any data, and we can let the caller
			// fall back to generic code.
			//
			// As for EINVAL, that is what we see if, for example,
			// dst or src refer to a pipe rather than a regular
			// file. This is another case where no data has been
			// transfered, so we consider it unhandled.
			//
			// If the file is on NFS, we can see EOPNOTSUPP.
			// See issue #40731.
			//
			// If the process is running inside a Docker container,
			// we might see EPERM instead of ENOSYS. See issue
			// #40893. Since EPERM might also be a legitimate error,
			// don't mark copy_file_range(2) as unsupported.
			return 0, false, nil
		case nil:
			if n == 0 {
				// src is at EOF, which means we are done.
				return written, true, nil
			}
			remain -= n
			written += n
		default:
			return written, true, err
		}
	}
	return written, true, nil
}

// copyFileRange performs one round of copy_file_range(2).
func copyFileRange(dst, src *FD, max int) (written int64, err error) {
	// The signature of copy_file_range(2) is:
	//
	// ssize_t copy_file_range(int fd_in, loff_t *off_in,
	//                         int fd_out, loff_t *off_out,
	//                         size_t len, unsigned int flags);
	//
	// Note that in the call to unix.CopyFileRange below, we use nil
	// values for off_in and off_out. For the system call, this means
	// "use and update the file offsets". That is why we must acquire
	// locks for both file descriptors (and why this whole machinery is
	// in the internal/poll package to begin with).
	if err := dst.writeLock(); err != nil {
		return 0, err
	}
	defer dst.writeUnlock()
	if err := src.readLock(); err != nil {
		return 0, err
	}
	defer src.readUnlock()
	var n int
	for {
		n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0)
		if err != syscall.EINTR {
			break
		}
	}
	return int64(n), err
}
<commit_msg>internal/poll: treat copy_file_range EIO as not-handled<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poll

import (
	"internal/syscall/unix"
	"sync/atomic"
	"syscall"
)

var copyFileRangeSupported int32 = 1 // accessed atomically

const maxCopyFileRangeRound = 1 << 30

// CopyFileRange copies at most remain bytes of data from src to dst, using
// the copy_file_range system call. dst and src must refer to regular files.
func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) {
	if atomic.LoadInt32(&copyFileRangeSupported) == 0 {
		return 0, false, nil
	}
	for remain > 0 {
		max := remain
		if max > maxCopyFileRangeRound {
			max = maxCopyFileRangeRound
		}
		n, err := copyFileRange(dst, src, int(max))
		switch err {
		case syscall.ENOSYS:
			// copy_file_range(2) was introduced in Linux 4.5.
			// Go supports Linux >= 2.6.33, so the system call
			// may not be present.
			//
			// If we see ENOSYS, we have certainly not transfered
			// any data, so we can tell the caller that we
			// couldn't handle the transfer and let them fall
			// back to more generic code.
			//
			// Seeing ENOSYS also means that we will not try to
			// use copy_file_range(2) again.
			atomic.StoreInt32(&copyFileRangeSupported, 0)
			return 0, false, nil
		case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM:
			// Prior to Linux 5.3, it was not possible to
			// copy_file_range across file systems. Similarly to
			// the ENOSYS case above, if we see EXDEV, we have
			// not transfered any data, and we can let the caller
			// fall back to generic code.
			//
			// As for EINVAL, that is what we see if, for example,
			// dst or src refer to a pipe rather than a regular
			// file. This is another case where no data has been
			// transfered, so we consider it unhandled.
			//
			// If src and dst are on CIFS, we can see EIO.
			// See issue #42334.
			//
			// If the file is on NFS, we can see EOPNOTSUPP.
			// See issue #40731.
			//
			// If the process is running inside a Docker container,
			// we might see EPERM instead of ENOSYS. See issue
			// #40893. Since EPERM might also be a legitimate error,
			// don't mark copy_file_range(2) as unsupported.
			return 0, false, nil
		case nil:
			if n == 0 {
				// src is at EOF, which means we are done.
				return written, true, nil
			}
			remain -= n
			written += n
		default:
			return written, true, err
		}
	}
	return written, true, nil
}

// copyFileRange performs one round of copy_file_range(2).
func copyFileRange(dst, src *FD, max int) (written int64, err error) {
	// The signature of copy_file_range(2) is:
	//
	// ssize_t copy_file_range(int fd_in, loff_t *off_in,
	//                         int fd_out, loff_t *off_out,
	//                         size_t len, unsigned int flags);
	//
	// Note that in the call to unix.CopyFileRange below, we use nil
	// values for off_in and off_out. For the system call, this means
	// "use and update the file offsets". That is why we must acquire
	// locks for both file descriptors (and why this whole machinery is
	// in the internal/poll package to begin with).
	if err := dst.writeLock(); err != nil {
		return 0, err
	}
	defer dst.writeUnlock()
	if err := src.readLock(); err != nil {
		return 0, err
	}
	defer src.readUnlock()
	var n int
	for {
		n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0)
		if err != syscall.EINTR {
			break
		}
	}
	return int64(n), err
}
<|endoftext|>


<commit_before>package util

// reference: https://github.com/mohae/deepcopy
import (
	"reflect"
)

func DeepClone(v interface{}) interface{} {
	dst := reflect.New(reflect.TypeOf(v)).Elem()
	deepCopy(dst, reflect.ValueOf(v))
	return dst.Interface()
}

func deepCopy(dst, src reflect.Value) {
	switch src.Kind() {
	case reflect.Interface:
		deepCopy(dst.Elem(), src.Elem())
	case reflect.Ptr:
		value := src.Elem()
		if !value.IsValid() {
			return
		}
		dst.Set(reflect.New(value.Type()))
		deepCopy(dst.Elem(), value)
	case reflect.Map:
		dst.Set(reflect.MakeMap(src.Type()))
		keys := src.MapKeys()
		for _, key := range keys {
			value := src.MapIndex(key)
			newValue := reflect.New(value.Type()).Elem()
			deepCopy(newValue, value)
			dst.SetMapIndex(key, newValue)
		}
	case reflect.Slice:
		dst.Set(reflect.MakeSlice(src.Type(), src.Len(), src.Cap()))
		for i := 0; i < src.Len(); i++ {
			deepCopy(dst.Index(i), src.Index(i))
		}
	case reflect.Struct:
		for i := 0; i < src.NumField(); i++ {
			deepCopy(dst.Field(i), src.Field(i))
		}
	default:
		dst.Set(src)
	}
}
<commit_msg>bug fix: interface<commit_after>package util

// reference: https://github.com/mohae/deepcopy
import (
	"reflect"
)

func DeepClone(v interface{}) interface{} {
	dst := reflect.New(reflect.TypeOf(v)).Elem()
	deepCopy(dst, reflect.ValueOf(v))
	return dst.Interface()
}

func deepCopy(dst, src reflect.Value) {
	switch src.Kind() {
	case reflect.Interface:
		value := src.Elem()
		if !value.IsValid() {
			return
		}
		newValue := reflect.New(value.Type()).Elem()
		deepCopy(newValue, value)
		dst.Set(newValue)
	case reflect.Ptr:
		value := src.Elem()
		if !value.IsValid() {
			return
		}
		dst.Set(reflect.New(value.Type()))
		deepCopy(dst.Elem(), value)
	case reflect.Map:
		dst.Set(reflect.MakeMap(src.Type()))
		keys := src.MapKeys()
		for _, key := range keys {
			value := src.MapIndex(key)
			newValue := reflect.New(value.Type()).Elem()
			deepCopy(newValue, value)
			dst.SetMapIndex(key, newValue)
		}
	case reflect.Slice:
		dst.Set(reflect.MakeSlice(src.Type(), src.Len(), src.Cap()))
		for i := 0; i < src.Len(); i++ {
			deepCopy(dst.Index(i), src.Index(i))
		}
	case reflect.Struct:
		for i := 0; i < src.NumField(); i++ {
			deepCopy(dst.Field(i), src.Field(i))
		}
	default:
		dst.Set(src)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package util

import "testing"

func TestAssertEquals(t *testing.T) {
	v := 1
	AssertEquals(t, v, v)
	if t.Failed() {
		t.Errorf("AssertEquals(t, %d, %d) fails, want %d equals %d", v, v, v, v)
	}
}
<commit_msg>Change error message of AssertEquals test<commit_after>// Copyright 2014 Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package util

import "testing"

func TestAssertEquals(t *testing.T) {
	v := 1
	AssertEquals(t, v, v)
	if t.Failed() {
		t.Errorf("AssertEquals(t, %d, %d) fails; want %d equals %d", v, v, v, v)
	}
}
<|endoftext|>


<commit_before>package util

import (
	"testing"
	"net/http"
)

type testObj struct {
	Name string `json:"name"`
	UrlType string `json:"url_type"`
	Normal map[string]interface{} `json:"normal"`
	RunList []string `json:"run_list"`
}

func (to *testObj) GetName() string {
	return to.Name
}

func (to *testObj) URLType() string {
	return to.UrlType
}

// The strange URLs are because the config doesn't get parsed here, so it ends
// up using the really-really default settings.

func TestObjURL(t *testing.T){
	obj := &testObj{ Name: "foo", UrlType: "bar" }
	url := ObjURL(obj)
	expectedUrl := "http://:0/bar/foo"
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
}

func TestCustomObjUrl(t *testing.T){
	obj := &testObj{ Name: "foo", UrlType: "bar" }
	url := CustomObjURL(obj, "/baz")
	expectedUrl := "http://:0/bar/foo/baz"
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
}

func TestCustomURL(t *testing.T){
	initUrl := "/foo/bar"
	url := CustomURL(initUrl)
	expectedUrl := "http://:0/foo/bar"
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
	initUrl = "foo/bar"
	url = CustomURL(initUrl)
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
}

func TestGerror(t *testing.T){
	errmsg := "foo bar"
	err := Errorf(errmsg)
	if err.Error() != errmsg {
		t.Errorf("expected %s to match %s", err.Error(), errmsg)
	}
	if err.Status() != http.StatusBadRequest {
		t.Errorf("err.Status() did not return expected default")
	}
	err.SetStatus(http.StatusNotFound)
	if err.Status() != http.StatusNotFound {
		t.Errorf("SetStatus did not set Status correctly")
	}
}

func TestFlatten(t *testing.T){
	rl := []string{ "recipe[foo]", "role[bar]" }
	normmap := make(map[string]interface{})
	normmap["foo"] = "bar"
	normmap["baz"] = "buz"
	normmap["slice"] = []string{ "fee", "fie", "fo" }
	normmap["map"] = make(map[string]interface{})
	normmap["map"].(map[string]interface{})["first"] = "mook"
	normmap["map"].(map[string]interface{})["second"] = "nork"
	obj := &testObj{ Name: "foo", UrlType: "bar", RunList: rl, Normal: normmap }
	flattened := FlattenObj(obj)
	if _, ok := flattened["name"]; !ok {
		t.Errorf("obj name was not flattened correctly")
	}
	if flattened["name"].(string) != obj.Name {
		t.Errorf("flattened name not correct, wanted %s got %v", obj.Name, flattened["name"])
	}
	if _, ok := flattened["foo"]; !ok {
		t.Errorf("Foo should have been set, but it wasn't")
	}
	if _, ok := flattened["normal"]; ok {
		t.Errorf("The 'normal' field was set, but shouldn't have been.")
	}
	if _, ok := flattened["map_first"]; !ok {
		t.Errorf("normal -> map -> second should have been flattened to map_second, but it wasn't")
	}
	if r, ok := flattened["recipe"]; ok {
		if r.([]string)[0] != "foo" {
			t.Errorf("recipe list should have included foo, but it had %v instead", r.([]string)[0])
		}
	} else {
		t.Errorf("No recipe list")
	}
	if r, ok := flattened["role"]; ok {
		if r.([]string)[0] != "bar" {
			t.Errorf("role list should have included bar, but it had %v instead", r.([]string)[0])
		}
	} else {
		t.Errorf("No role list")
	}
}

func TestMapify(t *testing.T){
	rl := []string{ "recipe[foo]", "role[bar]" }
	normmap := make(map[string]interface{})
	normmap["foo"] = "bar"
	normmap["baz"] = "buz"
	normmap["slice"] = []string{ "fee", "fie", "fo" }
	normmap["map"] = make(map[string]interface{})
	normmap["map"].(map[string]interface{})["first"] = "mook"
	normmap["map"].(map[string]interface{})["second"] = "nork"
	obj := &testObj{ Name: "foo", UrlType: "bar", RunList: rl, Normal: normmap }
	mapify := MapifyObject(obj)
	if mapify["name"].(string) != obj.Name {
		t.Errorf("Mapify names didn't match, expecte %s, got %v", obj.Name, mapify["name"])
	}
	if _, ok := mapify["normal"]; !ok {
		t.Errorf("There should have been a normal key for the map")
	}
	if _, ok := mapify["foo"]; ok {
		t.Errorf("There was a foo key in mapify, and there should not have been.")
	}
}

func TestIndexify(t *testing.T){
	rl := []string{ "recipe[foo]", "role[bar]" }
	normmap := make(map[string]interface{})
	normmap["foo"] = "bar"
	normmap["baz"] = "buz"
	normmap["slice"] = []string{ "fee", "fie", "fo" }
	normmap["map"] = make(map[string]interface{})
	normmap["map"].(map[string]interface{})["first"] = "mook"
	normmap["map"].(map[string]interface{})["second"] = "nork"
	obj := &testObj{ Name: "foo", UrlType: "bar", RunList: rl, Normal: normmap }
	flatten := FlattenObj(obj)
	indexificate := Indexify(flatten)
	if indexificate[0] != "baz:buz" {
		t.Errorf("The first element of the indexified object should have been 'baz:buz', but instead it was %s", indexificate[0])
	}
}

func TestValidateName(t *testing.T){
	goodName := "foo-bar"
	badName := "FAh!!"
	if !ValidateName(goodName){
		t.Errorf("%s should have passed name validation, but didn't", goodName)
	}
	if ValidateName(badName){
		t.Errorf("%s should not have passed name validation, but somehow did", badName)
	}
}
<commit_msg>Getting moving along on tests again finally<commit_after>package util

import (
	"testing"
	"net/http"
)

type testObj struct {
	Name string `json:"name"`
	UrlType string `json:"url_type"`
	Normal map[string]interface{} `json:"normal"`
	RunList []string `json:"run_list"`
}

func (to *testObj) GetName() string {
	return to.Name
}

func (to *testObj) URLType() string {
	return to.UrlType
}

// The strange URLs are because the config doesn't get parsed here, so it ends
// up using the really-really default settings.

func TestObjURL(t *testing.T){
	obj := &testObj{ Name: "foo", UrlType: "bar" }
	url := ObjURL(obj)
	expectedUrl := "http://:0/bar/foo"
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
}

func TestCustomObjUrl(t *testing.T){
	obj := &testObj{ Name: "foo", UrlType: "bar" }
	url := CustomObjURL(obj, "/baz")
	expectedUrl := "http://:0/bar/foo/baz"
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
}

func TestCustomURL(t *testing.T){
	initUrl := "/foo/bar"
	url := CustomURL(initUrl)
	expectedUrl := "http://:0/foo/bar"
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
	initUrl = "foo/bar"
	url = CustomURL(initUrl)
	if url != expectedUrl {
		t.Errorf("expected %s, got %s", expectedUrl, url)
	}
}

func TestGerror(t *testing.T){
	errmsg := "foo bar"
	err := Errorf(errmsg)
	if err.Error() != errmsg {
		t.Errorf("expected %s to match %s", err.Error(), errmsg)
	}
	if err.Status() != http.StatusBadRequest {
		t.Errorf("err.Status() did not return expected default")
	}
	err.SetStatus(http.StatusNotFound)
	if err.Status() != http.StatusNotFound {
		t.Errorf("SetStatus did not set Status correctly")
	}
}

func TestFlatten(t *testing.T){
	rl := []string{ "recipe[foo]", "role[bar]" }
	normmap := make(map[string]interface{})
	normmap["foo"] = "bar"
	normmap["baz"] = "buz"
	normmap["slice"] = []string{ "fee", "fie", "fo" }
	normmap["map"] = make(map[string]interface{})
	normmap["map"].(map[string]interface{})["first"] = "mook"
	normmap["map"].(map[string]interface{})["second"] = "nork"
	obj := &testObj{ Name: "foo", UrlType: "bar", RunList: rl, Normal: normmap }
	flattened := FlattenObj(obj)
	if _, ok := flattened["name"]; !ok {
		t.Errorf("obj name was not flattened correctly")
	}
	if flattened["name"].(string) != obj.Name {
		t.Errorf("flattened name not correct, wanted %s got %v", obj.Name, flattened["name"])
	}
	if _, ok := flattened["foo"]; !ok {
		t.Errorf("Foo should have been set, but it wasn't")
	}
	if _, ok := flattened["normal"]; ok {
		t.Errorf("The 'normal' field was set, but shouldn't have been.")
	}
	if _, ok := flattened["map_first"]; !ok {
		t.Errorf("normal -> map -> second should have been flattened to map_second, but it wasn't")
	}
	if r, ok := flattened["recipe"]; ok {
		if r.([]string)[0] != "foo" {
			t.Errorf("recipe list should have included foo, but it had %v instead", r.([]string)[0])
		}
	} else {
		t.Errorf("No recipe list")
	}
	if r, ok := flattened["role"]; ok {
		if r.([]string)[0] != "bar" {
			t.Errorf("role list should have included bar, but it had %v instead", r.([]string)[0])
		}
	} else {
		t.Errorf("No role list")
	}
}

func TestMapify(t *testing.T){
	rl := []string{ "recipe[foo]", "role[bar]" }
	normmap := make(map[string]interface{})
	normmap["foo"] = "bar"
	normmap["baz"] = "buz"
	normmap["slice"] = []string{ "fee", "fie", "fo" }
	normmap["map"] = make(map[string]interface{})
	normmap["map"].(map[string]interface{})["first"] = "mook"
	normmap["map"].(map[string]interface{})["second"] = "nork"
	obj := &testObj{ Name: "foo", UrlType: "bar", RunList: rl, Normal: normmap }
	mapify := MapifyObject(obj)
	if mapify["name"].(string) != obj.Name {
		t.Errorf("Mapify names didn't match, expecte %s, got %v", obj.Name, mapify["name"])
	}
	if _, ok := mapify["normal"]; !ok {
		t.Errorf("There should have been a normal key for the map")
	}
	if _, ok := mapify["foo"]; ok {
		t.Errorf("There was a foo key in mapify, and there should not have been.")
	}
}

func TestIndexify(t *testing.T){
	rl := []string{ "recipe[foo]", "role[bar]" }
	normmap := make(map[string]interface{})
	normmap["foo"] = "bar"
	normmap["baz"] = "buz"
	normmap["slice"] = []string{ "fee", "fie", "fo" }
	normmap["map"] = make(map[string]interface{})
	normmap["map"].(map[string]interface{})["first"] = "mook"
	normmap["map"].(map[string]interface{})["second"] = "nork"
	obj := &testObj{ Name: "foo", UrlType: "bar", RunList: rl, Normal: normmap }
	flatten := FlattenObj(obj)
	indexificate := Indexify(flatten)
	if indexificate[0] != "baz:buz" {
		t.Errorf("The first element of the indexified object should have been 'baz:buz', but instead it was %s", indexificate[0])
	}
}

func TestValidateName(t *testing.T){
	goodName := "foo-bar.baz"
	badName := "FAh!!"
	if !ValidateName(goodName){
		t.Errorf("%s should have passed name validation, but didn't", goodName)
	}
	if ValidateName(badName){
		t.Errorf("%s should not have passed name validation, but somehow did", badName)
	}
}

func TestValidateDBagName(t *testing.T){
	goodName := "foo-bar"
	badName := "FaH!!"
	if !ValidateName(goodName){
		t.Errorf("%s should have passed data bag name validation, but didn't", goodName)
	}
	if ValidateName(badName){
		t.Errorf("%s should not have passed data bag name validation, but somehow did", badName)
	}
}

func TestValidateEnvName(t *testing.T){
	goodName := "foo-bar"
	badName := "FAh!!"
	if !ValidateName(goodName){
		t.Errorf("%s should have passed env name validation, but didn't", goodName)
	}
	if ValidateName(badName){
		t.Errorf("%s should not have passed env name validation, but somehow did", badName)
	}
}

// A lot of the validations get taken care of with chef pedant, honestly
func TestValidateAsVersion(t *testing.T){
	goodVersion := "1.0.0"
	goodVersion2 := "1.0"
	badVer1 := "1"
	badVer2 := "foo"
	var badVer3 interface{}
	badVer3 = nil

	if _, err := ValidateAsVersion(goodVersion); err != nil {
		t.Errorf("%s should have passed version validation, but didn't", goodVersion)
	}
	if _, err := ValidateAsVersion(goodVersion2); err != nil {
		t.Errorf("%s should have passed version validation, but didn't", goodVersion2)
	}
	if _, err := ValidateAsVersion(badVer1); err == nil {
		t.Errorf("%s should not have passed version validation, but did", badVer1)
	}
	if _, err := ValidateAsVersion(badVer2); err == nil {
		t.Errorf("%s should not have passed version validation, but did", badVer2)
	}
	if v, err := ValidateAsVersion(badVer3); err != nil {
		t.Errorf("nil should have passed version validation, but did")
	} else if v != "0.0.0"{
		t.Errorf("Should have come back as 0.0.0, but it came back as %v", v)
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"html/template"
	"net/http"
	"path/filepath"

	"github.com/UniversityRadioYork/2016-site/structs"
	myradio "github.com/UniversityRadioYork/myradio-go"
)

// TemplatePrefix is the constant containing the filepath prefix for templates.
const TemplatePrefix = "views"

// BaseTemplates is the array of 'base' templates used in each template render.
var BaseTemplates = []string{
	"partials/header.tmpl",
	"partials/footer.tmpl",
	"elements/navbar.tmpl",
	"partials/base.tmpl",
}

// RenderTemplate renders a 2016site template on the ResponseWriter w.
//
// This function automatically adds in the 2016site base templates, performs
// error handling, and builds a global context.
//
// The PageContext context gives the context for the page to be rendered, sent
// to the template as PageContext.
// The interface{} data gives the data to be sent to the template as PageData.
//
// The string mainTmpl gives the name, relative to views, of the main
// template to render.  The variadic argument addTmpls names any additional
// templates mainTmpl depends on.
//
// RenderTemplate returns any error that occurred when rendering the template.
func RenderTemplate(w http.ResponseWriter, context structs.PageContext, data interface{}, mainTmpl string, addTmpls ...string) error {
	var err error

	td := structs.Globals{
		PageContext: context,
		PageData:    data,
	}

	ownTmpls := append(addTmpls, mainTmpl)
	baseTmpls := append(BaseTemplates, ownTmpls...)

	var tmpls []string
	for _, baseTmpl := range baseTmpls {
		tmpls = append(tmpls, filepath.Join(TemplatePrefix, baseTmpl))
	}

	t := template.New("base.tmpl")
	t.Funcs(template.FuncMap{
		"html": renderHTML,
		"limitShowMeta": func(a []myradio.ShowMeta, start int, end int) []myradio.ShowMeta {
			if len(a) < end {
				return a[start:]
			} else if len(a) < start {
				return nil
			}
			return a[start:end]

		},
		//Takes a splice of seasons and returns the total number of episodes
		"showCount": func(seasons []myradio.Season) int {
			var c = 0
			for _, season := range seasons {
				//Something about JSON being read as a float 64 so needing to convert to an int
				c += int(season.NumEpisodes.Value.(float64))
			}
			return c
		},
		"showsToHours": func(shows []myradio.ShowMeta) int {
			//TODO: finish This
			return 10
		},
	})
	t, err = t.ParseFiles(tmpls...)
	if err != nil {
		return err
	}

	return t.Execute(w, td)
}

// renderHTML takes some html as a string and returns a template.HTML
//
// Handles plain text gracefully.
func renderHTML(value interface{}) template.HTML {
	return template.HTML(fmt.Sprint(value))
}
<commit_msg>make it clear that the shows to hours method is not yet complete<commit_after>package utils

import (
	"fmt"
	"html/template"
	"net/http"
	"path/filepath"

	"github.com/UniversityRadioYork/2016-site/structs"
	myradio "github.com/UniversityRadioYork/myradio-go"
)

// TemplatePrefix is the constant containing the filepath prefix for templates.
const TemplatePrefix = "views"

// BaseTemplates is the array of 'base' templates used in each template render.
var BaseTemplates = []string{
	"partials/header.tmpl",
	"partials/footer.tmpl",
	"elements/navbar.tmpl",
	"partials/base.tmpl",
}

// RenderTemplate renders a 2016site template on the ResponseWriter w.
//
// This function automatically adds in the 2016site base templates, performs
// error handling, and builds a global context.
//
// The PageContext context gives the context for the page to be rendered, sent
// to the template as PageContext.
// The interface{} data gives the data to be sent to the template as PageData.
//
// The string mainTmpl gives the name, relative to views, of the main
// template to render.  The variadic argument addTmpls names any additional
// templates mainTmpl depends on.
//
// RenderTemplate returns any error that occurred when rendering the template.
func RenderTemplate(w http.ResponseWriter, context structs.PageContext, data interface{}, mainTmpl string, addTmpls ...string) error {
	var err error

	td := structs.Globals{
		PageContext: context,
		PageData:    data,
	}

	ownTmpls := append(addTmpls, mainTmpl)
	baseTmpls := append(BaseTemplates, ownTmpls...)

	var tmpls []string
	for _, baseTmpl := range baseTmpls {
		tmpls = append(tmpls, filepath.Join(TemplatePrefix, baseTmpl))
	}

	t := template.New("base.tmpl")
	t.Funcs(template.FuncMap{
		"html": renderHTML,
		"limitShowMeta": func(a []myradio.ShowMeta, start int, end int) []myradio.ShowMeta {
			if len(a) < end {
				return a[start:]
			} else if len(a) < start {
				return nil
			}
			return a[start:end]

		},
		//Takes a splice of seasons and returns the total number of episodes
		"showCount": func(seasons []myradio.Season) int {
			var c = 0
			for _, season := range seasons {
				//Something about JSON being read as a float 64 so needing to convert to an int
				c += int(season.NumEpisodes.Value.(float64))
			}
			return c
		},
		"showsToHours": func(shows []myradio.ShowMeta) int {
			//TODO: finish This
			return -5
		},
	})
	t, err = t.ParseFiles(tmpls...)
	if err != nil {
		return err
	}

	return t.Execute(w, td)
}

// renderHTML takes some html as a string and returns a template.HTML
//
// Handles plain text gracefully.
func renderHTML(value interface{}) template.HTML {
	return template.HTML(fmt.Sprint(value))
}
<|endoftext|>


<commit_before>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package cmd

import (
	"fmt"
	"net/http"

	"github.com/Unknwon/log"
	"github.com/codegangsta/cli"
	"github.com/go-macaron/i18n"
	"github.com/go-macaron/pongo2"
	"gopkg.in/macaron.v1"

	"github.com/peachdocs/peach/models"
	"github.com/peachdocs/peach/modules/middleware"
	"github.com/peachdocs/peach/modules/setting"
	"github.com/peachdocs/peach/routers"
)

var Web = cli.Command{
	Name:   "web",
	Usage:  "Start Peach web server",
	Action: runWeb,
	Flags: []cli.Flag{
		stringFlag("config, c", "custom/app.ini", "Custom configuration file path"),
	},
}

func runWeb(ctx *cli.Context) {
	if ctx.IsSet("config") {
		setting.CustomConf = ctx.String("config")
	}
	setting.NewContext()
	models.NewContext()

	log.Info("Peach %s", setting.AppVer)

	m := macaron.New()
	m.Use(macaron.Logger())
	m.Use(macaron.Recovery())
	m.Use(macaron.Statics(macaron.StaticOptions{
		SkipLogging: setting.ProdMode,
	}, "custom/public", "public", models.HTMLRoot))
	m.Use(i18n.I18n(i18n.Options{
		Files:       setting.Docs.Locales,
		DefaultLang: setting.Docs.Langs[0],
	}))
	tplDir := "templates"
	if setting.Page.UseCustomTpl {
		tplDir = "custom/templates"
	}
	m.Use(pongo2.Pongoer(pongo2.Options{
		Directory: tplDir,
	}))
	m.Use(middleware.Contexter())

	m.Get("/", routers.Home)
	m.Get("/docs", routers.Docs)
	m.Get("/docs/images/*", routers.DocsStatic)
	m.Get("/docs/*", routers.Protect, routers.Docs)
	m.Post("/hook", routers.Hook)
	m.Get("/search", routers.Search)
	m.Get("/*", routers.Pages)

	m.NotFound(routers.NotFound)

	listenAddr := fmt.Sprintf("0.0.0.0:%d", setting.HTTPPort)
	log.Info("%s Listen on %s", setting.Site.Name, listenAddr)
	log.Fatal("Fail to start Peach: %v", http.ListenAndServe(listenAddr, m))
}
<commit_msg>update - 项目入口: cmd/web.go, 添加关键注释.<commit_after>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package cmd

import (
	"fmt"
	"net/http"

	"github.com/Unknwon/log"
	"github.com/codegangsta/cli"
	"github.com/go-macaron/i18n"
	"github.com/go-macaron/pongo2"
	"gopkg.in/macaron.v1"

	// 项目模块:
	"github.com/peachdocs/peach/models"                       // 数据库模型
	"github.com/peachdocs/peach/modules/middleware"           // 中间件
	"github.com/peachdocs/peach/modules/setting"              // 配置部分
	"github.com/peachdocs/peach/routers"                      // 路由部分
)

/*
	关键入口:
	- 从命令行获取参数, 执行服务启动操作
	- Action: 执行启动操作
 */
var Web = cli.Command{
	Name:   "web",
	Usage:  "Start Peach web server",
	Action: runWeb,		// todo: 关键入口
	Flags: []cli.Flag{
		stringFlag("config, c", "custom/app.ini", "Custom configuration file path"),
	},
}

/*
	关键入口:
	- 创建项目实例, 并启动服务

 */
func runWeb(ctx *cli.Context) {
	if ctx.IsSet("config") {
		setting.CustomConf = ctx.String("config")
	}
	setting.NewContext()
	models.NewContext()

	log.Info("Peach %s", setting.AppVer)

	//---------------------------------------
	//          关键入口:
	//---------------------------------------
	// 创建web 服务对象
	m := macaron.New()

	// 日志
	m.Use(macaron.Logger())
	m.Use(macaron.Recovery())

	// 静态资源处理
	m.Use(macaron.Statics(macaron.StaticOptions{
		SkipLogging: setting.ProdMode,
	}, "custom/public", "public", models.HTMLRoot))

	m.Use(i18n.I18n(i18n.Options{
		Files:       setting.Docs.Locales,
		DefaultLang: setting.Docs.Langs[0],
	}))

	// 模板资源:
	tplDir := "templates"
	if setting.Page.UseCustomTpl {
		tplDir = "custom/templates"
	}
	m.Use(pongo2.Pongoer(pongo2.Options{
		Directory: tplDir,
	}))

	// 服务中间件:
	m.Use(middleware.Contexter())

	//---------------------------------------
	//          路由配置:
	//---------------------------------------
	m.Get("/", routers.Home)
	m.Get("/docs", routers.Docs)
	m.Get("/docs/images/*", routers.DocsStatic)
	m.Get("/docs/*", routers.Protect, routers.Docs)
	m.Post("/hook", routers.Hook)
	m.Get("/search", routers.Search)
	m.Get("/*", routers.Pages)

	m.NotFound(routers.NotFound)

	listenAddr := fmt.Sprintf("0.0.0.0:%d", setting.HTTPPort)	// 设置服务 IP + 端口
	log.Info("%s Listen on %s", setting.Site.Name, listenAddr)
	log.Fatal("Fail to start Peach: %v", http.ListenAndServe(listenAddr, m))   // 启动服务
}
<|endoftext|>


<commit_before>// Run a web server capable of dumping bug reports sent by Riot.
// Requires Go 1.5+
// Usage:   BUGS_USER=user BUGS_PASS=password go run rageshake.go PORT
// Example: BUGS_USER=alice BUGS_PASS=secret go run rageshake.go 8080
package main

import (
	"bytes"
	"compress/gzip"
	"crypto/subtle"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var maxPayloadSize = 1024 * 1024 * 55 // 55 MB

type LogEntry struct {
	ID    string `json:"id"`
	Lines string `json:"lines"`
}

type Payload struct {
	Text      string     `json:"text"`
	Version   string     `json:"version"`
	UserAgent string     `json:"user_agent"`
	Logs      []LogEntry `json:"logs"`
}

func respond(code int, w http.ResponseWriter) {
	w.WriteHeader(code)
	w.Write([]byte("{}"))
}

func gzipAndSave(data []byte, dirname, fpath string) error {
	_ = os.MkdirAll(filepath.Join("bugs", dirname), os.ModePerm)
	fpath = filepath.Join("bugs", dirname, fpath)

	if _, err := os.Stat(fpath); err == nil {
		return fmt.Errorf("file already exists") // the user can just retry
	}
	var b bytes.Buffer
	gz := gzip.NewWriter(&b)
	if _, err := gz.Write(data); err != nil {
		return err
	}
	if err := gz.Flush(); err != nil {
		return err
	}
	if err := gz.Close(); err != nil {
		return err
	}
	if err := ioutil.WriteFile(fpath, b.Bytes(), 0644); err != nil {
		return err
	}
	return nil
}

func basicAuth(handler http.Handler, username, password, realm string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user, pass, ok := r.BasicAuth() // pull creds from the request

		// check user and pass securely
		if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(username)) != 1 || subtle.ConstantTimeCompare([]byte(pass), []byte(password)) != 1 {
			w.Header().Set("WWW-Authenticate", `Basic realm="`+realm+`"`)
			w.WriteHeader(401)
			w.Write([]byte("Unauthorised.\n"))
			return
		}

		handler.ServeHTTP(w, r)
	})
}

func main() {
	http.HandleFunc("/api/submit", func(w http.ResponseWriter, req *http.Request) {
		if req.Method != "POST" && req.Method != "OPTIONS" {
			respond(405, w)
			return
		}
		// Set CORS
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
		if req.Method == "OPTIONS" {
			respond(200, w)
			return
		}
		if length, err := strconv.Atoi(req.Header.Get("Content-Length")); err != nil || length > maxPayloadSize {
			respond(413, w)
			return
		}
		var p Payload
		if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
			respond(400, w)
			return
		}
		// Dump bug report to disk as form:
		//  "bugreport-20170115-112233.log.gz" => user text, version, user agent, # logs
		//  "bugreport-20170115-112233-0.log.gz" => most recent log
		//  "bugreport-20170115-112233-1.log.gz" => ...
		//  "bugreport-20170115-112233-N.log.gz" => oldest log
		t := time.Now().UTC()
		prefix := t.Format("2006-01-02/150405")
		summary := fmt.Sprintf(
			"%s\n\nNumber of logs: %d\nVersion: %s\nUser-Agent: %s\n", p.Text, len(p.Logs), p.Version, p.UserAgent,
		)
		if err := gzipAndSave([]byte(summary), prefix, "details.log.gz"); err != nil {
			respond(500, w)
			return
		}
		for i, log := range p.Logs {
			if err := gzipAndSave([]byte(log.Lines), prefix, fmt.Sprintf("logs-%d.log.gz", i)); err != nil {
				respond(500, w)
				return // TODO: Rollback?
			}
		}
		respond(200, w)
	})

	// Make sure bugs directory exists
	_ = os.Mkdir("bugs", os.ModePerm)

	// serve files under "bugs"
	fs := http.FileServer(http.Dir("bugs"))
	fs = http.StripPrefix("/api/listing/", fs)

	// set auth if env vars exist
	usr := os.Getenv("BUGS_USER")
	pass := os.Getenv("BUGS_PASS")
	if usr == "" || pass == "" {
		fmt.Println("BUGS_USER and BUGS_PASS env vars not found. No authentication is running for /api/listing")
	} else {
		fs = basicAuth(fs, usr, pass, "Enter username and password")
	}
	http.Handle("/api/listing/", fs)

	port := os.Args[1]
	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<commit_msg>Review comments: change realm name<commit_after>// Run a web server capable of dumping bug reports sent by Riot.
// Requires Go 1.5+
// Usage:   BUGS_USER=user BUGS_PASS=password go run rageshake.go PORT
// Example: BUGS_USER=alice BUGS_PASS=secret go run rageshake.go 8080
package main

import (
	"bytes"
	"compress/gzip"
	"crypto/subtle"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var maxPayloadSize = 1024 * 1024 * 55 // 55 MB

type LogEntry struct {
	ID    string `json:"id"`
	Lines string `json:"lines"`
}

type Payload struct {
	Text      string     `json:"text"`
	Version   string     `json:"version"`
	UserAgent string     `json:"user_agent"`
	Logs      []LogEntry `json:"logs"`
}

func respond(code int, w http.ResponseWriter) {
	w.WriteHeader(code)
	w.Write([]byte("{}"))
}

func gzipAndSave(data []byte, dirname, fpath string) error {
	_ = os.MkdirAll(filepath.Join("bugs", dirname), os.ModePerm)
	fpath = filepath.Join("bugs", dirname, fpath)

	if _, err := os.Stat(fpath); err == nil {
		return fmt.Errorf("file already exists") // the user can just retry
	}
	var b bytes.Buffer
	gz := gzip.NewWriter(&b)
	if _, err := gz.Write(data); err != nil {
		return err
	}
	if err := gz.Flush(); err != nil {
		return err
	}
	if err := gz.Close(); err != nil {
		return err
	}
	if err := ioutil.WriteFile(fpath, b.Bytes(), 0644); err != nil {
		return err
	}
	return nil
}

func basicAuth(handler http.Handler, username, password, realm string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user, pass, ok := r.BasicAuth() // pull creds from the request

		// check user and pass securely
		if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(username)) != 1 || subtle.ConstantTimeCompare([]byte(pass), []byte(password)) != 1 {
			w.Header().Set("WWW-Authenticate", `Basic realm="`+realm+`"`)
			w.WriteHeader(401)
			w.Write([]byte("Unauthorised.\n"))
			return
		}

		handler.ServeHTTP(w, r)
	})
}

func main() {
	http.HandleFunc("/api/submit", func(w http.ResponseWriter, req *http.Request) {
		if req.Method != "POST" && req.Method != "OPTIONS" {
			respond(405, w)
			return
		}
		// Set CORS
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
		if req.Method == "OPTIONS" {
			respond(200, w)
			return
		}
		if length, err := strconv.Atoi(req.Header.Get("Content-Length")); err != nil || length > maxPayloadSize {
			respond(413, w)
			return
		}
		var p Payload
		if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
			respond(400, w)
			return
		}
		// Dump bug report to disk as form:
		//  "bugreport-20170115-112233.log.gz" => user text, version, user agent, # logs
		//  "bugreport-20170115-112233-0.log.gz" => most recent log
		//  "bugreport-20170115-112233-1.log.gz" => ...
		//  "bugreport-20170115-112233-N.log.gz" => oldest log
		t := time.Now().UTC()
		prefix := t.Format("2006-01-02/150405")
		summary := fmt.Sprintf(
			"%s\n\nNumber of logs: %d\nVersion: %s\nUser-Agent: %s\n", p.Text, len(p.Logs), p.Version, p.UserAgent,
		)
		if err := gzipAndSave([]byte(summary), prefix, "details.log.gz"); err != nil {
			respond(500, w)
			return
		}
		for i, log := range p.Logs {
			if err := gzipAndSave([]byte(log.Lines), prefix, fmt.Sprintf("logs-%d.log.gz", i)); err != nil {
				respond(500, w)
				return // TODO: Rollback?
			}
		}
		respond(200, w)
	})

	// Make sure bugs directory exists
	_ = os.Mkdir("bugs", os.ModePerm)

	// serve files under "bugs"
	fs := http.FileServer(http.Dir("bugs"))
	fs = http.StripPrefix("/api/listing/", fs)

	// set auth if env vars exist
	usr := os.Getenv("BUGS_USER")
	pass := os.Getenv("BUGS_PASS")
	if usr == "" || pass == "" {
		fmt.Println("BUGS_USER and BUGS_PASS env vars not found. No authentication is running for /api/listing")
	} else {
		fs = basicAuth(fs, usr, pass, "Riot bug reports")
	}
	http.Handle("/api/listing/", fs)

	port := os.Args[1]
	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<|endoftext|>


<commit_before>// Run a web server capable of dumping bug reports sent by Riot.
// Requires Go 1.5+
// Usage:   go run rageshake.go PORT
// Example: go run rageshake.go 8080
package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"
)

var maxPayloadSize = 1024 * 1024 * 55 // 55 MB

type LogEntry struct {
	ID    string `json:"id"`
	Lines string `json:"lines"`
}

type Payload struct {
	Text      string     `json:"text"`
	Version   string     `json:"version"`
	UserAgent string     `json:"user_agent"`
	Logs      []LogEntry `json:"logs"`
}

func respond(code int, w http.ResponseWriter) {
	w.WriteHeader(code)
	w.Write([]byte("{}"))
}

func gzipAndSave(data []byte, filepath string) error {
	var b bytes.Buffer
	gz := gzip.NewWriter(&b)
	if _, err := gz.Write(data); err != nil {
		return err
	}
	if err := gz.Flush(); err != nil {
		return err
	}
	if err := gz.Close(); err != nil {
		return err
	}
	if err := ioutil.WriteFile(filepath, b.Bytes(), 0644); err != nil {
		return err
	}
	return nil
}

func main() {
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		if req.Method != "POST" && req.Method != "OPTIONS" {
			respond(405, w)
			return
		}
		// Set CORS
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
		if req.Method == "OPTIONS" {
			respond(200, w)
			return
		}
		if length, err := strconv.Atoi(req.Header.Get("Content-Length")); err != nil || length > maxPayloadSize {
			respond(413, w)
			return
		}
		var p Payload
		if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
			respond(400, w)
			return
		}
		// Dump bug report to disk as form:
		//  "bugreport-20170115-112233.log.gz" => user text, version, user agent, # logs
		//  "bugreport-20170115-112233-0.log.gz" => most recent log
		//  "bugreport-20170115-112233-1.log.gz" => ...
		//  "bugreport-20170115-112233-N.log.gz" => oldest log
		t := time.Now()
		prefix := t.Format("bugreport-20060102-150405")
		summary := fmt.Sprintf(
			"%s\n\nNumber of logs: %d\nVersion: %s\nUser-Agent: %s\n", p.Text, len(p.Logs), p.Version, p.UserAgent,
		)
		if err := gzipAndSave([]byte(summary), prefix+".log.gz"); err != nil {
			respond(500, w)
			return
		}
		for i, log := range p.Logs {
			if err := gzipAndSave([]byte(log.Lines), fmt.Sprintf("%s-%d.log.gz", prefix, i)); err != nil {
				respond(500, w)
				return // TODO: Rollback?
			}
		}
		respond(200, w)
	})

	port := os.Args[1]
	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<commit_msg>Fail the request if we clash files<commit_after>// Run a web server capable of dumping bug reports sent by Riot.
// Requires Go 1.5+
// Usage:   go run rageshake.go PORT
// Example: go run rageshake.go 8080
package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"
)

var maxPayloadSize = 1024 * 1024 * 55 // 55 MB

type LogEntry struct {
	ID    string `json:"id"`
	Lines string `json:"lines"`
}

type Payload struct {
	Text      string     `json:"text"`
	Version   string     `json:"version"`
	UserAgent string     `json:"user_agent"`
	Logs      []LogEntry `json:"logs"`
}

func respond(code int, w http.ResponseWriter) {
	w.WriteHeader(code)
	w.Write([]byte("{}"))
}

func gzipAndSave(data []byte, filepath string) error {
	if _, err := os.Stat(filepath); err == nil {
		return fmt.Errorf("file already exists") // the user can just retry
	}
	var b bytes.Buffer
	gz := gzip.NewWriter(&b)
	if _, err := gz.Write(data); err != nil {
		return err
	}
	if err := gz.Flush(); err != nil {
		return err
	}
	if err := gz.Close(); err != nil {
		return err
	}
	if err := ioutil.WriteFile(filepath, b.Bytes(), 0644); err != nil {
		return err
	}
	return nil
}

func main() {
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		if req.Method != "POST" && req.Method != "OPTIONS" {
			respond(405, w)
			return
		}
		// Set CORS
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
		if req.Method == "OPTIONS" {
			respond(200, w)
			return
		}
		if length, err := strconv.Atoi(req.Header.Get("Content-Length")); err != nil || length > maxPayloadSize {
			respond(413, w)
			return
		}
		var p Payload
		if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
			respond(400, w)
			return
		}
		// Dump bug report to disk as form:
		//  "bugreport-20170115-112233.log.gz" => user text, version, user agent, # logs
		//  "bugreport-20170115-112233-0.log.gz" => most recent log
		//  "bugreport-20170115-112233-1.log.gz" => ...
		//  "bugreport-20170115-112233-N.log.gz" => oldest log
		t := time.Now()
		prefix := t.Format("bugreport-20060102-150405")
		summary := fmt.Sprintf(
			"%s\n\nNumber of logs: %d\nVersion: %s\nUser-Agent: %s\n", p.Text, len(p.Logs), p.Version, p.UserAgent,
		)
		if err := gzipAndSave([]byte(summary), prefix+".log.gz"); err != nil {
			respond(500, w)
			return
		}
		for i, log := range p.Logs {
			if err := gzipAndSave([]byte(log.Lines), fmt.Sprintf("%s-%d.log.gz", prefix, i)); err != nil {
				respond(500, w)
				return // TODO: Rollback?
			}
		}
		respond(200, w)
	})

	port := os.Args[1]
	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<|endoftext|>


<commit_before>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package index

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"

	"github.com/ava-labs/avalanchego/utils/hashing"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/ava-labs/avalanchego/vms/components/avax"

	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

var (
	idxKey                         = []byte("idx")
	idxCompleteKey                 = []byte("complete")
	errIndexingRequiredFromGenesis = errors.New("running would create incomplete index. Allow incomplete indices or re-sync from genesis with indexing enabled")
	errCausesIncompleteIndex       = errors.New("running would create incomplete index. Allow incomplete indices or enable indexing")
)

// AddressTxsIndexer maintains information about which transactions changed
// the balances of which addresses. This includes both transactions that
// increase and decrease an address's balance.
// A transaction is said to change an address's balance if either is true:
// 1) A UTXO that the transaction consumes was at least partially owned by the address.
// 2) A UTXO that the transaction produces is at least partially owned by the address.
type AddressTxsIndexer interface {
	// Add is called during [txID]'s SemanticVerify.
	// [inputUTXOIDs] are the IDs of UTXOs [txID] consumes.
	// [outputUTXOs] are the UTXOs [txID] creates.
	// [getUTXOF] can be used to look up UTXOs by ID.
	// If the error is non-nil, do not persist [txID] to disk as accepted in the VM,
	// and shut down this chain.
	Add(
		txID ids.ID,
		inputUTXOIDs []*avax.UTXOID,
		outputUTXOs []*avax.UTXO,
		getUTXOF func(utxoID *avax.UTXOID) (*avax.UTXO, error),
	) error

	// Accept is called when [txID] is accepted.
	// Persists data about [txID] and what balances it changed.
	// If the error is non-nil, do not persist [txID] to disk as accepted in the VM,
	// and shut down this chain.
	Accept(txID ids.ID) error

	// Clear is called when [txID] is rejected or fails verification.
	// Clears unwritten state about the tx.
	Clear(ids.ID)

	// Read returns the IDs of transactions that changed [address]'s balance of [assetID].
	// The returned transactions are in order of increasing acceptance time.
	// The length of the returned slice <= [pageSize].
	// [cursor] is the offset to start reading from.
	Read(address ids.ShortID, assetID ids.ID, cursor, pageSize uint64) ([]ids.ID, error)
}

// indexer implements AddressTxsIndexer
type indexer struct {
	log     logging.Logger
	metrics metrics
	db      database.Database
	// txID -> Address -> AssetID --> exists if the address's balance
	// of the asset is changed by processing tx [txID]
	balanceChanges map[ids.ID]map[ids.ShortID]map[ids.ID]struct{}
}

// NewIndexer Returns a new AddressTxsIndexer.
// The returned indexer ignores UTXOs that are not type secp256k1fx.TransferOutput.
func NewIndexer(
	db database.Database,
	log logging.Logger,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
	allowIncompleteIndices bool,
) (AddressTxsIndexer, error) {
	i := &indexer{
		balanceChanges: make(map[ids.ID]map[ids.ShortID]map[ids.ID]struct{}),
		db:             db,
		log:            log,
	}
	// initialize the indexer
	if err := checkIndexStatus(i.db, true, allowIncompleteIndices); err != nil {
		return nil, err
	}
	// initialize the metrics
	if err := i.metrics.initialize(metricsNamespace, metricsRegisterer); err != nil {
		return nil, err
	}
	return i, nil
}

// add marks that [txID] changes the balance of [assetID] for [addrs]
// This data is either written in Accept() or cleared in Clear()
func (i *indexer) add(txID, assetID ids.ID, addrs [][]byte) error {
	for _, addressBytes := range addrs {
		address, err := ids.ToShortID(addressBytes)
		if err != nil {
			// should never happen
			return err
		}
		if _, exists := i.balanceChanges[txID]; !exists {
			i.balanceChanges[txID] = make(map[ids.ShortID]map[ids.ID]struct{})
		}
		if _, exists := i.balanceChanges[txID][address]; !exists {
			i.balanceChanges[txID][address] = make(map[ids.ID]struct{})
		}
		i.balanceChanges[txID][address][assetID] = struct{}{}
	}
	return nil
}

// See AddressTxsIndexer
func (i *indexer) Add(
	txID ids.ID,
	inputUTXOIDs []*avax.UTXOID,
	outputUTXOs []*avax.UTXO,
	getUTXOF func(utxoID *avax.UTXOID) (*avax.UTXO, error),
) error {
	utxos := outputUTXOs
	for _, utxoID := range inputUTXOIDs {
		utxo, err := getUTXOF(utxoID)
		if err != nil { // should never happen
			return fmt.Errorf("error finding UTXO %s: %s", utxoID, err)
		}
		utxos = append(utxos, utxo)
	}
	for _, utxo := range utxos {
		out, ok := utxo.Out.(avax.Addressable)
		if !ok {
			i.log.Verbo("skipping UTXO %s for indexing", utxo.InputID())
			continue
		}
		if err := i.add(txID, utxo.AssetID(), out.Addresses()); err != nil {
			return fmt.Errorf("error adding to index: %s", err)
		}
	}
	return nil
}

// Accept persists which balances [txID] changed.
// Associates all UTXOs in [i.balanceChanges] with transaction [txID].
// The database structure is:
// [address]
// |  [assetID]
// |  |
// |  | "idx" => 2 		Running transaction index key, represents the next index
// |  | "0"   => txID1
// |  | "1"   => txID1
// See AddressTxsIndexer
func (i *indexer) Accept(txID ids.ID) error {
	for address, assetIDs := range i.balanceChanges[txID] {
		addressPrefixDB := prefixdb.New(address[:], i.db)
		for assetID := range assetIDs {
			assetPrefixDB := prefixdb.New(assetID[:], addressPrefixDB)

			var idx uint64
			idxBytes, err := assetPrefixDB.Get(idxKey)
			switch err {
			case nil:
				// index is found, parse stored [idxBytes]
				idx = binary.BigEndian.Uint64(idxBytes)
			case database.ErrNotFound:
				// idx not found; this must be the first entry.
				idx = 0
				idxBytes = make([]byte, wrappers.LongLen)
				binary.BigEndian.PutUint64(idxBytes, idx)
			default:
				// Unexpected error
				return fmt.Errorf("unexpected error when indexing txID %s: %s", txID, err)
			}

			// write the [txID] at the index
			i.log.Verbo("writing address/assetID/index/txID %s/%s/%d/%s", address, assetID, idx, txID)
			if err := assetPrefixDB.Put(idxBytes, txID[:]); err != nil {
				return fmt.Errorf("failed to write txID while indexing %s: %s", txID, err)
			}

			// increment and store the index for next use
			idx++
			binary.BigEndian.PutUint64(idxBytes, idx)

			if err := assetPrefixDB.Put(idxKey, idxBytes); err != nil {
				return fmt.Errorf("failed to write index txID while indexing %s: %s", txID, err)
			}
		}
	}
	// delete already written [txID] from the map
	delete(i.balanceChanges, txID)
	i.metrics.numTxsIndexed.Observe(1)
	return nil
}

// Read returns IDs of transactions that changed [address]'s balance of [assetID],
// starting at [cursor], in order of transaction acceptance. e.g. if [cursor] == 1, does
// not return the first transaction that changed the balance. (This is for for pagination.)
// Returns at most [pageSize] elements.
// See AddressTxsIndexer
func (i *indexer) Read(address ids.ShortID, assetID ids.ID, cursor, pageSize uint64) ([]ids.ID, error) {
	// setup prefix DBs
	addressTxDB := prefixdb.New(address[:], i.db)
	assetPrefixDB := prefixdb.New(assetID[:], addressTxDB)

	// get cursor in bytes
	cursorBytes := make([]byte, wrappers.LongLen)
	binary.BigEndian.PutUint64(cursorBytes, cursor)

	// start reading from the cursor bytes, numeric keys maintain the order (see Accept)
	iter := assetPrefixDB.NewIteratorWithStart(cursorBytes)
	defer iter.Release()

	var txIDs []ids.ID
	for uint64(len(txIDs)) < pageSize && iter.Next() {
		if bytes.Equal(idxKey, iter.Key()) {
			// This key has the next index to use, not a tx ID
			continue
		}

		// get the value, make sure its in the right format
		txIDBytes := iter.Value()
		if len(txIDBytes) != hashing.HashLen {
			return nil, fmt.Errorf("invalid tx ID %s", txIDBytes)
		}

		// get the  txID and append to our list
		var txID ids.ID
		copy(txID[:], txIDBytes)
		txIDs = append(txIDs, txID)
	}
	return txIDs, nil
}

// Clear clears data about which balances [txID] changed.
func (i *indexer) Clear(txID ids.ID) {
	i.balanceChanges[txID] = make(map[ids.ShortID]map[ids.ID]struct{})
}

// checkIndexStatus checks the indexing status in the database, returning error if the state
// with respect to provided parameters is invalid
func checkIndexStatus(db database.KeyValueReaderWriter, enableIndexing, allowIncomplete bool) error {
	// verify whether the index is complete.
	idxComplete, err := database.GetBool(db, idxCompleteKey)
	if err == database.ErrNotFound {
		// We've not run before. Mark whether indexing is enabled this run.
		return database.PutBool(db, idxCompleteKey, enableIndexing)
	} else if err != nil {
		return err
	}

	if idxComplete && enableIndexing {
		// indexing has been enabled in the past and we're enabling it now
		return nil
	}

	if !idxComplete && enableIndexing && !allowIncomplete {
		// In a previous run, we did not index so it's incomplete.
		// indexing was disabled before but now we want to index.
		return errIndexingRequiredFromGenesis
	} else if !idxComplete {
		// either indexing is disabled, or incomplete indices are ok, so we don't care that index is incomplete
		return nil
	}

	// the index is complete
	if !enableIndexing && !allowIncomplete { // indexing is disabled this run
		return errCausesIncompleteIndex
	} else if !enableIndexing {
		// running without indexing makes it incomplete
		return database.PutBool(db, idxCompleteKey, false)
	}

	return nil
}

type noIndexer struct{}

func NewNoIndexer(db database.Database, allowIncomplete bool) (AddressTxsIndexer, error) {
	if err := checkIndexStatus(db, false, allowIncomplete); err != nil {
		return nil, err
	}
	return &noIndexer{}, nil
}

func (i *noIndexer) Add(ids.ID, []*avax.UTXOID, []*avax.UTXO, func(utxoID *avax.UTXOID) (*avax.UTXO, error)) error {
	return nil
}

func (i *noIndexer) Accept(ids.ID) error {
	return nil
}

func (i *noIndexer) Clear(ids.ID) {}

func (i *noIndexer) Read(ids.ShortID, ids.ID, uint64, uint64) ([]ids.ID, error) {
	return nil, nil
}
<commit_msg>clearer return<commit_after>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package index

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"

	"github.com/ava-labs/avalanchego/utils/hashing"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/ava-labs/avalanchego/vms/components/avax"

	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

var (
	idxKey                         = []byte("idx")
	idxCompleteKey                 = []byte("complete")
	errIndexingRequiredFromGenesis = errors.New("running would create incomplete index. Allow incomplete indices or re-sync from genesis with indexing enabled")
	errCausesIncompleteIndex       = errors.New("running would create incomplete index. Allow incomplete indices or enable indexing")
)

// AddressTxsIndexer maintains information about which transactions changed
// the balances of which addresses. This includes both transactions that
// increase and decrease an address's balance.
// A transaction is said to change an address's balance if either is true:
// 1) A UTXO that the transaction consumes was at least partially owned by the address.
// 2) A UTXO that the transaction produces is at least partially owned by the address.
type AddressTxsIndexer interface {
	// Add is called during [txID]'s SemanticVerify.
	// [inputUTXOIDs] are the IDs of UTXOs [txID] consumes.
	// [outputUTXOs] are the UTXOs [txID] creates.
	// [getUTXOF] can be used to look up UTXOs by ID.
	// If the error is non-nil, do not persist [txID] to disk as accepted in the VM,
	// and shut down this chain.
	Add(
		txID ids.ID,
		inputUTXOIDs []*avax.UTXOID,
		outputUTXOs []*avax.UTXO,
		getUTXOF func(utxoID *avax.UTXOID) (*avax.UTXO, error),
	) error

	// Accept is called when [txID] is accepted.
	// Persists data about [txID] and what balances it changed.
	// If the error is non-nil, do not persist [txID] to disk as accepted in the VM,
	// and shut down this chain.
	Accept(txID ids.ID) error

	// Clear is called when [txID] is rejected or fails verification.
	// Clears unwritten state about the tx.
	Clear(ids.ID)

	// Read returns the IDs of transactions that changed [address]'s balance of [assetID].
	// The returned transactions are in order of increasing acceptance time.
	// The length of the returned slice <= [pageSize].
	// [cursor] is the offset to start reading from.
	Read(address ids.ShortID, assetID ids.ID, cursor, pageSize uint64) ([]ids.ID, error)
}

// indexer implements AddressTxsIndexer
type indexer struct {
	log     logging.Logger
	metrics metrics
	db      database.Database
	// txID -> Address -> AssetID --> exists if the address's balance
	// of the asset is changed by processing tx [txID]
	balanceChanges map[ids.ID]map[ids.ShortID]map[ids.ID]struct{}
}

// NewIndexer Returns a new AddressTxsIndexer.
// The returned indexer ignores UTXOs that are not type secp256k1fx.TransferOutput.
func NewIndexer(
	db database.Database,
	log logging.Logger,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
	allowIncompleteIndices bool,
) (AddressTxsIndexer, error) {
	i := &indexer{
		balanceChanges: make(map[ids.ID]map[ids.ShortID]map[ids.ID]struct{}),
		db:             db,
		log:            log,
	}
	// initialize the indexer
	if err := checkIndexStatus(i.db, true, allowIncompleteIndices); err != nil {
		return nil, err
	}
	// initialize the metrics
	if err := i.metrics.initialize(metricsNamespace, metricsRegisterer); err != nil {
		return nil, err
	}
	return i, nil
}

// add marks that [txID] changes the balance of [assetID] for [addrs]
// This data is either written in Accept() or cleared in Clear()
func (i *indexer) add(txID, assetID ids.ID, addrs [][]byte) error {
	for _, addressBytes := range addrs {
		address, err := ids.ToShortID(addressBytes)
		if err != nil {
			// should never happen
			return err
		}
		if _, exists := i.balanceChanges[txID]; !exists {
			i.balanceChanges[txID] = make(map[ids.ShortID]map[ids.ID]struct{})
		}
		if _, exists := i.balanceChanges[txID][address]; !exists {
			i.balanceChanges[txID][address] = make(map[ids.ID]struct{})
		}
		i.balanceChanges[txID][address][assetID] = struct{}{}
	}
	return nil
}

// See AddressTxsIndexer
func (i *indexer) Add(
	txID ids.ID,
	inputUTXOIDs []*avax.UTXOID,
	outputUTXOs []*avax.UTXO,
	getUTXOF func(utxoID *avax.UTXOID) (*avax.UTXO, error),
) error {
	utxos := outputUTXOs
	for _, utxoID := range inputUTXOIDs {
		utxo, err := getUTXOF(utxoID)
		if err != nil { // should never happen
			return fmt.Errorf("error finding UTXO %s: %s", utxoID, err)
		}
		utxos = append(utxos, utxo)
	}
	for _, utxo := range utxos {
		out, ok := utxo.Out.(avax.Addressable)
		if !ok {
			i.log.Verbo("skipping UTXO %s for indexing", utxo.InputID())
			continue
		}
		if err := i.add(txID, utxo.AssetID(), out.Addresses()); err != nil {
			return fmt.Errorf("error adding to index: %s", err)
		}
	}
	return nil
}

// Accept persists which balances [txID] changed.
// Associates all UTXOs in [i.balanceChanges] with transaction [txID].
// The database structure is:
// [address]
// |  [assetID]
// |  |
// |  | "idx" => 2 		Running transaction index key, represents the next index
// |  | "0"   => txID1
// |  | "1"   => txID1
// See AddressTxsIndexer
func (i *indexer) Accept(txID ids.ID) error {
	for address, assetIDs := range i.balanceChanges[txID] {
		addressPrefixDB := prefixdb.New(address[:], i.db)
		for assetID := range assetIDs {
			assetPrefixDB := prefixdb.New(assetID[:], addressPrefixDB)

			var idx uint64
			idxBytes, err := assetPrefixDB.Get(idxKey)
			switch err {
			case nil:
				// index is found, parse stored [idxBytes]
				idx = binary.BigEndian.Uint64(idxBytes)
			case database.ErrNotFound:
				// idx not found; this must be the first entry.
				idx = 0
				idxBytes = make([]byte, wrappers.LongLen)
				binary.BigEndian.PutUint64(idxBytes, idx)
			default:
				// Unexpected error
				return fmt.Errorf("unexpected error when indexing txID %s: %s", txID, err)
			}

			// write the [txID] at the index
			i.log.Verbo("writing address/assetID/index/txID %s/%s/%d/%s", address, assetID, idx, txID)
			if err := assetPrefixDB.Put(idxBytes, txID[:]); err != nil {
				return fmt.Errorf("failed to write txID while indexing %s: %s", txID, err)
			}

			// increment and store the index for next use
			idx++
			binary.BigEndian.PutUint64(idxBytes, idx)

			if err := assetPrefixDB.Put(idxKey, idxBytes); err != nil {
				return fmt.Errorf("failed to write index txID while indexing %s: %s", txID, err)
			}
		}
	}
	// delete already written [txID] from the map
	delete(i.balanceChanges, txID)
	i.metrics.numTxsIndexed.Observe(1)
	return nil
}

// Read returns IDs of transactions that changed [address]'s balance of [assetID],
// starting at [cursor], in order of transaction acceptance. e.g. if [cursor] == 1, does
// not return the first transaction that changed the balance. (This is for for pagination.)
// Returns at most [pageSize] elements.
// See AddressTxsIndexer
func (i *indexer) Read(address ids.ShortID, assetID ids.ID, cursor, pageSize uint64) ([]ids.ID, error) {
	// setup prefix DBs
	addressTxDB := prefixdb.New(address[:], i.db)
	assetPrefixDB := prefixdb.New(assetID[:], addressTxDB)

	// get cursor in bytes
	cursorBytes := make([]byte, wrappers.LongLen)
	binary.BigEndian.PutUint64(cursorBytes, cursor)

	// start reading from the cursor bytes, numeric keys maintain the order (see Accept)
	iter := assetPrefixDB.NewIteratorWithStart(cursorBytes)
	defer iter.Release()

	var txIDs []ids.ID
	for uint64(len(txIDs)) < pageSize && iter.Next() {
		if bytes.Equal(idxKey, iter.Key()) {
			// This key has the next index to use, not a tx ID
			continue
		}

		// get the value, make sure its in the right format
		txIDBytes := iter.Value()
		if len(txIDBytes) != hashing.HashLen {
			return nil, fmt.Errorf("invalid tx ID %s", txIDBytes)
		}

		// get the  txID and append to our list
		var txID ids.ID
		copy(txID[:], txIDBytes)
		txIDs = append(txIDs, txID)
	}
	return txIDs, nil
}

// Clear clears data about which balances [txID] changed.
func (i *indexer) Clear(txID ids.ID) {
	i.balanceChanges[txID] = make(map[ids.ShortID]map[ids.ID]struct{})
}

// checkIndexStatus checks the indexing status in the database, returning error if the state
// with respect to provided parameters is invalid
func checkIndexStatus(db database.KeyValueReaderWriter, enableIndexing, allowIncomplete bool) error {
	// verify whether the index is complete.
	idxComplete, err := database.GetBool(db, idxCompleteKey)
	if err == database.ErrNotFound {
		// We've not run before. Mark whether indexing is enabled this run.
		return database.PutBool(db, idxCompleteKey, enableIndexing)
	} else if err != nil {
		return err
	}

	if idxComplete && enableIndexing {
		// indexing has been enabled in the past and we're enabling it now
		return nil
	}

	if !idxComplete && enableIndexing && !allowIncomplete {
		// In a previous run, we did not index so it's incomplete.
		// indexing was disabled before but now we want to index.
		return errIndexingRequiredFromGenesis
	} else if !idxComplete {
		// either indexing is disabled, or incomplete indices are ok, so we don't care that index is incomplete
		return nil
	}

	// the index is complete
	if !enableIndexing && !allowIncomplete { // indexing is disabled this run
		return errCausesIncompleteIndex
	} else if !enableIndexing {
		// running without indexing makes it incomplete
		return database.PutBool(db, idxCompleteKey, false)
	}

	return nil
}

type noIndexer struct{}

func NewNoIndexer(db database.Database, allowIncomplete bool) (AddressTxsIndexer, error) {
	return &noIndexer{}, checkIndexStatus(db, false, allowIncomplete)
}

func (i *noIndexer) Add(ids.ID, []*avax.UTXOID, []*avax.UTXO, func(utxoID *avax.UTXOID) (*avax.UTXO, error)) error {
	return nil
}

func (i *noIndexer) Accept(ids.ID) error {
	return nil
}

func (i *noIndexer) Clear(ids.ID) {}

func (i *noIndexer) Read(ids.ShortID, ids.ID, uint64, uint64) ([]ids.ID, error) {
	return nil, nil
}
<|endoftext|>