<commit_before>package sockd

import (
	cryptRand "crypto/rand"
	"encoding/binary"
	"fmt"
	"github.com/HouzuoGuo/laitos/misc"
	"io"
	pseudoRand "math/rand"
	"net"
	"strconv"
	"strings"
	"time"
)

const (
	AddressTypeMask  byte = 0xf
	AddressTypeIndex      = 0
	AddressTypeIPv4       = 1
	AddressTypeDM         = 3
	AddressTypeIPv6       = 4

	IPPacketIndex    = 1
	IPv4PacketLength = net.IPv4len + 2
	IPv6PacketLength = net.IPv6len + 2

	DMAddrIndex        = 2
	DMAddrLengthIndex  = 1
	DMAddrHeaderLength = 2
)

var TCPDurationStats = misc.NewStats()

func (daemon *Daemon) StartAndBlockTCP() error {
	var err error
	listener, err := net.Listen("tcp", net.JoinHostPort(daemon.Address, strconv.Itoa(daemon.TCPPort)))
	if err != nil {
		return fmt.Errorf("sockd.StartAndBlockTCP: failed to listen on %s:%d - %v", daemon.Address, daemon.TCPPort, err)
	}
	defer listener.Close()
	daemon.logger.Info("StartAndBlockTCP", "", nil, "going to listen for connections")
	daemon.tcpListener = listener

	for {
		if misc.EmergencyLockDown {
			return misc.ErrEmergencyLockDown
		}
		conn, err := listener.Accept()
		if err != nil {
			if strings.Contains(err.Error(), "closed") {
				return nil
			} else {
				return fmt.Errorf("sockd.StartAndBlockTCP: failed to accept new connection - %v", err)
			}
		}
		clientIP := conn.RemoteAddr().(*net.TCPAddr).IP.String()
		if daemon.rateLimitTCP.Add(clientIP, true) {
			go NewTCPCipherConnection(daemon, conn, daemon.cipher.Copy(), daemon.logger).HandleTCPConnection()
		} else {
			conn.Close()
		}
	}
}

type TCPCipherConnection struct {
	net.Conn
	*Cipher
	daemon            *Daemon
	readBuf, writeBuf []byte
	logger            misc.Logger
}

func NewTCPCipherConnection(daemon *Daemon, netConn net.Conn, cip *Cipher, logger misc.Logger) *TCPCipherConnection {
	return &TCPCipherConnection{
		Conn:     netConn,
		daemon:   daemon,
		Cipher:   cip,
		readBuf:  make([]byte, MaxPacketSize),
		writeBuf: make([]byte, MaxPacketSize),
		logger:   logger,
	}
}

func (conn *TCPCipherConnection) Close() error {
	return conn.Conn.Close()
}

func (conn *TCPCipherConnection) Read(b []byte) (n int, err error) {
	if conn.DecryptionStream == nil {
		iv := make([]byte, conn.IVLength)
		if _, err = io.ReadFull(conn.Conn, iv); err != nil {
			return
		}
		conn.InitDecryptionStream(iv)
		if len(conn.IV) == 0 {
			conn.IV = iv
		}
	}

	cipherData := conn.readBuf
	if len(b) > len(cipherData) {
		cipherData = make([]byte, len(b))
	} else {
		cipherData = cipherData[:len(b)]
	}

	n, err = conn.Conn.Read(cipherData)
	if n > 0 {
		conn.Decrypt(b[0:n], cipherData[0:n])
	}
	return
}

func (conn *TCPCipherConnection) Write(buf []byte) (n int, err error) {
	bufSize := len(buf)
	headerLen := len(buf) - bufSize

	var iv []byte
	if conn.EncryptionStream == nil {
		iv = conn.InitEncryptionStream()
	}

	cipherData := conn.writeBuf
	dataSize := len(buf) + len(iv)
	if dataSize > len(cipherData) {
		cipherData = make([]byte, dataSize)
	} else {
		cipherData = cipherData[:dataSize]
	}

	if iv != nil {
		copy(cipherData, iv)
	}

	conn.Encrypt(cipherData[len(iv):], buf)
	n, err = conn.Conn.Write(cipherData)

	if n >= headerLen {
		n -= headerLen
	}
	return
}

func (conn *TCPCipherConnection) ParseRequest() (destIP, destAddr string, err error) {
	conn.SetReadDeadline(time.Now().Add(IOTimeoutSec))

	buf := make([]byte, 269)
	if _, err = io.ReadFull(conn, buf[:AddressTypeIndex+1]); err != nil {
		return
	}

	var reqStart, reqEnd int
	addrType := buf[AddressTypeIndex]
	maskedType := addrType & AddressTypeMask
	switch maskedType {
	case AddressTypeIPv4:
		reqStart, reqEnd = IPPacketIndex, IPPacketIndex+IPv4PacketLength
	case AddressTypeIPv6:
		reqStart, reqEnd = IPPacketIndex, IPPacketIndex+IPv6PacketLength
	case AddressTypeDM:
		if _, err = io.ReadFull(conn, buf[AddressTypeIndex+1:DMAddrLengthIndex+1]); err != nil {
			return
		}
		reqStart, reqEnd = DMAddrIndex, DMAddrIndex+int(buf[DMAddrLengthIndex])+DMAddrHeaderLength
	default:
		err = fmt.Errorf("TCPCipherConnection.ParseRequest: unknown mask type %d", maskedType)
		return
	}

	if _, err = io.ReadFull(conn, buf[reqStart:reqEnd]); err != nil {
		return
	}

	switch maskedType {
	case AddressTypeIPv4:
		destIP = net.IP(buf[IPPacketIndex : IPPacketIndex+net.IPv4len]).String()
	case AddressTypeIPv6:
		destIP = net.IP(buf[IPPacketIndex : IPPacketIndex+net.IPv6len]).String()
	case AddressTypeDM:
		destIP = string(buf[DMAddrIndex : DMAddrIndex+int(buf[DMAddrLengthIndex])])
	}
	port := binary.BigEndian.Uint16(buf[reqEnd-2 : reqEnd])
	if port < 1 {
		err = fmt.Errorf("TCPCipherConnection.ParseRequest: invalid destination port %d", port)
		return
	}
	destAddr = net.JoinHostPort(destIP, strconv.Itoa(int(port)))
	return
}

func (conn *TCPCipherConnection) WriteRandAndClose() {
	defer conn.Close()
	randBuf := make([]byte, RandNum(1, 20, 300))
	_, err := cryptRand.Read(randBuf)
	if err != nil {
		conn.logger.Warning("WriteRandAndClose", conn.Conn.RemoteAddr().String(), err, "failed to get random bytes")
		return
	}
	conn.SetDeadline(time.Now().Add(IOTimeoutSec))
	if _, err := conn.Write(randBuf); err != nil {
		conn.logger.Warning("WriteRandAndClose", conn.Conn.RemoteAddr().String(), err, "failed to write random bytes")
	}
}

func (conn *TCPCipherConnection) HandleTCPConnection() {
	beginTimeNano := time.Now().UnixNano()
	defer func() {
		TCPDurationStats.Trigger(float64(time.Now().UnixNano() - beginTimeNano))
	}()
	remoteAddr := conn.RemoteAddr().String()
	destIP, destAddr, err := conn.ParseRequest()
	if err != nil {
		conn.logger.Warning("HandleTCPConnection", remoteAddr, err, "failed to get destination address")
		conn.WriteRandAndClose()
		return
	}
	if strings.ContainsRune(destAddr, 0x00) {
		conn.logger.Warning("HandleTCPConnection", remoteAddr, nil, "will not serve invalid destination address with 0 in it")
		conn.WriteRandAndClose()
		return
	}
	if conn.daemon.DNSDaemon.IsInBlacklist(destIP) {
		conn.logger.Info("HandleTCPConnection", remoteAddr, nil, "will not serve blacklisted address %s", destIP)
		conn.Close()
		return
	}
	dest, err := net.DialTimeout("tcp", destAddr, IOTimeoutSec)
	if err != nil {
		conn.logger.Warning("HandleTCPConnection", remoteAddr, err, "failed to connect to destination \"%s\"", destAddr)
		conn.Close()
		return
	}
	go PipeTCPConnection(conn, dest, true)
	PipeTCPConnection(dest, conn, false)
	return
}

func PipeTCPConnection(fromConn, toConn net.Conn, doWriteRand bool) {
	defer toConn.Close()
	buf := make([]byte, MaxPacketSize)
	for {
		fromConn.SetReadDeadline(time.Now().Add(IOTimeoutSec))
		length, err := fromConn.Read(buf)
		if length > 0 {
			toConn.SetWriteDeadline(time.Now().Add(IOTimeoutSec))
			if _, err := toConn.Write(buf[:length]); err != nil {
				return
			}
		}
		if err != nil {
			if doWriteRand {
				WriteRand(fromConn)
			}
			return
		}
	}
}

func WriteRand(conn net.Conn) {
	randBytesWritten := 0
	for i := 0; i < RandNum(1, 2, 3); i++ {
		randBuf := make([]byte, RandNum(40, 60, 900))
		if _, err := pseudoRand.Read(randBuf); err != nil {
			misc.DefaultLogger.Warning("sockd.WriteRand", conn.RemoteAddr().String(), err, "failed to get random bytes")
			break
		}
		conn.SetDeadline(time.Now().Add(time.Duration(RandNum(140, 220, 350)) * time.Millisecond))
		if n, err := conn.Write(randBuf); err != nil && !strings.Contains(err.Error(), "closed") && !strings.Contains(err.Error(), "broken") {
			misc.DefaultLogger.Warning("sockd.WriteRand", conn.RemoteAddr().String(), err, "failed to write random bytes")
			break
		} else {
			randBytesWritten += n
		}
	}
	if pseudoRand.Intn(100) < 2 {
		misc.DefaultLogger.Info("sockd.WriteRand", conn.RemoteAddr().String(), nil, "wrote %d rand bytes", randBytesWritten)
	}
}
<commit_msg>attempt to solve what seemed like a race condition causing TCP Write function to panic in sock<commit_after>package sockd

import (
	cryptRand "crypto/rand"
	"encoding/binary"
	"fmt"
	"github.com/HouzuoGuo/laitos/misc"
	"io"
	pseudoRand "math/rand"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	AddressTypeMask  byte = 0xf
	AddressTypeIndex      = 0
	AddressTypeIPv4       = 1
	AddressTypeDM         = 3
	AddressTypeIPv6       = 4

	IPPacketIndex    = 1
	IPv4PacketLength = net.IPv4len + 2
	IPv6PacketLength = net.IPv6len + 2

	DMAddrIndex        = 2
	DMAddrLengthIndex  = 1
	DMAddrHeaderLength = 2
)

var TCPDurationStats = misc.NewStats()

func (daemon *Daemon) StartAndBlockTCP() error {
	var err error
	listener, err := net.Listen("tcp", net.JoinHostPort(daemon.Address, strconv.Itoa(daemon.TCPPort)))
	if err != nil {
		return fmt.Errorf("sockd.StartAndBlockTCP: failed to listen on %s:%d - %v", daemon.Address, daemon.TCPPort, err)
	}
	defer listener.Close()
	daemon.logger.Info("StartAndBlockTCP", "", nil, "going to listen for connections")
	daemon.tcpListener = listener

	for {
		if misc.EmergencyLockDown {
			return misc.ErrEmergencyLockDown
		}
		conn, err := listener.Accept()
		if err != nil {
			if strings.Contains(err.Error(), "closed") {
				return nil
			} else {
				return fmt.Errorf("sockd.StartAndBlockTCP: failed to accept new connection - %v", err)
			}
		}
		clientIP := conn.RemoteAddr().(*net.TCPAddr).IP.String()
		if daemon.rateLimitTCP.Add(clientIP, true) {
			go NewTCPCipherConnection(daemon, conn, daemon.cipher.Copy(), daemon.logger).HandleTCPConnection()
		} else {
			conn.Close()
		}
	}
}

type TCPCipherConnection struct {
	net.Conn
	*Cipher
	daemon            *Daemon
	mutex             sync.Mutex
	readBuf, writeBuf []byte
	logger            misc.Logger
}

func NewTCPCipherConnection(daemon *Daemon, netConn net.Conn, cip *Cipher, logger misc.Logger) *TCPCipherConnection {
	return &TCPCipherConnection{
		Conn:     netConn,
		daemon:   daemon,
		Cipher:   cip,
		readBuf:  make([]byte, MaxPacketSize),
		writeBuf: make([]byte, MaxPacketSize),
		logger:   logger,
	}
}

func (conn *TCPCipherConnection) Close() error {
	return conn.Conn.Close()
}

func (conn *TCPCipherConnection) Read(b []byte) (n int, err error) {
	if conn.DecryptionStream == nil {
		iv := make([]byte, conn.IVLength)
		if _, err = io.ReadFull(conn.Conn, iv); err != nil {
			return
		}
		conn.InitDecryptionStream(iv)
		if len(conn.IV) == 0 {
			conn.IV = iv
		}
	}

	cipherData := conn.readBuf
	if len(b) > len(cipherData) {
		cipherData = make([]byte, len(b))
	} else {
		cipherData = cipherData[:len(b)]
	}

	n, err = conn.Conn.Read(cipherData)
	if n > 0 {
		conn.Decrypt(b[0:n], cipherData[0:n])
	}
	return
}

func (conn *TCPCipherConnection) Write(buf []byte) (n int, err error) {
	conn.mutex.Lock()
	bufSize := len(buf)
	headerLen := len(buf) - bufSize

	var iv []byte
	if conn.EncryptionStream == nil {
		iv = conn.InitEncryptionStream()
	}

	cipherData := conn.writeBuf
	dataSize := len(buf) + len(iv)
	if dataSize > len(cipherData) {
		cipherData = make([]byte, dataSize)
	} else {
		cipherData = cipherData[:dataSize]
	}

	if iv != nil {
		copy(cipherData, iv)
	}

	conn.Encrypt(cipherData[len(iv):], buf)
	n, err = conn.Conn.Write(cipherData)

	if n >= headerLen {
		n -= headerLen
	}
	conn.mutex.Unlock()
	return
}

func (conn *TCPCipherConnection) ParseRequest() (destIP, destAddr string, err error) {
	conn.SetReadDeadline(time.Now().Add(IOTimeoutSec))

	buf := make([]byte, 269)
	if _, err = io.ReadFull(conn, buf[:AddressTypeIndex+1]); err != nil {
		return
	}

	var reqStart, reqEnd int
	addrType := buf[AddressTypeIndex]
	maskedType := addrType & AddressTypeMask
	switch maskedType {
	case AddressTypeIPv4:
		reqStart, reqEnd = IPPacketIndex, IPPacketIndex+IPv4PacketLength
	case AddressTypeIPv6:
		reqStart, reqEnd = IPPacketIndex, IPPacketIndex+IPv6PacketLength
	case AddressTypeDM:
		if _, err = io.ReadFull(conn, buf[AddressTypeIndex+1:DMAddrLengthIndex+1]); err != nil {
			return
		}
		reqStart, reqEnd = DMAddrIndex, DMAddrIndex+int(buf[DMAddrLengthIndex])+DMAddrHeaderLength
	default:
		err = fmt.Errorf("TCPCipherConnection.ParseRequest: unknown mask type %d", maskedType)
		return
	}

	if _, err = io.ReadFull(conn, buf[reqStart:reqEnd]); err != nil {
		return
	}

	switch maskedType {
	case AddressTypeIPv4:
		destIP = net.IP(buf[IPPacketIndex : IPPacketIndex+net.IPv4len]).String()
	case AddressTypeIPv6:
		destIP = net.IP(buf[IPPacketIndex : IPPacketIndex+net.IPv6len]).String()
	case AddressTypeDM:
		destIP = string(buf[DMAddrIndex : DMAddrIndex+int(buf[DMAddrLengthIndex])])
	}
	port := binary.BigEndian.Uint16(buf[reqEnd-2 : reqEnd])
	if port < 1 {
		err = fmt.Errorf("TCPCipherConnection.ParseRequest: invalid destination port %d", port)
		return
	}
	destAddr = net.JoinHostPort(destIP, strconv.Itoa(int(port)))
	return
}

func (conn *TCPCipherConnection) WriteRandAndClose() {
	defer conn.Close()
	randBuf := make([]byte, RandNum(1, 20, 300))
	_, err := cryptRand.Read(randBuf)
	if err != nil {
		conn.logger.Warning("WriteRandAndClose", conn.Conn.RemoteAddr().String(), err, "failed to get random bytes")
		return
	}
	conn.SetDeadline(time.Now().Add(IOTimeoutSec))
	if _, err := conn.Write(randBuf); err != nil {
		conn.logger.Warning("WriteRandAndClose", conn.Conn.RemoteAddr().String(), err, "failed to write random bytes")
	}
}

func (conn *TCPCipherConnection) HandleTCPConnection() {
	beginTimeNano := time.Now().UnixNano()
	defer func() {
		TCPDurationStats.Trigger(float64(time.Now().UnixNano() - beginTimeNano))
	}()
	remoteAddr := conn.RemoteAddr().String()
	destIP, destAddr, err := conn.ParseRequest()
	if err != nil {
		conn.logger.Warning("HandleTCPConnection", remoteAddr, err, "failed to get destination address")
		conn.WriteRandAndClose()
		return
	}
	if strings.ContainsRune(destAddr, 0x00) {
		conn.logger.Warning("HandleTCPConnection", remoteAddr, nil, "will not serve invalid destination address with 0 in it")
		conn.WriteRandAndClose()
		return
	}
	if conn.daemon.DNSDaemon.IsInBlacklist(destIP) {
		conn.logger.Info("HandleTCPConnection", remoteAddr, nil, "will not serve blacklisted address %s", destIP)
		conn.Close()
		return
	}
	dest, err := net.DialTimeout("tcp", destAddr, IOTimeoutSec)
	if err != nil {
		conn.logger.Warning("HandleTCPConnection", remoteAddr, err, "failed to connect to destination \"%s\"", destAddr)
		conn.Close()
		return
	}
	go PipeTCPConnection(conn, dest, true)
	PipeTCPConnection(dest, conn, false)
	return
}

func PipeTCPConnection(fromConn, toConn net.Conn, doWriteRand bool) {
	defer toConn.Close()
	buf := make([]byte, MaxPacketSize)
	for {
		fromConn.SetReadDeadline(time.Now().Add(IOTimeoutSec))
		length, err := fromConn.Read(buf)
		if length > 0 {
			toConn.SetWriteDeadline(time.Now().Add(IOTimeoutSec))
			if _, err := toConn.Write(buf[:length]); err != nil {
				return
			}
		}
		if err != nil {
			if doWriteRand {
				WriteRand(fromConn)
			}
			return
		}
	}
}

func WriteRand(conn net.Conn) {
	randBytesWritten := 0
	for i := 0; i < RandNum(1, 2, 3); i++ {
		randBuf := make([]byte, RandNum(40, 60, 900))
		if _, err := pseudoRand.Read(randBuf); err != nil {
			misc.DefaultLogger.Warning("sockd.WriteRand", conn.RemoteAddr().String(), err, "failed to get random bytes")
			break
		}
		conn.SetDeadline(time.Now().Add(time.Duration(RandNum(140, 220, 350)) * time.Millisecond))
		if n, err := conn.Write(randBuf); err != nil && !strings.Contains(err.Error(), "closed") && !strings.Contains(err.Error(), "broken") {
			misc.DefaultLogger.Warning("sockd.WriteRand", conn.RemoteAddr().String(), err, "failed to write random bytes")
			break
		} else {
			randBytesWritten += n
		}
	}
	if pseudoRand.Intn(100) < 2 {
		misc.DefaultLogger.Info("sockd.WriteRand", conn.RemoteAddr().String(), nil, "wrote %d rand bytes", randBytesWritten)
	}
}
<|endoftext|>


<commit_before>/**
 * ws.go - A light weight webserver for static content
 * development and prototyping route based web API.
 *
 * Supports both http and https protocols. Dynamic route
 * processing available via Otto JavaScript virtual machines.
 *
 * @author R. S. Doiel, <rsdoiel@yahoo.com>
 * copyright (c) 2014
 * All rights reserved.
 * @license BSD 2-Clause License
 */
package main

import (
	"./cfg"
	"./fsengine"
	"./keygen"
	"./ottoengine"
	"./wslog"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

var revision = "v0.0.0-alpha"

// command line parameters that override environment variables
var (
	useTLS   bool
	docroot  string
	host     string
	port     int
	cert     string
	key      string
	otto     bool
	ottoPath string
	version  bool
	doKeygen bool
	doInit   bool
	help     bool
)

type stringValue string

var usage = func(exit_code int, msg string) {
	var fh = os.Stderr
	if exit_code == 0 {
		fh = os.Stdout
	}
	fmt.Fprintf(fh, `%s
 USAGE %s [options]

 EXAMPLES

 OPTIONS

`, msg, os.Args[0])

	flag.VisitAll(func(f *flag.Flag) {
		fmt.Fprintf(fh, "\t-%s\t(defaults to %s) %s\n", f.Name, f.DefValue, f.Usage)
	})

	fmt.Fprintf(fh, `

 copyright (c) 2014 all rights reserved.
 Released under the Simplified BSD License
 See: http://opensource.org/licenses/bsd-license.php

`)
	os.Exit(exit_code)
}

func requestLog(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		wslog.LogRequest(r.Method, r.URL, r.RemoteAddr, r.Proto, r.Referer(), r.UserAgent())
		handler.ServeHTTP(w, r)
	})
}

func webserver(config *cfg.Cfg) error {
	// If otto is enabled add routes and handle them.
	if config.Otto == true {
		ottoPath, err := filepath.Abs(config.OttoPath)
		if err != nil {
			log.Fatalf("Can't read %s: %s\n", config.OttoPath, err)
		}
		programs, err := ottoengine.Load(ottoPath)
		if err != nil {
			log.Fatalf("Load error: %s\n", err)
		}
		ottoengine.AddRoutes(programs)
	}

	// Restricted FileService excluding dot files and directories
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// hande off this request/response pair to the fsengine
		fsengine.Engine(config, w, r)
	})

	// Now start up the server and log transactions
	if config.UseTLS == true {
		if config.Cert == "" || config.Key == "" {
			log.Fatalf("TLS set true but missing key or certificate")
		}
		log.Println("Starting https://" + net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)))
		return http.ListenAndServeTLS(net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)), config.Cert, config.Key, requestLog(http.DefaultServeMux))
	}
	log.Println("Starting http://" + net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)))
	// Now start up the server and log transactions
	return http.ListenAndServe(net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)), requestLog(http.DefaultServeMux))
}

func defaultEnvBool(environmentVar string, defaultValue bool) bool {
	tmp := strings.ToLower(os.Getenv(environmentVar))
	if tmp == "true" {
		return true
	}
	if tmp == "false" {
		return false
	}
	return defaultValue
}

func defaultEnvString(environmentVar string, defaultValue string) string {
	tmp := os.Getenv(environmentVar)
	if tmp != "" {
		return tmp
	}
	return defaultValue
}

func defaultEnvInt(environmentVar string, defaultValue int) int {
	tmp := os.Getenv(environmentVar)
	if tmp != "" {
		i, err := strconv.Atoi(tmp)
		if err != nil {
			usage(1, environmentVar+" must be an integer.")
		}
		return i
	}
	return defaultValue
}

func init() {
	const (
		helpUsage     = "This help document."
		keygenUsage   = "Interactive tool to generate TLS certificates and keys"
		initUsage     = "Creates a basic project structure in the current working directory"
		useTLSUsage   = "When true this turns on TLS (https) support."
		keyUsage      = "Path to your SSL key pem file."
		certUsage     = "path to your SSL cert pem file."
		docrootUsage  = "This is your document root for static files."
		hostUsage     = "Set this hostname for webserver."
		portUsage     = "Set the port number to listen on."
		ottoUsage     = "When true this option turns on ottoengine. Uses the path defined by WS_OTTO_PATH environment variable or one provided by -O option."
		ottoPathUsage = "Turns on otto engine using the path for route JavaScript route handlers"
		versionUsage  = "Display the version number of ws command."
	)

	flag.BoolVar(&help, "help", false, helpUsage)
	flag.BoolVar(&help, "h", false, helpUsage)
	flag.BoolVar(&doKeygen, "keygen", false, keygenUsage)
	flag.BoolVar(&doInit, "init", false, initUsage)
	flag.BoolVar(&version, "version", false, versionUsage)
	flag.BoolVar(&version, "v", false, versionUsage)

	// Settable via environment
	useTLS = defaultEnvBool("WS_TLS", false)
	key = defaultEnvString("WS_KEY", "")
	cert = defaultEnvString("WS_CERT", "")
	docroot = defaultEnvString("WS_DOCROOT", "")
	otto = defaultEnvBool("WS_OTTO", false)
	ottoPath = defaultEnvString("WS_OTTO_PATH", "")
	host = defaultEnvString("WS_HOST", "localhost")
	port = defaultEnvInt("WS_PORT", 8000)

	flag.BoolVar(&useTLS, "tls", useTLS, useTLSUsage)
	flag.StringVar(&key, "key", key, keyUsage)
	flag.StringVar(&cert, "cert", cert, certUsage)
	flag.StringVar(&docroot, "docroot", docroot, docrootUsage)
	flag.StringVar(&docroot, "D", docroot, docrootUsage)
	flag.StringVar(&host, "host", host, hostUsage)
	flag.StringVar(&host, "H", host, hostUsage)
	flag.IntVar(&port, "port", port, portUsage)
	flag.IntVar(&port, "P", port, portUsage)
	flag.BoolVar(&otto, "otto", otto, ottoUsage)
	flag.BoolVar(&otto, "o", otto, ottoUsage)
	flag.StringVar(&ottoPath, "otto-path", ottoPath, ottoPathUsage)
	flag.StringVar(&ottoPath, "O", ottoPath, ottoPathUsage)
}

func main() {
	flag.Parse()
	if version == true {
		fmt.Println(revision)
		os.Exit(0)
	}
	if help == true {
		usage(0, "")
	}

	config, err := cfg.Configure(docroot, host, port, useTLS, cert, key, otto, ottoPath)
	if err != nil {
		usage(1, fmt.Sprintf("%s", err))
	}

	if doKeygen == true {
		certFilename, keyFilename, err := keygen.Keygen("etc/ssl", "cert.pem", "key.pem")
		if err != nil {
			log.Fatalf("%s\n", err)
		}
		fmt.Printf("Wrote %s and %s\n", certFilename, keyFilename)
		os.Exit(0)
	}

	if doInit == true {
		err := cfg.InitProject()
		if err != nil {
			log.Fatalf("%s\n", err)
		}
		os.Exit(0)
	}

	fmt.Printf("\n\n"+
		"          TLS: %t\n"+
		"         Cert: %s\n"+
		"          Key: %s\n"+
		"      Docroot: %s\n"+
		"         Host: %s\n"+
		"         Port: %d\n"+
		"       Run as: %s\n\n"+
		" Otto enabled: %t\n"+
		"         Path: %s\n"+
		"\n\n",
		config.UseTLS,
		config.Cert,
		config.Key,
		config.Docroot,
		config.Hostname,
		config.Port,
		config.Username,
		config.Otto,
		config.OttoPath)
	err = webserver(config)
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}
<commit_msg>bumping version number<commit_after>/**
 * ws.go - A light weight webserver for static content
 * development and prototyping route based web API.
 *
 * Supports both http and https protocols. Dynamic route
 * processing available via Otto JavaScript virtual machines.
 *
 * @author R. S. Doiel, <rsdoiel@yahoo.com>
 * copyright (c) 2014
 * All rights reserved.
 * @license BSD 2-Clause License
 */
package main

import (
	"./cfg"
	"./fsengine"
	"./keygen"
	"./ottoengine"
	"./wslog"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

var revision = "v0.0.1"

// command line parameters that override environment variables
var (
	useTLS   bool
	docroot  string
	host     string
	port     int
	cert     string
	key      string
	otto     bool
	ottoPath string
	version  bool
	doKeygen bool
	doInit   bool
	help     bool
)

type stringValue string

var usage = func(exit_code int, msg string) {
	var fh = os.Stderr
	if exit_code == 0 {
		fh = os.Stdout
	}
	fmt.Fprintf(fh, `%s
 USAGE %s [options]

 EXAMPLES

 OPTIONS

`, msg, os.Args[0])

	flag.VisitAll(func(f *flag.Flag) {
		fmt.Fprintf(fh, "\t-%s\t(defaults to %s) %s\n", f.Name, f.DefValue, f.Usage)
	})

	fmt.Fprintf(fh, `

 copyright (c) 2014 all rights reserved.
 Released under the Simplified BSD License
 See: http://opensource.org/licenses/bsd-license.php

`)
	os.Exit(exit_code)
}

func requestLog(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		wslog.LogRequest(r.Method, r.URL, r.RemoteAddr, r.Proto, r.Referer(), r.UserAgent())
		handler.ServeHTTP(w, r)
	})
}

func webserver(config *cfg.Cfg) error {
	// If otto is enabled add routes and handle them.
	if config.Otto == true {
		ottoPath, err := filepath.Abs(config.OttoPath)
		if err != nil {
			log.Fatalf("Can't read %s: %s\n", config.OttoPath, err)
		}
		programs, err := ottoengine.Load(ottoPath)
		if err != nil {
			log.Fatalf("Load error: %s\n", err)
		}
		ottoengine.AddRoutes(programs)
	}

	// Restricted FileService excluding dot files and directories
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// hande off this request/response pair to the fsengine
		fsengine.Engine(config, w, r)
	})

	// Now start up the server and log transactions
	if config.UseTLS == true {
		if config.Cert == "" || config.Key == "" {
			log.Fatalf("TLS set true but missing key or certificate")
		}
		log.Println("Starting https://" + net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)))
		return http.ListenAndServeTLS(net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)), config.Cert, config.Key, requestLog(http.DefaultServeMux))
	}
	log.Println("Starting http://" + net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)))
	// Now start up the server and log transactions
	return http.ListenAndServe(net.JoinHostPort(config.Hostname, strconv.Itoa(config.Port)), requestLog(http.DefaultServeMux))
}

func defaultEnvBool(environmentVar string, defaultValue bool) bool {
	tmp := strings.ToLower(os.Getenv(environmentVar))
	if tmp == "true" {
		return true
	}
	if tmp == "false" {
		return false
	}
	return defaultValue
}

func defaultEnvString(environmentVar string, defaultValue string) string {
	tmp := os.Getenv(environmentVar)
	if tmp != "" {
		return tmp
	}
	return defaultValue
}

func defaultEnvInt(environmentVar string, defaultValue int) int {
	tmp := os.Getenv(environmentVar)
	if tmp != "" {
		i, err := strconv.Atoi(tmp)
		if err != nil {
			usage(1, environmentVar+" must be an integer.")
		}
		return i
	}
	return defaultValue
}

func init() {
	const (
		helpUsage     = "This help document."
		keygenUsage   = "Interactive tool to generate TLS certificates and keys"
		initUsage     = "Creates a basic project structure in the current working directory"
		useTLSUsage   = "When true this turns on TLS (https) support."
		keyUsage      = "Path to your SSL key pem file."
		certUsage     = "path to your SSL cert pem file."
		docrootUsage  = "This is your document root for static files."
		hostUsage     = "Set this hostname for webserver."
		portUsage     = "Set the port number to listen on."
		ottoUsage     = "When true this option turns on ottoengine. Uses the path defined by WS_OTTO_PATH environment variable or one provided by -O option."
		ottoPathUsage = "Turns on otto engine using the path for route JavaScript route handlers"
		versionUsage  = "Display the version number of ws command."
	)

	flag.BoolVar(&help, "help", false, helpUsage)
	flag.BoolVar(&help, "h", false, helpUsage)
	flag.BoolVar(&doKeygen, "keygen", false, keygenUsage)
	flag.BoolVar(&doInit, "init", false, initUsage)
	flag.BoolVar(&version, "version", false, versionUsage)
	flag.BoolVar(&version, "v", false, versionUsage)

	// Settable via environment
	useTLS = defaultEnvBool("WS_TLS", false)
	key = defaultEnvString("WS_KEY", "")
	cert = defaultEnvString("WS_CERT", "")
	docroot = defaultEnvString("WS_DOCROOT", "")
	otto = defaultEnvBool("WS_OTTO", false)
	ottoPath = defaultEnvString("WS_OTTO_PATH", "")
	host = defaultEnvString("WS_HOST", "localhost")
	port = defaultEnvInt("WS_PORT", 8000)

	flag.BoolVar(&useTLS, "tls", useTLS, useTLSUsage)
	flag.StringVar(&key, "key", key, keyUsage)
	flag.StringVar(&cert, "cert", cert, certUsage)
	flag.StringVar(&docroot, "docroot", docroot, docrootUsage)
	flag.StringVar(&docroot, "D", docroot, docrootUsage)
	flag.StringVar(&host, "host", host, hostUsage)
	flag.StringVar(&host, "H", host, hostUsage)
	flag.IntVar(&port, "port", port, portUsage)
	flag.IntVar(&port, "P", port, portUsage)
	flag.BoolVar(&otto, "otto", otto, ottoUsage)
	flag.BoolVar(&otto, "o", otto, ottoUsage)
	flag.StringVar(&ottoPath, "otto-path", ottoPath, ottoPathUsage)
	flag.StringVar(&ottoPath, "O", ottoPath, ottoPathUsage)
}

func main() {
	flag.Parse()
	if version == true {
		fmt.Println(revision)
		os.Exit(0)
	}
	if help == true {
		usage(0, "")
	}

	config, err := cfg.Configure(docroot, host, port, useTLS, cert, key, otto, ottoPath)
	if err != nil {
		usage(1, fmt.Sprintf("%s", err))
	}

	if doKeygen == true {
		certFilename, keyFilename, err := keygen.Keygen("etc/ssl", "cert.pem", "key.pem")
		if err != nil {
			log.Fatalf("%s\n", err)
		}
		fmt.Printf("Wrote %s and %s\n", certFilename, keyFilename)
		os.Exit(0)
	}

	if doInit == true {
		err := cfg.InitProject()
		if err != nil {
			log.Fatalf("%s\n", err)
		}
		os.Exit(0)
	}

	fmt.Printf("\n\n"+
		"          TLS: %t\n"+
		"         Cert: %s\n"+
		"          Key: %s\n"+
		"      Docroot: %s\n"+
		"         Host: %s\n"+
		"         Port: %d\n"+
		"       Run as: %s\n\n"+
		" Otto enabled: %t\n"+
		"         Path: %s\n"+
		"\n\n",
		config.UseTLS,
		config.Cert,
		config.Key,
		config.Docroot,
		config.Hostname,
		config.Port,
		config.Username,
		config.Otto,
		config.OttoPath)
	err = webserver(config)
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>package gps

import (
	"fmt"
	"sync"
)

type source interface {
	syncLocal() error
	checkExistence(sourceExistence) bool
	exportVersionTo(Version, string) error
	getManifestAndLock(ProjectRoot, Version) (Manifest, Lock, error)
	listPackages(ProjectRoot, Version) (PackageTree, error)
	listVersions() ([]Version, error)
	revisionPresentIn(Revision) (bool, error)
}

type sourceMetaCache struct {
	//Version  string                   // TODO(sdboyer) use this
	infos  map[Revision]projectInfo
	ptrees map[Revision]PackageTree
	vMap   map[UnpairedVersion]Revision
	rMap   map[Revision][]UnpairedVersion
	// TODO(sdboyer) mutexes. actually probably just one, b/c complexity
}

// projectInfo holds manifest and lock
type projectInfo struct {
	Manifest
	Lock
}

type existence struct {
	// The existence levels for which a search/check has been performed
	s sourceExistence

	// The existence levels verified to be present through searching
	f sourceExistence
}

func newMetaCache() *sourceMetaCache {
	return &sourceMetaCache{
		infos:  make(map[Revision]projectInfo),
		ptrees: make(map[Revision]PackageTree),
		vMap:   make(map[UnpairedVersion]Revision),
		rMap:   make(map[Revision][]UnpairedVersion),
	}
}

type baseVCSSource struct {
	// Object for the cache repository
	crepo *repo

	// Indicates the extent to which we have searched for, and verified, the
	// existence of the project/repo.
	ex existence

	// ProjectAnalyzer used to fulfill getManifestAndLock
	an ProjectAnalyzer

	// The project metadata cache. This is (or is intended to be) persisted to
	// disk, for reuse across solver runs.
	dc *sourceMetaCache

	// lvfunc allows the other vcs source types that embed this type to inject
	// their listVersions func into the baseSource, for use as needed.
	lvfunc func() (vlist []Version, err error)

	// Mutex to ensure only one listVersions runs at a time
	//
	// TODO(sdboyer) this is a horrible one-off hack, and must be removed once
	// source managers are refactored to properly serialize and fold-in calls to
	// these methods.
	lvmut sync.Mutex

	// Once-er to control access to syncLocal
	synconce sync.Once

	// The error, if any, that occurred on syncLocal
	syncerr error

	// Whether the cache has the latest info on versions
	cvsync bool
}

func (bs *baseVCSSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
	if err := bs.ensureCacheExistence(); err != nil {
		return nil, nil, err
	}

	rev, err := bs.toRevOrErr(v)
	if err != nil {
		return nil, nil, err
	}

	// Return the info from the cache, if we already have it
	if pi, exists := bs.dc.infos[rev]; exists {
		return pi.Manifest, pi.Lock, nil
	}

	// Cache didn't help; ensure our local is fully up to date.
	do := func() (err error) {
		bs.crepo.mut.Lock()
		// Always prefer a rev, if it's available
		if pv, ok := v.(PairedVersion); ok {
			err = bs.crepo.r.UpdateVersion(pv.Underlying().String())
		} else {
			err = bs.crepo.r.UpdateVersion(v.String())
		}

		bs.crepo.mut.Unlock()
		return
	}

	if err = do(); err != nil {
		// minimize network activity: only force local syncing if we had an err
		err = bs.syncLocal()
		if err != nil {
			return nil, nil, err
		}

		if err = do(); err != nil {
			// TODO(sdboyer) More-er proper-er error
			panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", bs.crepo.r.LocalPath(), v.String(), unwrapVcsErr(err)))
		}
	}

	bs.crepo.mut.RLock()
	m, l, err := bs.an.DeriveManifestAndLock(bs.crepo.r.LocalPath(), r)
	// TODO(sdboyer) cache results
	bs.crepo.mut.RUnlock()

	if err == nil {
		if l != nil {
			l = prepLock(l)
		}

		// If m is nil, prepManifest will provide an empty one.
		pi := projectInfo{
			Manifest: prepManifest(m),
			Lock:     l,
		}

		bs.dc.infos[rev] = pi

		return pi.Manifest, pi.Lock, nil
	}

	return nil, nil, unwrapVcsErr(err)
}

// toRevision turns a Version into a Revision, if doing so is possible based on
// the information contained in the version itself, or in the cache maps.
func (dc *sourceMetaCache) toRevision(v Version) Revision {
	switch t := v.(type) {
	case Revision:
		return t
	case PairedVersion:
		return t.Underlying()
	case UnpairedVersion:
		// This will return the empty rev (empty string) if we don't have a
		// record of it. It's up to the caller to decide, for example, if
		// it's appropriate to update the cache.
		return dc.vMap[t]
	default:
		panic(fmt.Sprintf("Unknown version type %T", v))
	}
}

// toUnpaired turns a Version into an UnpairedVersion, if doing so is possible
// based on the information contained in the version itself, or in the cache
// maps.
//
// If the input is a revision and multiple UnpairedVersions are associated with
// it, whatever happens to be the first is returned.
func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
	switch t := v.(type) {
	case UnpairedVersion:
		return t
	case PairedVersion:
		return t.Unpair()
	case Revision:
		if upv, has := dc.rMap[t]; has && len(upv) > 0 {
			return upv[0]
		}
		return nil
	default:
		panic(fmt.Sprintf("unknown version type %T", v))
	}
}

func (bs *baseVCSSource) revisionPresentIn(r Revision) (bool, error) {
	// First and fastest path is to check the data cache to see if the rev is
	// present. This could give us false positives, but the cases where that can
	// occur would require a type of cache staleness that seems *exceedingly*
	// unlikely to occur.
	if _, has := bs.dc.infos[r]; has {
		return true, nil
	} else if _, has := bs.dc.rMap[r]; has {
		return true, nil
	}

	err := bs.ensureCacheExistence()
	if err != nil {
		return false, err
	}

	bs.crepo.mut.RLock()
	defer bs.crepo.mut.RUnlock()
	return bs.crepo.r.IsReference(string(r)), nil
}

func (bs *baseVCSSource) ensureCacheExistence() error {
	// Technically, methods could could attempt to return straight from the
	// metadata cache even if the repo cache doesn't exist on disk. But that
	// would allow weird state inconsistencies (cache exists, but no repo...how
	// does that even happen?) that it'd be better to just not allow so that we
	// don't have to think about it elsewhere
	if !bs.checkExistence(existsInCache) {
		if bs.checkExistence(existsUpstream) {
			bs.crepo.mut.Lock()
			if bs.crepo.synced {
				// A second ensure call coming in while the first is completing
				// isn't terribly unlikely, especially for a large repo. In that
				// event, the synced flag will have flipped on by the time we
				// acquire the lock. If it has, there's no need to do this work
				// twice.
				bs.crepo.mut.Unlock()
				return nil
			}

			err := bs.crepo.r.Get()

			if err != nil {
				bs.crepo.mut.Unlock()
				return fmt.Errorf("failed to create repository cache for %s with err:\n%s", bs.crepo.r.Remote(), unwrapVcsErr(err))
			}

			bs.crepo.synced = true
			bs.ex.s |= existsInCache
			bs.ex.f |= existsInCache
			bs.crepo.mut.Unlock()
		} else {
			return fmt.Errorf("project %s does not exist upstream", bs.crepo.r.Remote())
		}
	}

	return nil
}

// checkExistence provides a direct method for querying existence levels of the
// source. It will only perform actual searching (local fs or over the network)
// if no previous attempt at that search has been made.
//
// Note that this may perform read-ish operations on the cache repo, and it
// takes a lock accordingly. This makes it unsafe to call from a segment where
// the cache repo mutex is already write-locked, as deadlock will occur.
func (bs *baseVCSSource) checkExistence(ex sourceExistence) bool {
	if bs.ex.s&ex != ex {
		if ex&existsInVendorRoot != 0 && bs.ex.s&existsInVendorRoot == 0 {
			panic("should now be implemented in bridge")
		}
		if ex&existsInCache != 0 && bs.ex.s&existsInCache == 0 {
			bs.crepo.mut.RLock()
			bs.ex.s |= existsInCache
			if bs.crepo.r.CheckLocal() {
				bs.ex.f |= existsInCache
			}
			bs.crepo.mut.RUnlock()
		}
		if ex&existsUpstream != 0 && bs.ex.s&existsUpstream == 0 {
			bs.crepo.mut.RLock()
			bs.ex.s |= existsUpstream
			if bs.crepo.r.Ping() {
				bs.ex.f |= existsUpstream
			}
			bs.crepo.mut.RUnlock()
		}
	}

	return ex&bs.ex.f == ex
}

// syncLocal ensures the local data we have about the source is fully up to date
// with what's out there over the network.
func (bs *baseVCSSource) syncLocal() error {
	// Ensure we only have one goroutine doing this at a time
	f := func() {
		// First, ensure the local instance exists
		bs.syncerr = bs.ensureCacheExistence()
		if bs.syncerr != nil {
			return
		}

		_, bs.syncerr = bs.lvfunc()
		if bs.syncerr != nil {
			return
		}

		// This case is really just for git repos, where the lvfunc doesn't
		// guarantee that the local repo is synced
		if !bs.crepo.synced {
			bs.crepo.mut.Lock()
			err := bs.crepo.r.Update()
			if err != nil {
				bs.syncerr = fmt.Errorf("failed fetching latest updates with err: %s", unwrapVcsErr(err))
				bs.crepo.mut.Unlock()
			}
			bs.crepo.synced = true
			bs.crepo.mut.Unlock()
		}
	}

	bs.synconce.Do(f)
	return bs.syncerr
}

func (bs *baseVCSSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
	if err = bs.ensureCacheExistence(); err != nil {
		return
	}

	var r Revision
	if r, err = bs.toRevOrErr(v); err != nil {
		return
	}

	// Return the ptree from the cache, if we already have it
	var exists bool
	if ptree, exists = bs.dc.ptrees[r]; exists {
		return
	}

	// Not in the cache; check out the version and do the analysis
	bs.crepo.mut.Lock()
	// Check out the desired version for analysis
	if r != "" {
		// Always prefer a rev, if it's available
		err = bs.crepo.r.UpdateVersion(string(r))
	} else {
		// If we don't have a rev, ensure the repo is up to date, otherwise we
		// could have a desync issue
		if !bs.crepo.synced {
			err = bs.crepo.r.Update()
			if err != nil {
				err = fmt.Errorf("could not fetch latest updates into repository: %s", unwrapVcsErr(err))
				return
			}
			bs.crepo.synced = true
		}
		err = bs.crepo.r.UpdateVersion(v.String())
	}

	if err == nil {
		ptree, err = ListPackages(bs.crepo.r.LocalPath(), string(pr))
		// TODO(sdboyer) cache errs?
		if err == nil {
			bs.dc.ptrees[r] = ptree
		}
	} else {
		err = unwrapVcsErr(err)
	}
	bs.crepo.mut.Unlock()

	return
}

// toRevOrErr makes all efforts to convert a Version into a rev, including
// updating the cache repo (if needed). It does not guarantee that the returned
// Revision actually exists in the repository (as one of the cheaper methods may
// have had bad data).
func (bs *baseVCSSource) toRevOrErr(v Version) (r Revision, err error) {
	r = bs.dc.toRevision(v)
	if r == "" {
		// Rev can be empty if:
		//  - The cache is unsynced
		//  - A version was passed that used to exist, but no longer does
		//  - A garbage version was passed. (Functionally indistinguishable from
		//  the previous)
		if !bs.cvsync {
			// call the lvfunc to sync the meta cache
			_, err = bs.lvfunc()
			if err != nil {
				return
			}
		}

		r = bs.dc.toRevision(v)
		// If we still don't have a rev, then the version's no good
		if r == "" {
			err = fmt.Errorf("version %s does not exist in source %s", v, bs.crepo.r.Remote())
		}
	}

	return
}

func (bs *baseVCSSource) exportVersionTo(v Version, to string) error {
	return bs.crepo.exportVersionTo(v, to)
}
<commit_msg>Fix bad conditional in syncLocal<commit_after>package gps

import (
	"fmt"
	"sync"
)

type source interface {
	syncLocal() error
	checkExistence(sourceExistence) bool
	exportVersionTo(Version, string) error
	getManifestAndLock(ProjectRoot, Version) (Manifest, Lock, error)
	listPackages(ProjectRoot, Version) (PackageTree, error)
	listVersions() ([]Version, error)
	revisionPresentIn(Revision) (bool, error)
}

type sourceMetaCache struct {
	//Version  string                   // TODO(sdboyer) use this
	infos  map[Revision]projectInfo
	ptrees map[Revision]PackageTree
	vMap   map[UnpairedVersion]Revision
	rMap   map[Revision][]UnpairedVersion
	// TODO(sdboyer) mutexes. actually probably just one, b/c complexity
}

// projectInfo holds manifest and lock
type projectInfo struct {
	Manifest
	Lock
}

type existence struct {
	// The existence levels for which a search/check has been performed
	s sourceExistence

	// The existence levels verified to be present through searching
	f sourceExistence
}

func newMetaCache() *sourceMetaCache {
	return &sourceMetaCache{
		infos:  make(map[Revision]projectInfo),
		ptrees: make(map[Revision]PackageTree),
		vMap:   make(map[UnpairedVersion]Revision),
		rMap:   make(map[Revision][]UnpairedVersion),
	}
}

type baseVCSSource struct {
	// Object for the cache repository
	crepo *repo

	// Indicates the extent to which we have searched for, and verified, the
	// existence of the project/repo.
	ex existence

	// ProjectAnalyzer used to fulfill getManifestAndLock
	an ProjectAnalyzer

	// The project metadata cache. This is (or is intended to be) persisted to
	// disk, for reuse across solver runs.
	dc *sourceMetaCache

	// lvfunc allows the other vcs source types that embed this type to inject
	// their listVersions func into the baseSource, for use as needed.
	lvfunc func() (vlist []Version, err error)

	// Mutex to ensure only one listVersions runs at a time
	//
	// TODO(sdboyer) this is a horrible one-off hack, and must be removed once
	// source managers are refactored to properly serialize and fold-in calls to
	// these methods.
	lvmut sync.Mutex

	// Once-er to control access to syncLocal
	synconce sync.Once

	// The error, if any, that occurred on syncLocal
	syncerr error

	// Whether the cache has the latest info on versions
	cvsync bool
}

func (bs *baseVCSSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
	if err := bs.ensureCacheExistence(); err != nil {
		return nil, nil, err
	}

	rev, err := bs.toRevOrErr(v)
	if err != nil {
		return nil, nil, err
	}

	// Return the info from the cache, if we already have it
	if pi, exists := bs.dc.infos[rev]; exists {
		return pi.Manifest, pi.Lock, nil
	}

	// Cache didn't help; ensure our local is fully up to date.
	do := func() (err error) {
		bs.crepo.mut.Lock()
		// Always prefer a rev, if it's available
		if pv, ok := v.(PairedVersion); ok {
			err = bs.crepo.r.UpdateVersion(pv.Underlying().String())
		} else {
			err = bs.crepo.r.UpdateVersion(v.String())
		}

		bs.crepo.mut.Unlock()
		return
	}

	if err = do(); err != nil {
		// minimize network activity: only force local syncing if we had an err
		err = bs.syncLocal()
		if err != nil {
			return nil, nil, err
		}

		if err = do(); err != nil {
			// TODO(sdboyer) More-er proper-er error
			panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", bs.crepo.r.LocalPath(), v.String(), unwrapVcsErr(err)))
		}
	}

	bs.crepo.mut.RLock()
	m, l, err := bs.an.DeriveManifestAndLock(bs.crepo.r.LocalPath(), r)
	// TODO(sdboyer) cache results
	bs.crepo.mut.RUnlock()

	if err == nil {
		if l != nil {
			l = prepLock(l)
		}

		// If m is nil, prepManifest will provide an empty one.
		pi := projectInfo{
			Manifest: prepManifest(m),
			Lock:     l,
		}

		bs.dc.infos[rev] = pi

		return pi.Manifest, pi.Lock, nil
	}

	return nil, nil, unwrapVcsErr(err)
}

// toRevision turns a Version into a Revision, if doing so is possible based on
// the information contained in the version itself, or in the cache maps.
func (dc *sourceMetaCache) toRevision(v Version) Revision {
	switch t := v.(type) {
	case Revision:
		return t
	case PairedVersion:
		return t.Underlying()
	case UnpairedVersion:
		// This will return the empty rev (empty string) if we don't have a
		// record of it. It's up to the caller to decide, for example, if
		// it's appropriate to update the cache.
		return dc.vMap[t]
	default:
		panic(fmt.Sprintf("Unknown version type %T", v))
	}
}

// toUnpaired turns a Version into an UnpairedVersion, if doing so is possible
// based on the information contained in the version itself, or in the cache
// maps.
//
// If the input is a revision and multiple UnpairedVersions are associated with
// it, whatever happens to be the first is returned.
func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
	switch t := v.(type) {
	case UnpairedVersion:
		return t
	case PairedVersion:
		return t.Unpair()
	case Revision:
		if upv, has := dc.rMap[t]; has && len(upv) > 0 {
			return upv[0]
		}
		return nil
	default:
		panic(fmt.Sprintf("unknown version type %T", v))
	}
}

func (bs *baseVCSSource) revisionPresentIn(r Revision) (bool, error) {
	// First and fastest path is to check the data cache to see if the rev is
	// present. This could give us false positives, but the cases where that can
	// occur would require a type of cache staleness that seems *exceedingly*
	// unlikely to occur.
	if _, has := bs.dc.infos[r]; has {
		return true, nil
	} else if _, has := bs.dc.rMap[r]; has {
		return true, nil
	}

	err := bs.ensureCacheExistence()
	if err != nil {
		return false, err
	}

	bs.crepo.mut.RLock()
	defer bs.crepo.mut.RUnlock()
	return bs.crepo.r.IsReference(string(r)), nil
}

func (bs *baseVCSSource) ensureCacheExistence() error {
	// Technically, methods could could attempt to return straight from the
	// metadata cache even if the repo cache doesn't exist on disk. But that
	// would allow weird state inconsistencies (cache exists, but no repo...how
	// does that even happen?) that it'd be better to just not allow so that we
	// don't have to think about it elsewhere
	if !bs.checkExistence(existsInCache) {
		if bs.checkExistence(existsUpstream) {
			bs.crepo.mut.Lock()
			if bs.crepo.synced {
				// A second ensure call coming in while the first is completing
				// isn't terribly unlikely, especially for a large repo. In that
				// event, the synced flag will have flipped on by the time we
				// acquire the lock. If it has, there's no need to do this work
				// twice.
				bs.crepo.mut.Unlock()
				return nil
			}

			err := bs.crepo.r.Get()

			if err != nil {
				bs.crepo.mut.Unlock()
				return fmt.Errorf("failed to create repository cache for %s with err:\n%s", bs.crepo.r.Remote(), unwrapVcsErr(err))
			}

			bs.crepo.synced = true
			bs.ex.s |= existsInCache
			bs.ex.f |= existsInCache
			bs.crepo.mut.Unlock()
		} else {
			return fmt.Errorf("project %s does not exist upstream", bs.crepo.r.Remote())
		}
	}

	return nil
}

// checkExistence provides a direct method for querying existence levels of the
// source. It will only perform actual searching (local fs or over the network)
// if no previous attempt at that search has been made.
//
// Note that this may perform read-ish operations on the cache repo, and it
// takes a lock accordingly. This makes it unsafe to call from a segment where
// the cache repo mutex is already write-locked, as deadlock will occur.
func (bs *baseVCSSource) checkExistence(ex sourceExistence) bool {
	if bs.ex.s&ex != ex {
		if ex&existsInVendorRoot != 0 && bs.ex.s&existsInVendorRoot == 0 {
			panic("should now be implemented in bridge")
		}
		if ex&existsInCache != 0 && bs.ex.s&existsInCache == 0 {
			bs.crepo.mut.RLock()
			bs.ex.s |= existsInCache
			if bs.crepo.r.CheckLocal() {
				bs.ex.f |= existsInCache
			}
			bs.crepo.mut.RUnlock()
		}
		if ex&existsUpstream != 0 && bs.ex.s&existsUpstream == 0 {
			bs.crepo.mut.RLock()
			bs.ex.s |= existsUpstream
			if bs.crepo.r.Ping() {
				bs.ex.f |= existsUpstream
			}
			bs.crepo.mut.RUnlock()
		}
	}

	return ex&bs.ex.f == ex
}

// syncLocal ensures the local data we have about the source is fully up to date
// with what's out there over the network.
func (bs *baseVCSSource) syncLocal() error {
	// Ensure we only have one goroutine doing this at a time
	f := func() {
		// First, ensure the local instance exists
		bs.syncerr = bs.ensureCacheExistence()
		if bs.syncerr != nil {
			return
		}

		_, bs.syncerr = bs.lvfunc()
		if bs.syncerr != nil {
			return
		}

		// This case is really just for git repos, where the lvfunc doesn't
		// guarantee that the local repo is synced
		if !bs.crepo.synced {
			bs.crepo.mut.Lock()
			err := bs.crepo.r.Update()
			if err != nil {
				bs.syncerr = fmt.Errorf("failed fetching latest updates with err: %s", unwrapVcsErr(err))
			} else {
				bs.crepo.synced = true
			}
			bs.crepo.mut.Unlock()
		}
	}

	bs.synconce.Do(f)
	return bs.syncerr
}

func (bs *baseVCSSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
	if err = bs.ensureCacheExistence(); err != nil {
		return
	}

	var r Revision
	if r, err = bs.toRevOrErr(v); err != nil {
		return
	}

	// Return the ptree from the cache, if we already have it
	var exists bool
	if ptree, exists = bs.dc.ptrees[r]; exists {
		return
	}

	// Not in the cache; check out the version and do the analysis
	bs.crepo.mut.Lock()
	// Check out the desired version for analysis
	if r != "" {
		// Always prefer a rev, if it's available
		err = bs.crepo.r.UpdateVersion(string(r))
	} else {
		// If we don't have a rev, ensure the repo is up to date, otherwise we
		// could have a desync issue
		if !bs.crepo.synced {
			err = bs.crepo.r.Update()
			if err != nil {
				err = fmt.Errorf("could not fetch latest updates into repository: %s", unwrapVcsErr(err))
				return
			}
			bs.crepo.synced = true
		}
		err = bs.crepo.r.UpdateVersion(v.String())
	}

	if err == nil {
		ptree, err = ListPackages(bs.crepo.r.LocalPath(), string(pr))
		// TODO(sdboyer) cache errs?
		if err == nil {
			bs.dc.ptrees[r] = ptree
		}
	} else {
		err = unwrapVcsErr(err)
	}
	bs.crepo.mut.Unlock()

	return
}

// toRevOrErr makes all efforts to convert a Version into a rev, including
// updating the cache repo (if needed). It does not guarantee that the returned
// Revision actually exists in the repository (as one of the cheaper methods may
// have had bad data).
func (bs *baseVCSSource) toRevOrErr(v Version) (r Revision, err error) {
	r = bs.dc.toRevision(v)
	if r == "" {
		// Rev can be empty if:
		//  - The cache is unsynced
		//  - A version was passed that used to exist, but no longer does
		//  - A garbage version was passed. (Functionally indistinguishable from
		//  the previous)
		if !bs.cvsync {
			// call the lvfunc to sync the meta cache
			_, err = bs.lvfunc()
			if err != nil {
				return
			}
		}

		r = bs.dc.toRevision(v)
		// If we still don't have a rev, then the version's no good
		if r == "" {
			err = fmt.Errorf("version %s does not exist in source %s", v, bs.crepo.r.Remote())
		}
	}

	return
}

func (bs *baseVCSSource) exportVersionTo(v Version, to string) error {
	return bs.crepo.exportVersionTo(v, to)
}
<|endoftext|>


<commit_before>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Package surfer is a high level concurrency http client.
//
// It has `surf` and` phantom` download engines, highly simulated browser behavior, the function of analog login and so on.
//
// Features:
// - Both surf and phantomjs engines are supported
// - Support random User-Agent
// - Support cache cookie
// - Support http/https
//
// Usage:
// package main
//
// import (
//     "github.com/henrylee2cn/surfer"
//     "io/ioutil"
//     "log"
// )
//
// func main() {
//     // Use surf engine
//     resp, err := surfer.Download(&surfer.Request{
//         Url: "http://github.com/henrylee2cn/surfer",
//     })
//     if err != nil {
//         log.Fatal(err)
//     }
//     b, err := ioutil.ReadAll(resp.Body)
//     log.Println(string(b), err)
//
//     // Use phantomjs engine
//     resp, err = surfer.Download(&surfer.Request{
//         Url:          "http://github.com/henrylee2cn",
//         DownloaderID: 1,
//     })
//     if err != nil {
//         log.Fatal(err)
//     }
//     b, err = ioutil.ReadAll(resp.Body)
//     log.Println(string(b), err)
//     resp.Body.Close()
//     surfer.DestroyJsFiles()
// }
package surfer

import (
	"net/http"
	"sync"
)

var (
	surf          Surfer
	phantom       Surfer
	once_surf     sync.Once
	once_phantom  sync.Once
	tempJsDir     = "./tmp"
	phantomjsFile = "./phantomjs"
)

// Download 实现surfer下载器接口
func Download(req *Request) (resp *http.Response, err error) {
	switch req.DownloaderID {
	case SurfID:
		once_surf.Do(func() { surf = New() })
		resp, err = surf.Download(req)
	case PhomtomJsID:
		once_phantom.Do(func() { phantom = NewPhantom(phantomjsFile, tempJsDir) })
		resp, err = phantom.Download(req)
	}
	return
}

// 指定phantomjs可执行文件的位置
func SetPhantomJsFilePath(filePath string) {
	phantomjsFile = filePath
}

// DestroyJsFiles 销毁Phantomjs的js临时文件
func DestroyJsFiles() {
	if pt, ok := phantom.(*Phantom); ok {
		pt.DestroyJsFiles()
	}
}

// Surfer represents an core of HTTP web browser for crawler.
type Surfer interface {
	// GET @param url string, header http.Header, cookies []*http.Cookie
	// HEAD @param url string, header http.Header, cookies []*http.Cookie
	// POST PostForm @param url, referer string, values url.Values, header http.Header, cookies []*http.Cookie
	// POST-M PostMultipart @param url, referer string, values url.Values, header http.Header, cookies []*http.Cookie
	Download(*Request) (resp *http.Response, err error)
}
<commit_msg>Add default cookieJar to global default Surfer<commit_after>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Package surfer is a high level concurrency http client.
//
// It has `surf` and` phantom` download engines, highly simulated browser behavior, the function of analog login and so on.
//
// Features:
// - Both surf and phantomjs engines are supported
// - Support random User-Agent
// - Support cache cookie
// - Support http/https
//
// Usage:
// package main
//
// import (
//     "github.com/henrylee2cn/surfer"
//     "io/ioutil"
//     "log"
// )
//
// func main() {
//     // Use surf engine
//     resp, err := surfer.Download(&surfer.Request{
//         Url: "http://github.com/henrylee2cn/surfer",
//     })
//     if err != nil {
//         log.Fatal(err)
//     }
//     b, err := ioutil.ReadAll(resp.Body)
//     log.Println(string(b), err)
//
//     // Use phantomjs engine
//     resp, err = surfer.Download(&surfer.Request{
//         Url:          "http://github.com/henrylee2cn",
//         DownloaderID: 1,
//     })
//     if err != nil {
//         log.Fatal(err)
//     }
//     b, err = ioutil.ReadAll(resp.Body)
//     log.Println(string(b), err)
//     resp.Body.Close()
//     surfer.DestroyJsFiles()
// }
package surfer

import (
	"net/http"
	"net/http/cookiejar"
	"sync"
)

var (
	surf          Surfer
	phantom       Surfer
	once_surf     sync.Once
	once_phantom  sync.Once
	tempJsDir     = "./tmp"
	phantomjsFile = "./phantomjs"
	cookieJar, _  = cookiejar.New(nil)
)

// Download 实现surfer下载器接口
func Download(req *Request) (resp *http.Response, err error) {
	switch req.DownloaderID {
	case SurfID:
		once_surf.Do(func() { surf = New(cookieJar) })
		resp, err = surf.Download(req)
	case PhomtomJsID:
		once_phantom.Do(func() { phantom = NewPhantom(phantomjsFile, tempJsDir, cookieJar) })
		resp, err = phantom.Download(req)
	}
	return
}

// 指定phantomjs可执行文件的位置
func SetPhantomJsFilePath(filePath string) {
	phantomjsFile = filePath
}

// DestroyJsFiles 销毁Phantomjs的js临时文件
func DestroyJsFiles() {
	if pt, ok := phantom.(*Phantom); ok {
		pt.DestroyJsFiles()
	}
}

// Surfer represents an core of HTTP web browser for crawler.
type Surfer interface {
	// GET @param url string, header http.Header, cookies []*http.Cookie
	// HEAD @param url string, header http.Header, cookies []*http.Cookie
	// POST PostForm @param url, referer string, values url.Values, header http.Header, cookies []*http.Cookie
	// POST-M PostMultipart @param url, referer string, values url.Values, header http.Header, cookies []*http.Cookie
	Download(*Request) (resp *http.Response, err error)
}
<|endoftext|>


<commit_before>package brain_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	brainMethods "github.com/BytemarkHosting/bytemark-client/lib/requests/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/testutil"
	"github.com/BytemarkHosting/bytemark-client/lib/testutil/assert"
	"github.com/BytemarkHosting/bytemark-client/lib/util"
)

func TestUpdateMigration(t *testing.T) {
	tests := []struct {
		id            int
		modifications brain.MigrationJobModification
		// expected      map[string]interface{}
		shouldErr bool
	}{
		{
			id: 1,
			modifications: brain.MigrationJobModification{
				Cancel: brain.MigrationJobLocations{
					Discs: []util.NumberOrString{"disc.1234"},
					Pools: []util.NumberOrString{"tail1-sata4"},
					Tails: []util.NumberOrString{"tail2"},
				},
				Options: brain.MigrationJobOptions{
					Priority: 10,
				},
			},
		}, //end of the test
	}
	for i, test := range tests {
		testName := testutil.Name(i)
		rts := testutil.RequestTestSpec{
			Method:   "PUT",
			Endpoint: lib.BrainEndpoint,
			URL:      fmt.Sprintf("/admin/migration_jobs/%d", test.id),
			AssertRequest: assert.Body(func(t *testing.T, testName string, body string) {
				var req brain.MigrationJobModification
				err := json.Unmarshal([]byte(body), &req)
				if err != nil {
					t.Fatalf("failed to unmarshal request body: %v", err)
				}
				assert.Equal(t, testName, req, test.modifications)
			}),
		}
		rts.Run(t, testName, true, func(client lib.Client) {
			err := brainMethods.EditMigrationJob(client, test.id, test.modifications)
			if test.shouldErr {
				assert.NotEqual(t, testName, nil, err)
			} else {
				assert.Equal(t, testName, nil, err)
			}

		})
	}

}

func TestCancelMigrationJob(t *testing.T) {
	tests := []struct {
		id        int
		expected  map[string]interface{}
		shouldErr bool
	}{
		{
			id: 1,
			expected: map[string]interface{}{
				"cancel": map[string]interface{}{
					"all": true,
				},
			},
		},
	}
	for i, test := range tests {
		testName := testutil.Name(i)
		rts := testutil.RequestTestSpec{
			Method:        "PUT",
			Endpoint:      lib.BrainEndpoint,
			URL:           fmt.Sprintf("/admin/migration_jobs/%d", test.id),
			AssertRequest: assert.BodyUnmarshalEqual(test.expected),
		}
		rts.Run(t, testName, true, func(client lib.Client) {
			err := brainMethods.CancelMigrationJob(client, test.id)
			if test.shouldErr {
				assert.NotEqual(t, testName, nil, err)
			} else {
				assert.Equal(t, testName, nil, err)
			}
		})
	}
}
<commit_msg>remove some comments<commit_after>package brain_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	brainMethods "github.com/BytemarkHosting/bytemark-client/lib/requests/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/testutil"
	"github.com/BytemarkHosting/bytemark-client/lib/testutil/assert"
	"github.com/BytemarkHosting/bytemark-client/lib/util"
)

func TestUpdateMigration(t *testing.T) {
	tests := []struct {
		id            int
		modifications brain.MigrationJobModification
		shouldErr     bool
	}{
		{
			id: 1,
			modifications: brain.MigrationJobModification{
				Cancel: brain.MigrationJobLocations{
					Discs: []util.NumberOrString{"disc.1234"},
					Pools: []util.NumberOrString{"tail1-sata4"},
					Tails: []util.NumberOrString{"tail2"},
				},
				Options: brain.MigrationJobOptions{
					Priority: 10,
				},
			},
		},
	}
	for i, test := range tests {
		testName := testutil.Name(i)
		rts := testutil.RequestTestSpec{
			Method:   "PUT",
			Endpoint: lib.BrainEndpoint,
			URL:      fmt.Sprintf("/admin/migration_jobs/%d", test.id),
			AssertRequest: assert.Body(func(t *testing.T, testName string, body string) {
				var req brain.MigrationJobModification
				err := json.Unmarshal([]byte(body), &req)
				if err != nil {
					t.Fatalf("failed to unmarshal request body: %v", err)
				}
				assert.Equal(t, testName, req, test.modifications)
			}),
		}
		rts.Run(t, testName, true, func(client lib.Client) {
			err := brainMethods.EditMigrationJob(client, test.id, test.modifications)
			if test.shouldErr {
				assert.NotEqual(t, testName, nil, err)
			} else {
				assert.Equal(t, testName, nil, err)
			}

		})
	}

}

func TestCancelMigrationJob(t *testing.T) {
	tests := []struct {
		id        int
		expected  map[string]interface{}
		shouldErr bool
	}{
		{
			id: 1,
			expected: map[string]interface{}{
				"cancel": map[string]interface{}{
					"all": true,
				},
			},
		},
	}
	for i, test := range tests {
		testName := testutil.Name(i)
		rts := testutil.RequestTestSpec{
			Method:        "PUT",
			Endpoint:      lib.BrainEndpoint,
			URL:           fmt.Sprintf("/admin/migration_jobs/%d", test.id),
			AssertRequest: assert.BodyUnmarshalEqual(test.expected),
		}
		rts.Run(t, testName, true, func(client lib.Client) {
			err := brainMethods.CancelMigrationJob(client, test.id)
			if test.shouldErr {
				assert.NotEqual(t, testName, nil, err)
			} else {
				assert.Equal(t, testName, nil, err)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Package vlog add leveled log on std log(golang.org/pkg/log/)
// It implements most std log functions(except logger), variables
// and add provides V-style logging controlled by the -v flag or SetLogLevel()
// If flag.Parse be called before any logging, -v flag(default 0) use automaticlly.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Basic examples:
//  vlog.SetLogLevel(3)
//  vlog.GetLogLevel()
//
//	vlog.Println("Prepare to repel boarders")
//
//	vlog.Fatalf("Initialization failed: %s", err)
//
// See the documentation for the V function for an explanation of these examples:
//
//	if vlog.V(2) {
//		vlog.Print("Starting transaction...")
//	}
//
//	vlog.V(2).Println("Processed", nItems, "elements")
package vlog

import (
	"flag"
	"io"
	"log"
)

// These flags define which text to prefix to each log entry generated by the Logger.
const (
	// Bits or'ed together to control what's printed.
	// There is no control over the order they appear (the order listed
	// here) or the format they present (as described in the comments).
	// The prefix is followed by a colon only when Llongfile or Lshortfile
	// is specified.
	// For example, flags Ldate | Ltime (or LstdFlags) produce,
	//	2009/01/23 01:23:23 message
	// while flags Ldate | Ltime | Lmicroseconds | Llongfile produce,
	//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate         = log.Ldate             // the date in the local time zone: 2009/01/23
	Ltime         = log.Ltime             // the time in the local time zone: 01:23:23
	Lmicroseconds = log.Lmicroseconds     // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile     = log.Llongfile         // full file name and line number: /a/b/c/d.go:23
	Lshortfile    = log.Lshortfile        // final file name element and line number: d.go:23. overrides Llongfile
	LUTC          = log.LUTC              // if Ldate or Ltime is set, use UTC rather than the local time zone
	LstdFlags     = log.Ldate | log.Ltime // initial values for the standard logger
)

var level uint

// Verbose is a boolean type that implements Print like function.
// See the documentation of V for more information.
type Verbose bool

func init() {
	flag.UintVar(&level, "v", 0, "log level for V logs")
}

// Set log level, just use at parameter initialize zone.
func SetLogLevel(v uint) {
	level = v
}

// Get log level, just use at parameter initialize zone.
func GetLogLevel() uint {
	return level
}

func (vb Verbose) Printf(format string, v ...interface{}) {
	if vb {
		log.Printf(format, v)
	}
}

func (vb Verbose) Println(v ...interface{}) {
	if vb {
		log.Println(v)
	}
}

func (vb Verbose) Print(v ...interface{}) {
	if vb {
		log.Print(v)
	}
}

// Whether an individual call to V generates a log record depends on the setting of level.
func V(v uint) Verbose {
	if v <= level {
		return true
	}
	return false
}

// SetOutput sets the output destination for the standard logger.
func SetOutput(w io.Writer) {
	log.SetOutput(w)
}

// Flags returns the output flags for the standard logger.
func Flags() int {
	return log.Flags()
}

// SetFlags sets the output flags for the standard logger.
func SetFlags(flag int) {
	log.SetFlags(flag)
}

// Prefix returns the output prefix for the standard logger.
func Prefix() string {
	return log.Prefix()
}

// SetPrefix sets the output prefix for the standard logger.
func SetPrefix(prefix string) {
	log.SetPrefix(prefix)
}

// These functions write to the standard logger.

// Print calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Print.
func Print(v ...interface{}) {
	log.Print(v...)
}

// Printf calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Printf.
func Printf(format string, v ...interface{}) {
	log.Printf(format, v...)
}

// Println calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Println.
func Println(v ...interface{}) {
	log.Println(v...)
}

// Fatal is equivalent to Print() followed by a call to os.Exit(1).
func Fatal(v ...interface{}) {
	log.Fatal(v...)
}

// Fatalf is equivalent to Printf() followed by a call to os.Exit(1).
func Fatalf(format string, v ...interface{}) {
	log.Fatalf(format, v...)
}

// Fatalln is equivalent to Println() followed by a call to os.Exit(1).
func Fatalln(v ...interface{}) {
	log.Fatalln(v...)
}

// Panic is equivalent to Print() followed by a call to panic().
func Panic(v ...interface{}) {
	log.Panic(v...)
}

// Panicf is equivalent to Printf() followed by a call to panic().
func Panicf(format string, v ...interface{}) {
	log.Panicf(format, v...)
}

// Panicln is equivalent to Println() followed by a call to panic().
func Panicln(v ...interface{}) {
	log.Panicln(v...)
}

// Output writes the output for a logging event.  The string s contains
// the text to print after the prefix specified by the flags of the
// Logger.  A newline is appended if the last character of s is not
// already a newline.  Calldepth is the count of the number of
// frames to skip when computing the file name and line number
// if Llongfile or Lshortfile is set; a value of 1 will print the details
// for the caller of Output.
func Output(calldepth int, s string) error {
	return log.Output(calldepth, s)
}
<commit_msg>available version<commit_after>// Package vlog add leveled log on std log(golang.org/pkg/log/)
// It implements most std log functions(except logger), variables
// and add provides V-style logging controlled by the -v flag or SetLogLevel()
// If flag.Parse be called before any logging, -v flag(default 0) use automaticlly.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Basic examples:
//  vlog.SetLogLevel(3)
//  vlog.GetLogLevel()
//
//	vlog.Println("Prepare to repel boarders")
//
//	vlog.Fatalf("Initialization failed: %s", err)
//
// See the documentation for the V function for an explanation of these examples:
//
//	if vlog.V(2) {
//		vlog.Print("Starting transaction...")
//	}
//
//	vlog.V(2).Println("Processed", nItems, "elements")
package vlog

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
)

// These flags define which text to prefix to each log entry generated by the Logger.
const (
	// Bits or'ed together to control what's printed.
	// There is no control over the order they appear (the order listed
	// here) or the format they present (as described in the comments).
	// The prefix is followed by a colon only when Llongfile or Lshortfile
	// is specified.
	// For example, flags Ldate | Ltime (or LstdFlags) produce,
	//	2009/01/23 01:23:23 message
	// while flags Ldate | Ltime | Lmicroseconds | Llongfile produce,
	//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate         = log.Ldate             // the date in the local time zone: 2009/01/23
	Ltime         = log.Ltime             // the time in the local time zone: 01:23:23
	Lmicroseconds = log.Lmicroseconds     // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile     = log.Llongfile         // full file name and line number: /a/b/c/d.go:23
	Lshortfile    = log.Lshortfile        // final file name element and line number: d.go:23. overrides Llongfile
	LUTC          = log.LUTC              // if Ldate or Ltime is set, use UTC rather than the local time zone
	LstdFlags     = log.Ldate | log.Ltime // initial values for the standard logger
)

var (
	level uint

	std = log.New(os.Stderr, "", LstdFlags)
)

// Verbose is a boolean type that implements Print like function.
// See the documentation of V for more information.
type Verbose bool

func init() {
	flag.UintVar(&level, "v", 0, "log level for V logs")
}

// Set log level, just use at parameter initialize zone.
func SetLogLevel(v uint) {
	level = v
}

// Get log level, just use at parameter initialize zone.
func GetLogLevel() uint {
	return level
}

// Whether an individual call to V generates a log record depends on the setting of level.
func V(v uint) Verbose {
	if v <= level {
		return true
	}
	return false
}

func (vb Verbose) Printf(format string, v ...interface{}) {
	if vb {
		std.Output(2, fmt.Sprintf(format, v...))
	}
}

func (vb Verbose) Println(v ...interface{}) {
	if vb {
		std.Output(2, fmt.Sprintln(v...))
	}
}

func (vb Verbose) Print(v ...interface{}) {
	if vb {
		std.Output(2, fmt.Sprint(v...))
	}
}

// SetOutput sets the output destination for the standard logger.
func SetOutput(w io.Writer) {
	std.SetOutput(w)
}

// Flags returns the output flags for the standard logger.
func Flags() int {
	return std.Flags()
}

// SetFlags sets the output flags for the standard logger.
func SetFlags(flag int) {
	std.SetFlags(flag)
}

// Prefix returns the output prefix for the standard logger.
func Prefix() string {
	return std.Prefix()
}

// SetPrefix sets the output prefix for the standard logger.
func SetPrefix(prefix string) {
	std.SetPrefix(prefix)
}

// These functions write to the standard logger.

// Print calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Print.
func Print(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
}

// Printf calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Printf.
func Printf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
}

// Println calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Println.
func Println(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
}

// Fatal is equivalent to Print() followed by a call to os.Exit(1).
func Fatal(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
	os.Exit(1)
}

// Fatalf is equivalent to Printf() followed by a call to os.Exit(1).
func Fatalf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

// Fatalln is equivalent to Println() followed by a call to os.Exit(1).
func Fatalln(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
	os.Exit(1)
}

// Panic is equivalent to Print() followed by a call to panic().
func Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	std.Output(2, s)
	panic(s)
}

// Panicf is equivalent to Printf() followed by a call to panic().
func Panicf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	std.Output(2, s)
	panic(s)
}

// Panicln is equivalent to Println() followed by a call to panic().
func Panicln(v ...interface{}) {
	s := fmt.Sprintln(v...)
	std.Output(2, s)
	panic(s)
}

// Output writes the output for a logging event.  The string s contains
// the text to print after the prefix specified by the flags of the
// Logger.  A newline is appended if the last character of s is not
// already a newline.  Calldepth is the count of the number of
// frames to skip when computing the file name and line number
// if Llongfile or Lshortfile is set; a value of 1 will print the details
// for the caller of Output.
func Output(calldepth int, s string) error {
	return std.Output(calldepth+1, s) // +1 for this frame.
}
<|endoftext|>


<commit_before>package captain // import "github.com/harbur/captain/captain"

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
)

// Options that are passed by CLI are mapped here for consumption
type Options struct {
	debug     bool
	force     bool
	namespace string
	config    string
	images    []string
}

var options Options

func handleCmd() {

	var cmdBuild = &cobra.Command{
		Use:   "build [image]",
		Short: "Builds the docker image(s) of your repository",
		Long:  `It will build the docker image(s) described on captain.yml in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := NewConfig(options, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			Build(config)
		},
	}

	var cmdTest = &cobra.Command{
		Use:   "test",
		Short: "Runs the tests",
		Long:  `It will execute the commands described on test section in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := NewConfig(options, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			// Build everything before testing
			Build(config)
			Test(config)
		},
	}

	var cmdPush = &cobra.Command{
		Use:   "push",
		Short: "Pushes the images to remote registry",
		Long:  `It will push the generated images to the remote registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := NewConfig(options, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			// Build everything before pushing
			Build(config)
			Push(config)
		},
	}

	var cmdVersion = &cobra.Command{
		Use:   "version",
		Short: "Display version",
		Long:  `Displays the version of Captain.`,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("v0.1.0")
		},
	}

	var captainCmd = &cobra.Command{
		Use:   "captain",
		Short: "captain - build tool for Docker focused on CI/CD",
		Long: `
Captain, the CLI build tool for Docker made for Continuous Integration / Continuous Delivery.

It works by reading captain.yaml file which describes how to build, test, push and release the docker image(s) of your repository.`,
	}

	captainCmd.PersistentFlags().BoolVarP(&options.debug, "debug", "D", false, "Enable debug mode")
	captainCmd.PersistentFlags().StringVarP(&options.namespace, "namespace", "N", getNamespace(), "Set default image namespace")
	cmdBuild.Flags().BoolVarP(&options.force, "force", "f", false, "Force build even if image is already built")
	captainCmd.AddCommand(cmdBuild, cmdTest, cmdPush, cmdVersion)
	captainCmd.Execute()
}

func getNamespace() string {
	return os.Getenv("USER")
}
<commit_msg>Preparing release v0.1.1<commit_after>package captain // import "github.com/harbur/captain/captain"

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
)

// Options that are passed by CLI are mapped here for consumption
type Options struct {
	debug     bool
	force     bool
	namespace string
	config    string
	images    []string
}

var options Options

func handleCmd() {

	var cmdBuild = &cobra.Command{
		Use:   "build [image]",
		Short: "Builds the docker image(s) of your repository",
		Long:  `It will build the docker image(s) described on captain.yml in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := NewConfig(options, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			Build(config)
		},
	}

	var cmdTest = &cobra.Command{
		Use:   "test",
		Short: "Runs the tests",
		Long:  `It will execute the commands described on test section in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := NewConfig(options, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			// Build everything before testing
			Build(config)
			Test(config)
		},
	}

	var cmdPush = &cobra.Command{
		Use:   "push",
		Short: "Pushes the images to remote registry",
		Long:  `It will push the generated images to the remote registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := NewConfig(options, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			// Build everything before pushing
			Build(config)
			Push(config)
		},
	}

	var cmdVersion = &cobra.Command{
		Use:   "version",
		Short: "Display version",
		Long:  `Displays the version of Captain.`,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("v0.1.1")
		},
	}

	var captainCmd = &cobra.Command{
		Use:   "captain",
		Short: "captain - build tool for Docker focused on CI/CD",
		Long: `
Captain, the CLI build tool for Docker made for Continuous Integration / Continuous Delivery.

It works by reading captain.yaml file which describes how to build, test, push and release the docker image(s) of your repository.`,
	}

	captainCmd.PersistentFlags().BoolVarP(&options.debug, "debug", "D", false, "Enable debug mode")
	captainCmd.PersistentFlags().StringVarP(&options.namespace, "namespace", "N", getNamespace(), "Set default image namespace")
	cmdBuild.Flags().BoolVarP(&options.force, "force", "f", false, "Force build even if image is already built")
	captainCmd.AddCommand(cmdBuild, cmdTest, cmdPush, cmdVersion)
	captainCmd.Execute()
}

func getNamespace() string {
	return os.Getenv("USER")
}
<|endoftext|>


<commit_before>package algoholic

// Merge Sort, O(n lg n) worst-case. Very beautiful.
func MergeSort(ns []int) []int {
	// Base case - an empty or length 1 slice is trivially sorted.
	if len(ns) < 2 {
		// We need not allocate memory here as the at most 1 element will only be referenced
		// once.
		return ns
	}

	half := len(ns) / 2

	// The wonder of merge sort - we sort each half of the slice using... merge sort :-)
	// Where is your God now?
	ns1 := MergeSort(ns[:half])
	ns2 := MergeSort(ns[half:])

	// We now have 2 separately sorted slices, merge them into one.
	return Merge(ns1, ns2)
}

// Merge, O(n), merges two sorted slices into one.
func Merge(ns1, ns2 []int) []int {
	length := len(ns1) + len(ns2)
	ret := make([]int, length)

	i, j := 0, 0

	// We iterate through each element of the returned slice, placing elements of each of the
	// input slices in their appropriate places.
	//
	// Loop Invariant: ret[:k] consists of ns1[:i] and ns2[:j] in sorted order.
	for k := 0; k < length; k++ {
		switch {
		case j >= len(ns2) || ns1[i] <= ns2[j]:
			ret[k] = ns1[i]
			i++
		case i >= len(ns1) || ns2[j] <= ns1[i]:
			ret[k] = ns2[j]
			j++
		}
	}

	// When the loop is complete, i == len(ns1), j == len(ns2). Therefore our loop invariant
	// determines that ret consists of ns1 and ns2 in sorted order, which matches the purpose
	// of the function.
	return ret
}
<commit_msg>If we exceed slice length, just copy data.<commit_after>package algoholic

// Merge Sort, O(n lg n) worst-case. Very beautiful.
func MergeSort(ns []int) []int {
	// Base case - an empty or length 1 slice is trivially sorted.
	if len(ns) < 2 {
		// We need not allocate memory here as the at most 1 element will only be referenced
		// once.
		return ns
	}

	half := len(ns) / 2

	// The wonder of merge sort - we sort each half of the slice using... merge sort :-)
	// Where is your God now?
	ns1 := MergeSort(ns[:half])
	ns2 := MergeSort(ns[half:])

	// We now have 2 separately sorted slices, merge them into one.
	return Merge(ns1, ns2)
}

// Merge, O(n), merges two sorted slices into one.
func Merge(ns1, ns2 []int) []int {
	length := len(ns1) + len(ns2)
	ret := make([]int, length)

	i, j := 0, 0

	// We iterate through each element of the returned slice, placing elements of each of the
	// input slices in their appropriate places.
	//
	// Loop Invariant: ret[:k] consists of ns1[:i] and ns2[:j] in sorted order.
	for k := 0; k < length; k++ {
		if j >= len(ns2) {
			copy(ret[k:], ns1[i:])
			break
		}
		if i >= len(ns1) {
			copy(ret[k:], ns2[j:])
			break
		}

		switch {
		case ns1[i] <= ns2[j]:
			ret[k] = ns1[i]
			i++
		case ns2[j] <= ns1[i]:
			ret[k] = ns2[j]
			j++
		default:
			panic("Impossible.")
		}
	}

	// When the loop is complete, i == len(ns1), j == len(ns2). Therefore our loop invariant
	// determines that ret consists of ns1 and ns2 in sorted order, which matches the purpose
	// of the function.
	return ret
}
<|endoftext|>


<commit_before>package gedcom

import (
	"io"
	"strings"

	"github.com/MJKWoolnough/errors"
	"github.com/MJKWoolnough/parser"
)

const (
	alpha       = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"
	delim       = " "
	digit       = "0123456789"
	alphanum    = alpha + digit
	otherchar   = "!\"$%&'()*+,-./:;<=>?[\\]^`{|}~" // + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" // ANSEL
	nonAt       = alpha + digit + otherchar + " #"
	terminators = "\r\n"
	anyChar     = alpha + digit + otherchar + "# @"
	levelIgnore = terminators + delim + "	"
	invalidchar = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\xab\xac\xad\x1e\x1f\x7f" + terminators + "@"
)

const (
	tokenLevel parser.TokenType = iota
	tokenXref
	tokenTag
	tokenPointer
	tokenLine
	tokenEndLine
)

type tokeniser struct {
	parser.Parser
	options
}

func newTokeniser(r io.Reader, o options) *tokeniser {
	t := &tokeniser{
		Parser:  parser.New(parser.NewReaderTokeniser(r)),
		options: o,
	}
	t.TokeniserState(t.level)
	return t
}

func (t *tokeniser) level(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	p.AcceptRun(levelIgnore)
	p.Get()
	if p.Peek() == -1 {
		return p.Done()
	}
	if !p.Accept(digit) {
		p.Err = ErrInvalidLevel
		return p.Error()
	}
	p.AcceptRun(digit)
	if !p.Accept(delim) {
		p.Err = ErrMissingDelim
		return p.Error()
	}
	return parser.Token{
		tokenLevel,
		strings.TrimSpace(p.Get()),
	}, t.optionalXrefID
}

func (t *tokeniser) optionalXrefID(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	if p.Peek() == '@' {
		return t.xrefID(p)
	}
	return t.tag(p)
}

func (t *tokeniser) readPointer(p *parser.Tokeniser) (string, error) {
	if !p.Accept(alphanum) {
		return "", ErrInvalidPointer
	}
	p.AcceptRun(nonAt)
	if !p.Accept("@") {
		return "", ErrInvalidPointer
	}
	pointer := p.Get()
	return pointer[1 : len(pointer)-1], nil
}

func (t *tokeniser) xrefID(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	p.Accept("@")
	pointer, err := t.readPointer(p)
	if err != nil {
		p.Err = err
		return p.Error()
	}
	if !p.Accept(delim) {
		p.Err = ErrMissingDelim
		return p.Error()
	}
	p.Get()
	return parser.Token{
		tokenXref,
		strings.Trim(pointer, "@"),
	}, t.tag
}

func (t *tokeniser) tag(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	if !p.Accept(alphanum) {
		p.Err = ErrInvalidTag
		return p.Error()
	}
	p.AcceptRun(alphanum)
	tag := p.Get()
	if p.Accept(delim) {
		p.Get()
		return parser.Token{
			tokenTag,
			tag,
		}, t.lineValue
	}
	next := t.endLine
	if p.Peek() == -1 {
		next = (*parser.Tokeniser).Done
	} else {
		if !p.Accept(terminators) {
			p.Err = ErrInvalidTag
			return p.Error()
		}
		p.AcceptRun(terminators)
		p.Get()
	}
	return parser.Token{
		tokenTag,
		tag,
	}, next
}

func (t *tokeniser) endLine(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	return parser.Token{
		tokenEndLine,
		"",
	}, t.level
}

func (t *tokeniser) lineValue(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	if p.Peek() == '@' {
		p.Accept("@")
		if p.Peek() != '@' {
			pointer, err := t.readPointer(p)
			if err != nil {
				if !t.allowInvalidEscape {
					p.Err = err
					return p.Error()
				}
			} else {
				p.AcceptRun(terminators)
				p.Get()
				return parser.Token{
					tokenPointer,
					pointer,
				}, t.level
			}
		}
	}
	next := t.level
	for {
		for {
			if t.allowUnknownCharset && p.Except(invalidchar) {
				p.ExceptRun(invalidchar)
			} else if p.Accept(nonAt) {
				p.AcceptRun(nonAt)
			} else if p.Accept("@") {
				if t.allowInvalidEscape || p.Accept("@") {
					continue
				}
				if !p.Accept("#") {
					p.Err = ErrBadEscape
					return p.Error()
				}
				if t.allowUnknownCharset {
					p.ExceptRun(invalidchar)
				} else {
					p.AcceptRun(nonAt)
				}
				if !p.Accept("@") {
					p.Err = ErrBadEscape
					return p.Error()
				}
			} else {
				break
			}
		}
		pe := p.Peek()
		if pe == -1 {
			next = (*parser.Tokeniser).Done
			break
		}
		if strings.ContainsRune(terminators, pe) {
			p.AcceptRun(terminators)
			if !t.allowTerminatorsInValue {
				break
			}
			pe = p.Peek()
			if pe == -1 {
				next = (*parser.Tokeniser).Done
				break
			}
			if strings.ContainsRune(digit, pe) {
				break
			}
		} else {
			if !t.allowInvalidChars {
				p.Err = ErrBadChar
				return p.Error()
			}
			p.Except("")
		}
	}
	return parser.Token{
		tokenLine,
		strings.TrimSpace(p.Get()),
	}, next
}

// Errors
var (
	ErrInvalidLevel   errors.Error = "invalid level num"
	ErrMissingDelim   errors.Error = "missing delminitator"
	ErrInvalidPointer errors.Error = "invalid pointer string"
	ErrInvalidTag     errors.Error = "invalid tag"
	ErrBadEscape      errors.Error = "bad escape sequence"
	ErrBadChar        errors.Error = "bad character"
)
<commit_msg>go vet'd<commit_after>package gedcom

import (
	"io"
	"strings"

	"github.com/MJKWoolnough/errors"
	"github.com/MJKWoolnough/parser"
)

const (
	alpha       = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"
	delim       = " "
	digit       = "0123456789"
	alphanum    = alpha + digit
	otherchar   = "!\"$%&'()*+,-./:;<=>?[\\]^`{|}~" // + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" // ANSEL
	nonAt       = alpha + digit + otherchar + " #"
	terminators = "\r\n"
	anyChar     = alpha + digit + otherchar + "# @"
	levelIgnore = terminators + delim + "	"
	invalidchar = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\xab\xac\xad\x1e\x1f\x7f" + terminators + "@"
)

const (
	tokenLevel parser.TokenType = iota
	tokenXref
	tokenTag
	tokenPointer
	tokenLine
	tokenEndLine
)

type tokeniser struct {
	parser.Parser
	options
}

func newTokeniser(r io.Reader, o options) *tokeniser {
	t := &tokeniser{
		Parser:  parser.New(parser.NewReaderTokeniser(r)),
		options: o,
	}
	t.TokeniserState(t.level)
	return t
}

func (t *tokeniser) level(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	p.AcceptRun(levelIgnore)
	p.Get()
	if p.Peek() == -1 {
		return p.Done()
	}
	if !p.Accept(digit) {
		p.Err = ErrInvalidLevel
		return p.Error()
	}
	p.AcceptRun(digit)
	if !p.Accept(delim) {
		p.Err = ErrMissingDelim
		return p.Error()
	}
	return parser.Token{
		Type: tokenLevel,
		Data: strings.TrimSpace(p.Get()),
	}, t.optionalXrefID
}

func (t *tokeniser) optionalXrefID(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	if p.Peek() == '@' {
		return t.xrefID(p)
	}
	return t.tag(p)
}

func (t *tokeniser) readPointer(p *parser.Tokeniser) (string, error) {
	if !p.Accept(alphanum) {
		return "", ErrInvalidPointer
	}
	p.AcceptRun(nonAt)
	if !p.Accept("@") {
		return "", ErrInvalidPointer
	}
	pointer := p.Get()
	return pointer[1 : len(pointer)-1], nil
}

func (t *tokeniser) xrefID(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	p.Accept("@")
	pointer, err := t.readPointer(p)
	if err != nil {
		p.Err = err
		return p.Error()
	}
	if !p.Accept(delim) {
		p.Err = ErrMissingDelim
		return p.Error()
	}
	p.Get()
	return parser.Token{
		Type: tokenXref,
		Data: strings.Trim(pointer, "@"),
	}, t.tag
}

func (t *tokeniser) tag(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	if !p.Accept(alphanum) {
		p.Err = ErrInvalidTag
		return p.Error()
	}
	p.AcceptRun(alphanum)
	tag := p.Get()
	if p.Accept(delim) {
		p.Get()
		return parser.Token{
			Type: tokenTag,
			Data: tag,
		}, t.lineValue
	}
	next := t.endLine
	if p.Peek() == -1 {
		next = (*parser.Tokeniser).Done
	} else {
		if !p.Accept(terminators) {
			p.Err = ErrInvalidTag
			return p.Error()
		}
		p.AcceptRun(terminators)
		p.Get()
	}
	return parser.Token{
		Type: tokenTag,
		Data: tag,
	}, next
}

func (t *tokeniser) endLine(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	return parser.Token{
		Type: tokenEndLine,
		Data: "",
	}, t.level
}

func (t *tokeniser) lineValue(p *parser.Tokeniser) (parser.Token, parser.TokenFunc) {
	if p.Peek() == '@' {
		p.Accept("@")
		if p.Peek() != '@' {
			pointer, err := t.readPointer(p)
			if err != nil {
				if !t.allowInvalidEscape {
					p.Err = err
					return p.Error()
				}
			} else {
				p.AcceptRun(terminators)
				p.Get()
				return parser.Token{
					Type: tokenPointer,
					Data: pointer,
				}, t.level
			}
		}
	}
	next := t.level
	for {
		for {
			if t.allowUnknownCharset && p.Except(invalidchar) {
				p.ExceptRun(invalidchar)
			} else if p.Accept(nonAt) {
				p.AcceptRun(nonAt)
			} else if p.Accept("@") {
				if t.allowInvalidEscape || p.Accept("@") {
					continue
				}
				if !p.Accept("#") {
					p.Err = ErrBadEscape
					return p.Error()
				}
				if t.allowUnknownCharset {
					p.ExceptRun(invalidchar)
				} else {
					p.AcceptRun(nonAt)
				}
				if !p.Accept("@") {
					p.Err = ErrBadEscape
					return p.Error()
				}
			} else {
				break
			}
		}
		pe := p.Peek()
		if pe == -1 {
			next = (*parser.Tokeniser).Done
			break
		}
		if strings.ContainsRune(terminators, pe) {
			p.AcceptRun(terminators)
			if !t.allowTerminatorsInValue {
				break
			}
			pe = p.Peek()
			if pe == -1 {
				next = (*parser.Tokeniser).Done
				break
			}
			if strings.ContainsRune(digit, pe) {
				break
			}
		} else {
			if !t.allowInvalidChars {
				p.Err = ErrBadChar
				return p.Error()
			}
			p.Except("")
		}
	}
	return parser.Token{
		Type: tokenLine,
		Data: strings.TrimSpace(p.Get()),
	}, next
}

// Errors
var (
	ErrInvalidLevel   errors.Error = "invalid level num"
	ErrMissingDelim   errors.Error = "missing delminitator"
	ErrInvalidPointer errors.Error = "invalid pointer string"
	ErrInvalidTag     errors.Error = "invalid tag"
	ErrBadEscape      errors.Error = "bad escape sequence"
	ErrBadChar        errors.Error = "bad character"
)
<|endoftext|>