<commit_before>package bitmessage

// This file implements the main engine for this BitMessage node.

import (
	"fmt"
	"io"
	"log"
	"net"
	"time"

	"github.com/pmylund/go-bloom"
	"sync"
)

type Node struct {
	// All members can only be accessed by the main server routine inside
	// Run().
	cfg *Config
	// Stats. All access must be synchronized because it's often used by other
	// goroutines (UI).
	stats stats
	resp  responses
	// connectedNodes are all nodes we have a connection established to, for
	// each stream.
	connectedNodes streamNodes
	// Bloomfilter of unreachable IPs.
	// TODO: Rotate the filters and expire them.
	// TODO: Save it on disk.
	unreachableNodes *bloom.Filter
	// connectedNodes are all nodes we know of for each stream number in
	// addition to the connectedNodes.
	knownNodes streamNodes
	objects    objectsInventory
}

type responses struct {
	addrsChan   chan []extendedNetworkAddress
	addNodeChan chan extendedNetworkAddress
	delNodeChan chan extendedNetworkAddress
	invChan     chan objectsInventory
}

func (n *Node) Run() {
	n.connectedNodes = make(streamNodes)
	n.knownNodes = make(streamNodes)
	n.objects = make(objectsInventory)
	n.unreachableNodes = bloom.New(10000, 0.01)

	n.cfg = openConfig(PortNumber)

	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", PortNumber))
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Listening at", listener.Addr())
	n.resp = responses{
		make(chan []extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan objectsInventory),
	}
	go listen(listener.(*net.TCPListener), n.resp)
	n.bootstrap()
	// TODO: do this less frequently.
	// TODO: add signal handler for saving on exit.
	saveTick := time.Tick(time.Second * 1)
	for {
		select {
		case addrs := <-n.resp.addrsChan:
			log.Printf("nodesChan, got: %d nodes", len(addrs))
			// Only connect to stream one for now.

			needExtra := numNodesForMainStream - n.numStreamNodes(streamOne)
			i := 0
			// This is imprecise because I check the count of nodes using a
			// metric that is only updated after the connection is
			// established, not soon after a handshake goroutine is
			// dispatched. That's fine, we'll just a have a few extra too may
			// nodes.
			log.Println("need extra", needExtra)
			for _, addr := range addrs {
				if addr.Stream != streamOne {
					continue
				}
				if _, ok := n.connectedNodes[streamOne][addr.ipPort()]; ok {
					continue
				}
				if n.unreachableNodes.Test(addr.IP[:]) {
					continue
				}
				node := remoteNode{}
				if i <= needExtra {
					log.Println("handshaking with", addr.ipPort())
					// Nodes for which the connection attempt fail won't even
					// make it to n.knownNodes.
					go handshake(addr.ipPort(), node, n.resp)
					i++
				} else {
					n.addKnownNode(int(addr.Stream), addr.ipPort(), node)
				}
			}

		case addr := <-n.resp.addNodeChan:
			node := remoteNode{lastContacted: time.Now()}
			n.addNode(int(addr.Stream), addr.ipPort(), node)
		case addr := <-n.resp.delNodeChan:
			n.delNode(int(addr.Stream), addr.ipPort())
			n.unreachableNodes.Add(addr.IP[:])
			// XXX if connection counter drops below numNodesforMainStream,
			// get a node from knownNodes and promote it.
		case obj := <-n.resp.invChan:
			n.objects.merge(obj)
		case <-saveTick:
			n.cfg.save(n.connectedNodes)
		}
	}
}

type packet struct {
	b     []byte
	raddr net.Addr
}

// Read from TCP , writes slice of byte into channel.
func listen(listener *net.TCPListener, resp responses) {
	for {
		conn, err := listener.AcceptTCP()
		if err != nil {
			log.Fatal("Can't listen to network port:", err)
			return
		}
		go handleConn(conn, resp)
	}
}

type peerState struct {
	established    bool // when 'verack' has been sent and received.
	verackSent     bool
	verackReceived bool
	ipPort         ipPort
}

func handleConn(conn *net.TCPConn, resp responses) {
	defer conn.Close()

	p := &peerState{}
	p.ipPort = ipPort(conn.RemoteAddr().String())
	for {

		conn.SetReadDeadline(time.Now().Add(connectionTimeout))
		command, payload, err := readMessage(conn)
		if err != nil {
			log.Println("handleConn:", err)
			resp.delNodeChan <- p.ipPort.toNetworkAddress()
			return
		}
		log.Printf("got command: %v", command)
		switch command {
		case "version":
			err = handleVersion(conn, p, payload, resp.addNodeChan)
		case "addr":
			err = handleAddr(conn, p, payload, resp.addrsChan)
		case "verack":
			err = handleVerack(conn, p, resp.addNodeChan)
		case "inv":
			err = handleInv(conn, p, payload, resp.invChan)
		default:
			// XXX Used during development.
			log.Fatalf("ignoring unknown command %q", command)
		}
		if err != nil {
			log.Printf("error while processing command %v: %v", command, err)
			resp.delNodeChan <- ipPort(conn.RemoteAddr().String()).toNetworkAddress()
			// Disconnects from node.
			return
		}
	}
}

func handleVersion(conn io.Writer, p *peerState, payload io.Reader, addNode chan extendedNetworkAddress) error {
	if p.established {
		return fmt.Errorf("received a 'version' message from a host we already went through a version exchange. Closing the connection.")
	}
	version, err := parseVersion(payload)
	if err != nil {
		return fmt.Errorf("parseVersion: %v", err)
	}
	if version.Nonce == nonce {
		// Close connection to self.
		// TODO: put on ipPort blacklist.
		return fmt.Errorf("closing loop")
	}
	if version.Version != protocolVersion {
		return fmt.Errorf("protocol version not supported: got %d, wanted %d.Closing the connection", version.Version, protocolVersion)
	}
	if p.verackSent == false {
		writeVerack(conn)
	}
	p.verackSent = true
	if p.verackReceived {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleVerack(conn io.Writer, p *peerState, addNode chan extendedNetworkAddress) error {
	if p.verackReceived {
		return fmt.Errorf("received 'verack' twice from a node. Closing connection")
	}
	p.verackReceived = true
	if p.verackSent {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleAddr(conn io.Writer, p *peerState, payload io.Reader, respNodes chan []extendedNetworkAddress) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	addrs, err := parseAddr(payload)
	if err != nil {
		return fmt.Errorf("parseAddr error: %v. Closing connection", err)
	}
	respNodes <- addrs
	return nil
}

var once sync.Once

func handleInv(conn io.Writer, p *peerState, payload io.Reader, obj chan objectsInventory) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	invs, err := parseInv(payload)
	if err != nil {
		return fmt.Errorf("parseInv error: %v. Closing connection", err)
	}
	objects := make(objectsInventory)
	for _, inv := range invs {
		objects[inv.Hash] = p.ipPort
		// XXX Used during development.
		once.Do(
			func() {
				writeGetData(conn, []inventoryVector{inv})
			})
	}
	obj <- objects
	return nil
}

// Things needing implementation.
//
// - save the config frequently. be more resilient to BitMessage attacks.
// - Preserve the routing table for longer, don't delete nodes immediately
// - after they disconnect.
//
// from PyBitMessage:
//
// broadcastToSendDataQueues((0, 'shutdown', self.HOST))
//
// Zero out the list of already contacted nodes every 30 minutes, to give it
// another chance.
//
// self.receivedgetbiginv = False #Gets set to true once we receive a
// getbiginv message from our peer. An abusive peer might request it too much
// so we use this variable to check whether they have already asked for a big
// inv message.

// #Only one singleListener thread will ever exist. It creates the
// #receiveDataThread and sendDataThread for each incoming connection. Note
// #that it cannot set the stream number because it is not known yet- the
// #other node will have to tell us its stream number in a version message. If
// #we don't care about their stream, we will close the connection (within the
// #recversion function of the recieveData thread)

// #For each stream to which we connect, several outgoingSynSender threads
// #will exist and will collectively create 8 connections with peers.
//
// Random nonce used to detect connections to self.

type stats struct {
	streamConnectionCount map[int]int
}

// ipPort is a string that can be resolved with net.ResolveTCPAddr("tcp",
// ipPort) and the first part can be parsed by net.ParseIP(). It is illegal to
// create an ipPort that doesn't follow these conditions.
type ipPort string

func (ipPort ipPort) toNetworkAddress() extendedNetworkAddress {
	tcpAddr, _ := net.ResolveTCPAddr("tcp", string(ipPort))
	var rawIp [16]byte
	copy(rawIp[:], tcpAddr.IP)
	addr := extendedNetworkAddress{
		Time:   uint32(time.Now().Unix()),
		Stream: streamOne, // This should change after the version exchange.
		NetworkAddress: NetworkAddress{
			Services: ConnectionServiceNodeNetwork, //
			IP:       rawIp,
			Port:     uint16(tcpAddr.Port),
		},
	}
	return addr
}
<commit_msg>Simple bloomfilter to avoid trying to connect o unreachable nodes.<commit_after>package bitmessage

// This file implements the main engine for this BitMessage node.

import (
	"fmt"
	"io"
	"log"
	"net"
	"sync"
	"time"

	"github.com/pmylund/go-bloom"
)

type Node struct {
	// All members can only be accessed by the main server routine inside
	// Run().
	cfg *Config
	// Stats. All access must be synchronized because it's often used by other
	// goroutines (UI).
	stats stats
	resp  responses
	// connectedNodes are all nodes we have a connection established to, for
	// each stream.
	connectedNodes streamNodes
	// Bloomfilter of unreachable IPs.
	// TODO: Rotate the filters and expire them.
	// TODO: Save it on disk.
	unreachableNodes *bloom.Filter
	// connectedNodes are all nodes we know of for each stream number in
	// addition to the connectedNodes.
	knownNodes streamNodes
	objects    objectsInventory
}

type responses struct {
	addrsChan   chan []extendedNetworkAddress
	addNodeChan chan extendedNetworkAddress
	delNodeChan chan extendedNetworkAddress
	invChan     chan objectsInventory
}

func (n *Node) Run() {
	n.connectedNodes = make(streamNodes)
	n.knownNodes = make(streamNodes)
	n.objects = make(objectsInventory)
	n.unreachableNodes = bloom.New(10000, 0.01)

	n.cfg = openConfig(PortNumber)

	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", PortNumber))
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Listening at", listener.Addr())
	n.resp = responses{
		make(chan []extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan objectsInventory),
	}
	go listen(listener.(*net.TCPListener), n.resp)
	n.bootstrap()
	// TODO: do this less frequently.
	// TODO: add signal handler for saving on exit.
	saveTick := time.Tick(time.Second * 1)
	for {
		select {
		case addrs := <-n.resp.addrsChan:
			log.Printf("nodesChan, got: %d nodes", len(addrs))
			// Only connect to stream one for now.

			needExtra := numNodesForMainStream - n.numStreamNodes(streamOne)
			i := 0
			// This is imprecise because I check the count of nodes using a
			// metric that is only updated after the connection is
			// established, not soon after a handshake goroutine is
			// dispatched. That's fine, we'll just a have a few extra too may
			// nodes.
			log.Println("need extra", needExtra)
			for _, addr := range addrs {
				if addr.Stream != streamOne {
					continue
				}
				if _, ok := n.connectedNodes[streamOne][addr.ipPort()]; ok {
					continue
				}
				if n.unreachableNodes.Test(addr.IP[:]) {
					continue
				}
				node := remoteNode{}
				if i <= needExtra {
					log.Println("handshaking with", addr.ipPort())
					// Nodes for which the connection attempt fail won't even
					// make it to n.knownNodes.
					go handshake(addr.ipPort(), node, n.resp)
					i++
				} else {
					n.addKnownNode(int(addr.Stream), addr.ipPort(), node)
				}
			}

		case addr := <-n.resp.addNodeChan:
			node := remoteNode{lastContacted: time.Now()}
			n.addNode(int(addr.Stream), addr.ipPort(), node)
		case addr := <-n.resp.delNodeChan:
			n.delNode(int(addr.Stream), addr.ipPort())
			n.unreachableNodes.Add(addr.IP[:])
			// XXX if connection counter drops below numNodesforMainStream,
			// get a node from knownNodes and promote it.
		case obj := <-n.resp.invChan:
			n.objects.merge(obj)
		case <-saveTick:
			n.cfg.save(n.connectedNodes)
		}
	}
}

type packet struct {
	b     []byte
	raddr net.Addr
}

// Read from TCP , writes slice of byte into channel.
func listen(listener *net.TCPListener, resp responses) {
	for {
		conn, err := listener.AcceptTCP()
		if err != nil {
			log.Fatal("Can't listen to network port:", err)
			return
		}
		go handleConn(conn, resp)
	}
}

type peerState struct {
	established    bool // when 'verack' has been sent and received.
	verackSent     bool
	verackReceived bool
	ipPort         ipPort
}

func handleConn(conn *net.TCPConn, resp responses) {
	defer conn.Close()

	p := &peerState{}
	p.ipPort = ipPort(conn.RemoteAddr().String())
	for {

		conn.SetReadDeadline(time.Now().Add(connectionTimeout))
		command, payload, err := readMessage(conn)
		if err != nil {
			log.Println("handleConn:", err)
			resp.delNodeChan <- p.ipPort.toNetworkAddress()
			return
		}
		log.Printf("got command: %v", command)
		switch command {
		case "version":
			err = handleVersion(conn, p, payload, resp.addNodeChan)
		case "addr":
			err = handleAddr(conn, p, payload, resp.addrsChan)
		case "verack":
			err = handleVerack(conn, p, resp.addNodeChan)
		case "inv":
			err = handleInv(conn, p, payload, resp.invChan)
		default:
			// XXX Used during development.
			log.Fatalf("ignoring unknown command %q", command)
		}
		if err != nil {
			log.Printf("error while processing command %v: %v", command, err)
			resp.delNodeChan <- ipPort(conn.RemoteAddr().String()).toNetworkAddress()
			// Disconnects from node.
			return
		}
	}
}

func handleVersion(conn io.Writer, p *peerState, payload io.Reader, addNode chan extendedNetworkAddress) error {
	if p.established {
		return fmt.Errorf("received a 'version' message from a host we already went through a version exchange. Closing the connection.")
	}
	version, err := parseVersion(payload)
	if err != nil {
		return fmt.Errorf("parseVersion: %v", err)
	}
	if version.Nonce == nonce {
		// Close connection to self.
		// TODO: put on ipPort blacklist.
		return fmt.Errorf("closing loop")
	}
	if version.Version != protocolVersion {
		return fmt.Errorf("protocol version not supported: got %d, wanted %d.Closing the connection", version.Version, protocolVersion)
	}
	if p.verackSent == false {
		writeVerack(conn)
	}
	p.verackSent = true
	if p.verackReceived {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleVerack(conn io.Writer, p *peerState, addNode chan extendedNetworkAddress) error {
	if p.verackReceived {
		return fmt.Errorf("received 'verack' twice from a node. Closing connection")
	}
	p.verackReceived = true
	if p.verackSent {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleAddr(conn io.Writer, p *peerState, payload io.Reader, respNodes chan []extendedNetworkAddress) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	addrs, err := parseAddr(payload)
	if err != nil {
		return fmt.Errorf("parseAddr error: %v. Closing connection", err)
	}
	respNodes <- addrs
	return nil
}

var once sync.Once

func handleInv(conn io.Writer, p *peerState, payload io.Reader, obj chan objectsInventory) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	invs, err := parseInv(payload)
	if err != nil {
		return fmt.Errorf("parseInv error: %v. Closing connection", err)
	}
	objects := make(objectsInventory)
	for _, inv := range invs {
		objects[inv.Hash] = p.ipPort
		// XXX Used during development.
		once.Do(
			func() {
				writeGetData(conn, []inventoryVector{inv})
			})
	}
	obj <- objects
	return nil
}

// Things needing implementation.
//
// - save the config frequently. be more resilient to BitMessage attacks.
// - Preserve the routing table for longer, don't delete nodes immediately
// - after they disconnect.
//
// from PyBitMessage:
//
// broadcastToSendDataQueues((0, 'shutdown', self.HOST))
//
// Zero out the list of already contacted nodes every 30 minutes, to give it
// another chance.
//
// self.receivedgetbiginv = False #Gets set to true once we receive a
// getbiginv message from our peer. An abusive peer might request it too much
// so we use this variable to check whether they have already asked for a big
// inv message.

// #Only one singleListener thread will ever exist. It creates the
// #receiveDataThread and sendDataThread for each incoming connection. Note
// #that it cannot set the stream number because it is not known yet- the
// #other node will have to tell us its stream number in a version message. If
// #we don't care about their stream, we will close the connection (within the
// #recversion function of the recieveData thread)

// #For each stream to which we connect, several outgoingSynSender threads
// #will exist and will collectively create 8 connections with peers.
//
// Random nonce used to detect connections to self.

type stats struct {
	streamConnectionCount map[int]int
}

// ipPort is a string that can be resolved with net.ResolveTCPAddr("tcp",
// ipPort) and the first part can be parsed by net.ParseIP(). It is illegal to
// create an ipPort that doesn't follow these conditions.
type ipPort string

func (ipPort ipPort) toNetworkAddress() extendedNetworkAddress {
	tcpAddr, _ := net.ResolveTCPAddr("tcp", string(ipPort))
	var rawIp [16]byte
	copy(rawIp[:], tcpAddr.IP)
	addr := extendedNetworkAddress{
		Time:   uint32(time.Now().Unix()),
		Stream: streamOne, // This should change after the version exchange.
		NetworkAddress: NetworkAddress{
			Services: ConnectionServiceNodeNetwork, //
			IP:       rawIp,
			Port:     uint16(tcpAddr.Port),
		},
	}
	return addr
}
<|endoftext|>


<commit_before>/*
The MIT License (MIT)

Copyright (c) 2016 Lau, Chok Sheak (for software "findfile")

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"bufio"
	"os"
	"strconv"
)

/**************************************************************************/

// Types.

type (
	contextLines struct {
		startIndex int
		lines      []contextLine
	}

	contextLine struct {
		lineAsString        string
		lineAsStringIsValid bool
	}
)

/**************************************************************************/

// Variables.

var (
	preContextLines         contextLines
	postContextLines        contextLines
	contextLinesFileScanner *bufio.Scanner
)

/**************************************************************************/

// Setup context lines.

func setupContextLines() {
	if optionContextLines.value == 0 {
		return
	}

	preContextLines = createContextLines(optionContextLines.value)
	postContextLines = createContextLines(optionContextLines.value)
}

func createContextLines(numLines int) contextLines {
	newContextLines := contextLines{
		startIndex: 0,
		lines:      make([]contextLine, numLines),
	}

	return newContextLines
}

/**************************************************************************/

// Reading lines and maintaining context lines.

func setFileForScanning(fileHandle *os.File) {
	contextLinesFileScanner = bufio.NewScanner(fileHandle)
}

// This function will read one line whenever it gets called.
func hasNextLineInFile() bool {
	return contextLinesFileScanner.Scan()
}

func getNextLineFromFile() string {
	line := contextLinesFileScanner.Text()
	numBytesRead += int64(len(line))
	return line
}

func addToContextLineIndex(index, delta int) int {
	index += delta
	if index < 0 {
		index += optionContextLines.value
	} else if index >= optionContextLines.value {
		index -= optionContextLines.value
	}
	return index
}

func getContextLineByDelta(delta int) contextLine {
	if (delta < -optionContextLines.value) || (delta > optionContextLines.value) {
		panic("delta out of range: " + strconv.Itoa(delta))
	}
	if delta == 0 {
		panic("delta cannot be zero")
	}

	if delta < 0 {
		index := addToContextLineIndex(preContextLines.startIndex, delta)
		return preContextLines.lines[index]
	}

	index := addToContextLineIndex(postContextLines.startIndex, delta-1)
	return postContextLines.lines[index]
}

// Check context lines.
func hasNextLineInFileOrCache() bool {
	if optionContextLines.value == 0 {
		return hasNextLineInFile()
	}

	if postContextLines.lines[postContextLines.startIndex].lineAsStringIsValid {
		return true
	}

	return hasNextLineInFile()
}

// Maintain context lines.
func getNextLineFromFileOrCache() string {
	if optionContextLines.value == 0 {
		return getNextLineFromFile()
	}

	// Read from post context lines.
	if postContextLines.lines[postContextLines.startIndex].lineAsStringIsValid {
		postContextLines.lines[postContextLines.startIndex].lineAsStringIsValid = false
		line := postContextLines.lines[postContextLines.startIndex].lineAsString
		postContextLines.startIndex = addToContextLineIndex(postContextLines.startIndex, 1)
		return line
	}

	return getNextLineFromFile()
}

func pushToPreContextLines(line string) {
	if optionContextLines.value == 0 {
		return
	}
	preContextLines.startIndex = addToContextLineIndex(preContextLines.startIndex, 1)
	preContextLines.lines[preContextLines.startIndex].lineAsStringIsValid = true
	preContextLines.lines[preContextLines.startIndex].lineAsString = line
}

func fillPostContextLines() {
	if optionContextLines.value == 0 {
		return
	}

	i := postContextLines.startIndex
	for {
		if !postContextLines.lines[i].lineAsStringIsValid {
			if !hasNextLineInFile() {
				break
			}

			postContextLines.lines[i].lineAsString = getNextLineFromFile()
			postContextLines.lines[i].lineAsStringIsValid = true
		}

		i = addToContextLineIndex(i, 1)
		if i == postContextLines.startIndex {
			break
		}
	}
}

/**************************************************************************/
<commit_msg>Fix bug with pre-context line offset<commit_after>/*
The MIT License (MIT)

Copyright (c) 2016 Lau, Chok Sheak (for software "findfile")

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"bufio"
	"os"
	"strconv"
)

/**************************************************************************/

// Types.

type (
	contextLines struct {
		startIndex int
		lines      []contextLine
	}

	contextLine struct {
		lineAsString        string
		lineAsStringIsValid bool
	}
)

/**************************************************************************/

// Variables.

var (
	preContextLines         contextLines
	postContextLines        contextLines
	contextLinesFileScanner *bufio.Scanner
)

/**************************************************************************/

// Setup context lines.

func setupContextLines() {
	if optionContextLines.value == 0 {
		return
	}

	preContextLines = createContextLines(optionContextLines.value)
	postContextLines = createContextLines(optionContextLines.value)
}

func createContextLines(numLines int) contextLines {
	newContextLines := contextLines{
		startIndex: 0,
		lines:      make([]contextLine, numLines),
	}

	return newContextLines
}

/**************************************************************************/

// Reading lines and maintaining context lines.

func setFileForScanning(fileHandle *os.File) {
	contextLinesFileScanner = bufio.NewScanner(fileHandle)
}

// This function will read one line whenever it gets called.
func hasNextLineInFile() bool {
	return contextLinesFileScanner.Scan()
}

func getNextLineFromFile() string {
	line := contextLinesFileScanner.Text()
	numBytesRead += int64(len(line))
	return line
}

func addToContextLineIndex(index, delta int) int {
	index += delta
	if index < 0 {
		index += optionContextLines.value
	} else if index >= optionContextLines.value {
		index -= optionContextLines.value
	}
	return index
}

func getContextLineByDelta(delta int) contextLine {
	if (delta < -optionContextLines.value) || (delta > optionContextLines.value) {
		panic("delta out of range: " + strconv.Itoa(delta))
	}
	if delta == 0 {
		panic("delta cannot be zero")
	}

	if delta < 0 {
		index := addToContextLineIndex(preContextLines.startIndex, delta+1)
		return preContextLines.lines[index]
	}

	index := addToContextLineIndex(postContextLines.startIndex, delta-1)
	return postContextLines.lines[index]
}

// Check context lines.
func hasNextLineInFileOrCache() bool {
	if optionContextLines.value == 0 {
		return hasNextLineInFile()
	}

	if postContextLines.lines[postContextLines.startIndex].lineAsStringIsValid {
		return true
	}

	return hasNextLineInFile()
}

// Maintain context lines.
func getNextLineFromFileOrCache() string {
	if optionContextLines.value == 0 {
		return getNextLineFromFile()
	}

	// Read from post context lines.
	if postContextLines.lines[postContextLines.startIndex].lineAsStringIsValid {
		postContextLines.lines[postContextLines.startIndex].lineAsStringIsValid = false
		line := postContextLines.lines[postContextLines.startIndex].lineAsString
		postContextLines.startIndex = addToContextLineIndex(postContextLines.startIndex, 1)
		return line
	}

	return getNextLineFromFile()
}

func pushToPreContextLines(line string) {
	if optionContextLines.value == 0 {
		return
	}
	preContextLines.startIndex = addToContextLineIndex(preContextLines.startIndex, 1)
	preContextLines.lines[preContextLines.startIndex].lineAsStringIsValid = true
	preContextLines.lines[preContextLines.startIndex].lineAsString = line
}

func fillPostContextLines() {
	if optionContextLines.value == 0 {
		return
	}

	i := postContextLines.startIndex
	for {
		if !postContextLines.lines[i].lineAsStringIsValid {
			if !hasNextLineInFile() {
				break
			}

			postContextLines.lines[i].lineAsString = getNextLineFromFile()
			postContextLines.lines[i].lineAsStringIsValid = true
		}

		i = addToContextLineIndex(i, 1)
		if i == postContextLines.startIndex {
			break
		}
	}
}

/**************************************************************************/
<|endoftext|>


<commit_before>package just

import (
	"errors"
	"mime/multipart"
	"net/url"
	"reflect"
	"strconv"
	"time"
)

func marshalUrlValues(ptr interface{}) ([]byte, error) {
	t, v := reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem()
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return nil, errors.New("Array and slice by root element not supported, only structure!")
	}
	values := make(url.Values)
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		if structFieldKind == reflect.Slice {
			for i := 0; i < structField.Len(); i++ {
				values.Add(inputFieldName, structField.Index(i).String())
			}
		} else {
			values.Set(inputFieldName, structField.String())
		}
	}
	return []byte(values.Encode()), nil
}

// mapForm - form mapping (+ multipart files support)
func mapForm(values map[string][]string, files map[string][]*multipart.FileHeader, ptr interface{}) error {
	return recursiveTreeMapForm(values, files, reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem())
}

func recursiveTreeMapForm(values map[string][]string, files map[string][]*multipart.FileHeader, t reflect.Type, v reflect.Value) error {
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return errors.New("Array and slice by root element not supported, only structure!")
	}
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			if typeField.Type.Kind() == reflect.Struct {
				if err := recursiveTreeMapForm(values, files, typeField.Type, structField); err != nil {
					return err
				}
			}
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		inputValue, existsValue := values[inputFieldName]
		if !existsValue {
			// Check files
			if files != nil && len(files) > 0 {
				if list, ok := files[inputFieldName]; ok {
					if numFiles := len(list); numFiles > 0 {
						if structFieldKind == reflect.Slice {
							if structField.Type().Elem().Name() == "FileHeader" {
								slice := reflect.MakeSlice(structField.Type(), numFiles, numFiles)
								for i := 0; i < numFiles; i++ {
									slice.Index(i).Set(reflect.ValueOf(list[i]))
								}
								v.Field(i).Set(slice)
							}
						} else if structField.Type().Elem().Name() == "FileHeader" {
							structField.Set(reflect.ValueOf(list[0]))
						}
					}
				}
			}
			continue
		}
		if numElements := len(inputValue); numElements > 0 {
			if structFieldKind == reflect.Slice {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElements, numElements)
				for i := 0; i < numElements; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				v.Field(i).Set(slice)
			} else {
				if _, isTime := structField.Interface().(time.Time); isTime {
					if err := setTimeField(inputValue[0], typeField, structField); err != nil {
						return err
					}
					continue
				}
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int:
		return setIntField(val, 0, structField)
	case reflect.Int8:
		return setIntField(val, 8, structField)
	case reflect.Int16:
		return setIntField(val, 16, structField)
	case reflect.Int32:
		return setIntField(val, 32, structField)
	case reflect.Int64:
		return setIntField(val, 64, structField)
	case reflect.Uint:
		return setUintField(val, 0, structField)
	case reflect.Uint8:
		return setUintField(val, 8, structField)
	case reflect.Uint16:
		return setUintField(val, 16, structField)
	case reflect.Uint32:
		return setUintField(val, 32, structField)
	case reflect.Uint64:
		return setUintField(val, 64, structField)
	case reflect.Bool:
		return setBoolField(val, structField)
	case reflect.Float32:
		return setFloatField(val, 32, structField)
	case reflect.Float64:
		return setFloatField(val, 64, structField)
	case reflect.String:
		structField.SetString(val)
	default:
		return errors.New("Unknown type")
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return nil
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}

func setTimeField(val string, structField reflect.StructField, value reflect.Value) error {
	timeFormat := structField.Tag.Get("time_format")
	if timeFormat == "" {
		return errors.New("Blank time format")
	}
	if val == "" {
		value.Set(reflect.ValueOf(time.Time{}))
		return nil
	}
	l := time.Local
	if isUTC, _ := strconv.ParseBool(structField.Tag.Get("time_utc")); isUTC {
		l = time.UTC
	}
	t, err := time.ParseInLocation(timeFormat, val, l)
	if err != nil {
		return err
	}
	value.Set(reflect.ValueOf(t))
	return nil
}
<commit_msg>- Add support ptr values in form mapping<commit_after>package just

import (
	"errors"
	"mime/multipart"
	"net/url"
	"reflect"
	"strconv"
	"time"
)

func marshalUrlValues(ptr interface{}) ([]byte, error) {
	t, v := reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem()
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return nil, errors.New("Array and slice by root element not supported, only structure!")
	}
	values := make(url.Values)
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		if structFieldKind == reflect.Slice {
			for i := 0; i < structField.Len(); i++ {
				values.Add(inputFieldName, structField.Index(i).String())
			}
		} else {
			values.Set(inputFieldName, structField.String())
		}
	}
	return []byte(values.Encode()), nil
}

// mapForm - form mapping (+ multipart files support)
func mapForm(values map[string][]string, files map[string][]*multipart.FileHeader, ptr interface{}) error {
	return recursiveTreeMapForm(values, files, reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem())
}

func recursiveTreeMapForm(values map[string][]string, files map[string][]*multipart.FileHeader, t reflect.Type, v reflect.Value) error {
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return errors.New("Array and slice by root element not supported, only structure!")
	}
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			if typeField.Type.Kind() == reflect.Struct {
				if err := recursiveTreeMapForm(values, files, typeField.Type, structField); err != nil {
					return err
				}
			}
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		inputValue, existsValue := values[inputFieldName]
		if !existsValue {
			// Check files
			if files != nil && len(files) > 0 {
				if list, ok := files[inputFieldName]; ok {
					if numFiles := len(list); numFiles > 0 {
						if structFieldKind == reflect.Slice {
							if structField.Type().Elem().Name() == "FileHeader" {
								slice := reflect.MakeSlice(structField.Type(), numFiles, numFiles)
								for i := 0; i < numFiles; i++ {
									slice.Index(i).Set(reflect.ValueOf(list[i]))
								}
								v.Field(i).Set(slice)
							}
						} else if structField.Type().Elem().Name() == "FileHeader" {
							structField.Set(reflect.ValueOf(list[0]))
						}
					}
				}
			}
			continue
		}
		if numElements := len(inputValue); numElements > 0 {
			if structFieldKind == reflect.Slice || structFieldKind == reflect.Array {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElements, numElements)
				for i := 0; i < numElements; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				v.Field(i).Set(slice)
			} else {
				if _, isTime := structField.Interface().(time.Time); isTime {
					if err := setTimeField(inputValue[0], typeField, structField); err != nil {
						return err
					}
					continue
				}
				kind := typeField.Type.Kind()
				if kind == reflect.Ptr {
					kind = typeField.Type.Elem().Kind()
					if structField.IsNil() {
						structField.Set(reflect.New(typeField.Type.Elem()))
					}
				}
				if err := setWithProperType(kind, inputValue[0], reflect.Indirect(structField)); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int:
		return setIntField(val, 0, structField)
	case reflect.Int8:
		return setIntField(val, 8, structField)
	case reflect.Int16:
		return setIntField(val, 16, structField)
	case reflect.Int32:
		return setIntField(val, 32, structField)
	case reflect.Int64:
		return setIntField(val, 64, structField)
	case reflect.Uint:
		return setUintField(val, 0, structField)
	case reflect.Uint8:
		return setUintField(val, 8, structField)
	case reflect.Uint16:
		return setUintField(val, 16, structField)
	case reflect.Uint32:
		return setUintField(val, 32, structField)
	case reflect.Uint64:
		return setUintField(val, 64, structField)
	case reflect.Bool:
		return setBoolField(val, structField)
	case reflect.Float32:
		return setFloatField(val, 32, structField)
	case reflect.Float64:
		return setFloatField(val, 64, structField)
	case reflect.String:
		structField.SetString(val)
	default:
		return errors.New("Unknown type")
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return nil
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}

func setTimeField(val string, structField reflect.StructField, value reflect.Value) error {
	timeFormat := structField.Tag.Get("time_format")
	if timeFormat == "" {
		return errors.New("Blank time format")
	}
	if val == "" {
		value.Set(reflect.ValueOf(time.Time{}))
		return nil
	}
	l := time.Local
	if isUTC, _ := strconv.ParseBool(structField.Tag.Get("time_utc")); isUTC {
		l = time.UTC
	}
	t, err := time.ParseInLocation(timeFormat, val, l)
	if err != nil {
		return err
	}
	value.Set(reflect.ValueOf(t))
	return nil
}
<|endoftext|>


<commit_before>package frame

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"

	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/shell"
)

var OptionNorc = false
var OptionGoColorable = true
var OptionEnableVirtualTerminalProcessing = false

type ScriptEngineForOption interface {
	SetArg([]string)
	RunFile(context.Context, string) ([]byte, error)
	RunString(context.Context, string) error
}

func OptionParse(sh *shell.Shell, e ScriptEngineForOption) (func(context.Context) error, error) {
	args := os.Args[1:]
	for i := 0; i < len(args); i++ {
		arg1 := args[i]
		if arg1 == "-k" {
			i++
			if i >= len(args) {
				return nil, errors.New("-k: requires parameters")
			}
			return func(ctx context.Context) error {
				sh.Interpret(ctx, args[i])
				return nil
			}, nil
		} else if arg1 == "-c" {
			i++
			if i >= len(args) {
				return nil, errors.New("-c: requires parameters")
			}
			return func(ctx context.Context) error {
				sh.Interpret(ctx, args[i])
				return io.EOF
			}, nil
		} else if arg1 == "-b" {
			i++
			if i >= len(args) {
				return nil, errors.New("-b: requires parameters")
			}
			data, err := base64.StdEncoding.DecodeString(args[i])
			if err != nil {
				return nil, err
			}
			text := string(data)
			return func(ctx context.Context) error {
				sh.Interpret(ctx, text)
				return io.EOF
			}, nil
		} else if arg1 == "-f" {
			i++
			if i >= len(args) {
				return nil, errors.New("-f: requires parameters")
			}
			if strings.HasSuffix(strings.ToLower(args[i]), ".lua") {
				// lua script
				return func(ctx context.Context) error {
					e.SetArg(args[i:])
					_, err := e.RunFile(ctx, args[i])
					if err != nil {
						return err
					} else {
						return io.EOF
					}
				}, nil
			} else {
				return func(ctx context.Context) error {
					// command script
					if err := sh.Source(ctx, args[i]); err != nil {
						return err
					}
					return io.EOF
				}, nil
			}
		} else if arg1 == "-e" {
			i++
			if i >= len(args) {
				return nil, errors.New("-e: requires parameters")
			}
			return func(ctx context.Context) error {
				e.SetArg(args[i:])
				err := e.RunString(ctx, args[i])
				if err != nil {
					return err
				} else {
					return io.EOF
				}
			}, nil
		} else if arg1 == "--norc" {
			OptionNorc = true
		} else if arg1 == "--lua-file" {
			i++
			if i >= len(args) {
				return nil, errors.New("--lua-file: requires parameters")
			}
			return func(ctx context.Context) error {
				e.SetArg(args[i:])
				_, err := e.RunFile(ctx, args[i])
				if err != nil {
					return err
				} else {
					return io.EOF
				}
			}, nil
		} else if arg1 == "--show-version-only" {
			OptionNorc = true
			return func(context.Context) error {
				fmt.Printf("%s-%s\n", Version, runtime.GOARCH)
				return io.EOF
			}, nil
		} else if arg1 == "--go-colorable" {
			OptionGoColorable = true
		} else if arg1 == "--no-go-colorable" {
			OptionGoColorable = false
		} else if arg1 == "--enable-virtual-terminal-processing" {
			OptionEnableVirtualTerminalProcessing = true
		} else if arg1 == "--disable-virtual-terminal-processing" {
			OptionEnableVirtualTerminalProcessing = false
		} else if arg1 == "--look-curdir-first" {
			shell.LookCurdirOrder = dos.LookCurdirFirst
		} else if arg1 == "--look-curdir-last" {
			shell.LookCurdirOrder = dos.LookCurdirLast
		} else if arg1 == "--look-curdir-never" {
			shell.LookCurdirOrder = dos.LookCurdirNever
		} else {
			fmt.Fprintf(os.Stderr, "%s: unknown parameter\n", arg1)
		}
	}

	return nil, nil
}

var SilentMode = false
<commit_msg>Make one function per one option.<commit_after>package frame

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"

	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/shell"
)

var OptionNorc = false
var OptionGoColorable = true
var OptionEnableVirtualTerminalProcessing = false

type ScriptEngineForOption interface {
	SetArg([]string)
	RunFile(context.Context, string) ([]byte, error)
	RunString(context.Context, string) error
}

type optionArg struct {
	args []string
	sh   *shell.Shell
	e    ScriptEngineForOption
}

type optionT struct {
	F func()
	V func(*optionArg) (func(context.Context) error, error)
	U string
}

var optionMap = map[string]optionT{
	"-k": optionT{
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-k: requires parameters")
			}
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, p.args[0])
				return nil
			}, nil
		},
	},
	"-c": optionT{
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-c: requires parameters")
			}
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, p.args[0])
				return io.EOF
			}, nil
		},
	},
	"-b": optionT{
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-b: requires parameters")
			}
			data, err := base64.StdEncoding.DecodeString(p.args[0])
			if err != nil {
				return nil, err
			}
			text := string(data)
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, text)
				return io.EOF
			}, nil
		},
	},
	"-f": optionT{
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-f: requires parameters")
			}
			if strings.HasSuffix(strings.ToLower(p.args[0]), ".lua") {
				// lua script
				return func(ctx context.Context) error {
					p.e.SetArg(p.args)
					_, err := p.e.RunFile(ctx, p.args[0])
					if err != nil {
						return err
					} else {
						return io.EOF
					}
				}, nil
			} else {
				return func(ctx context.Context) error {
					// command script
					if err := p.sh.Source(ctx, p.args[0]); err != nil {
						return err
					}
					return io.EOF
				}, nil
			}
		},
	},
	"-e": optionT{
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-e: requires parameters")
			}
			return func(ctx context.Context) error {
				p.e.SetArg(p.args)
				err := p.e.RunString(ctx, p.args[0])
				if err != nil {
					return err
				} else {
					return io.EOF
				}
			}, nil
		},
	},
	"--lua-file": optionT{
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("--lua-file: requires parameters")
			}
			return func(ctx context.Context) error {
				p.e.SetArg(p.args)
				_, err := p.e.RunFile(ctx, p.args[0])
				if err != nil {
					return err
				} else {
					return io.EOF
				}
			}, nil
		},
	},
	"--show-version-only": optionT{
		V: func(p *optionArg) (func(context.Context) error, error) {
			OptionNorc = true
			return func(context.Context) error {
				fmt.Printf("%s-%s\n", Version, runtime.GOARCH)
				return io.EOF
			}, nil
		},
	},
	"--disable-virtual-terminal-processing": optionT{
		F: func() {
			OptionEnableVirtualTerminalProcessing = false
		},
	},
	"--enable-virtual-terminal-processing": optionT{
		F: func() {
			OptionEnableVirtualTerminalProcessing = true
		},
	},
	"--no-go-colorable": optionT{
		F: func() {
			OptionGoColorable = false
		},
	},
	"--go-colorable": optionT{
		F: func() {
			OptionGoColorable = true
		},
	},
	"--norc": optionT{
		F: func() {
			OptionNorc = true
		},
	},
	"--look-curdir-first": optionT{
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirFirst
		},
	},
	"--look-curdir-last": optionT{
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirLast
		},
	},
	"--look-curdir-never": optionT{
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirNever
		},
	},
}

func OptionParse(sh *shell.Shell, e ScriptEngineForOption) (func(context.Context) error, error) {
	args := os.Args[1:]

	for i := 0; i < len(args); i++ {
		if f, ok := optionMap[args[i]]; ok {
			if f.F != nil {
				f.F()
			}
			if f.V != nil {
				return f.V(&optionArg{
					args: args[i+1:],
					sh:   sh,
					e:    e,
				})
			}
		} else {
			fmt.Fprintf(os.Stderr, "%s: unknown parameter\n", args[i])
		}
	}
	return nil, nil
}

var SilentMode = false
<|endoftext|>


<commit_before>package frame

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/mattn/go-colorable"

	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/history"
	"github.com/zetamatta/nyagos/readline"
	"github.com/zetamatta/nyagos/shell"
)

type CmdStreamConsole struct {
	shell.CmdSeeker
	DoPrompt func() (int, error)
	History  *history.Container
	Editor   *readline.Editor
	HistPath string
}

var console io.Writer
var prevOptionGoColorable bool = false

func GetConsole() io.Writer {
	if console == nil || prevOptionGoColorable != OptionGoColorable {
		if dos.IsEscapeSequenceAvailable() {
			console = os.Stdout
			dos.EnableStdoutVirtualTerminalProcessing()
		} else if OptionGoColorable {
			console = colorable.NewColorableStdout()
		} else {
			console = os.Stdout
			if OptionEnableVirtualTerminalProcessing {
				dos.EnableStdoutVirtualTerminalProcessing()
			}
		}
		prevOptionGoColorable = OptionGoColorable
	}
	return console
}

func NewCmdStreamConsole(doPrompt func() (int, error)) *CmdStreamConsole {
	history1 := &history.Container{}
	this := &CmdStreamConsole{
		History: history1,
		Editor: &readline.Editor{
			History: history1,
			Prompt:  doPrompt,
			Writer:  bufio.NewWriter(GetConsole())},
		HistPath: filepath.Join(AppDataDir(), "nyagos.history"),
		CmdSeeker: shell.CmdSeeker{
			PlainHistory: []string{},
			Pointer:      -1,
		},
	}
	history1.Load(this.HistPath)
	history1.Save(this.HistPath)
	return this
}

func (this *CmdStreamConsole) ReadLine(ctx context.Context) (context.Context, string, error) {
	if this.Pointer >= 0 {
		if this.Pointer < len(this.PlainHistory) {
			this.Pointer++
			return ctx, this.PlainHistory[this.Pointer-1], nil
		}
		this.Pointer = -1
	}
	var line string
	var err error
	for {
		line, err = this.Editor.ReadLine(ctx)
		if err != nil {
			return ctx, line, err
		}
		var isReplaced bool
		line, isReplaced, err = this.History.Replace(line)
		if err != nil {
			return ctx, line, err
		}
		if isReplaced {
			fmt.Fprintln(os.Stdout, line)
		}
		if line != "" {
			break
		}
	}
	row := history.NewHistoryLine(line)
	this.History.PushLine(row)
	fd, err := os.OpenFile(this.HistPath, os.O_APPEND|os.O_CREATE, 0600)
	if err == nil {
		fmt.Fprintln(fd, row.String())
		fd.Close()
	} else {
		fmt.Fprintln(os.Stderr, err.Error())
	}
	this.PlainHistory = append(this.PlainHistory, line)
	return ctx, line, err
}
<commit_msg>If `git.exe push` turn off Windows10's escape sequence, turn on again.<commit_after>package frame

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/mattn/go-colorable"

	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/history"
	"github.com/zetamatta/nyagos/readline"
	"github.com/zetamatta/nyagos/shell"
)

type CmdStreamConsole struct {
	shell.CmdSeeker
	DoPrompt func() (int, error)
	History  *history.Container
	Editor   *readline.Editor
	HistPath string
}

var console io.Writer
var prevOptionGoColorable bool = false

var isEscapeSequenceAvailable = false

func GetConsole() io.Writer {
	if isEscapeSequenceAvailable {
		dos.EnableStdoutVirtualTerminalProcessing()
		console = os.Stdout
	} else if console == nil || prevOptionGoColorable != OptionGoColorable {
		if dos.IsEscapeSequenceAvailable() {
			console = os.Stdout
			dos.EnableStdoutVirtualTerminalProcessing()
			isEscapeSequenceAvailable = true
		} else if OptionGoColorable {
			console = colorable.NewColorableStdout()
		} else {
			console = os.Stdout
			if OptionEnableVirtualTerminalProcessing {
				dos.EnableStdoutVirtualTerminalProcessing()
			}
		}
		prevOptionGoColorable = OptionGoColorable
	}
	return console
}

func NewCmdStreamConsole(doPrompt func() (int, error)) *CmdStreamConsole {
	history1 := &history.Container{}
	this := &CmdStreamConsole{
		History: history1,
		Editor: &readline.Editor{
			History: history1,
			Prompt:  doPrompt,
			Writer:  bufio.NewWriter(GetConsole())},
		HistPath: filepath.Join(AppDataDir(), "nyagos.history"),
		CmdSeeker: shell.CmdSeeker{
			PlainHistory: []string{},
			Pointer:      -1,
		},
	}
	history1.Load(this.HistPath)
	history1.Save(this.HistPath)
	return this
}

func (this *CmdStreamConsole) ReadLine(ctx context.Context) (context.Context, string, error) {
	if this.Pointer >= 0 {
		if this.Pointer < len(this.PlainHistory) {
			this.Pointer++
			return ctx, this.PlainHistory[this.Pointer-1], nil
		}
		this.Pointer = -1
	}
	var line string
	var err error
	for {
		line, err = this.Editor.ReadLine(ctx)
		if err != nil {
			return ctx, line, err
		}
		var isReplaced bool
		line, isReplaced, err = this.History.Replace(line)
		if err != nil {
			return ctx, line, err
		}
		if isReplaced {
			fmt.Fprintln(os.Stdout, line)
		}
		if line != "" {
			break
		}
	}
	row := history.NewHistoryLine(line)
	this.History.PushLine(row)
	fd, err := os.OpenFile(this.HistPath, os.O_APPEND|os.O_CREATE, 0600)
	if err == nil {
		fmt.Fprintln(fd, row.String())
		fd.Close()
	} else {
		fmt.Fprintln(os.Stderr, err.Error())
	}
	this.PlainHistory = append(this.PlainHistory, line)
	return ctx, line, err
}
<|endoftext|>


<commit_before>// Mostly copied from Go's src/cmd/gofmt:
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/google/blueprint/parser"
)

var (
	// main operation modes
	list            = flag.Bool("l", false, "list files that would be modified by bpmodify")
	write           = flag.Bool("w", false, "write result to (source) file instead of stdout")
	doDiff          = flag.Bool("d", false, "display diffs instead of rewriting files")
	sortLists       = flag.Bool("s", false, "sort touched lists, even if they were unsorted")
	parameter       = flag.String("parameter", "deps", "name of parameter to modify on each module")
	targetedModules = new(identSet)
	addIdents       = new(identSet)
	removeIdents    = new(identSet)
)

func init() {
	flag.Var(targetedModules, "m", "comma or whitespace separated list of modules on which to operate")
	flag.Var(addIdents, "a", "comma or whitespace separated list of identifiers to add")
	flag.Var(removeIdents, "r", "comma or whitespace separated list of identifiers to remove")
	flag.Usage = usage
}

var (
	exitCode = 0
)

func report(err error) {
	fmt.Fprintln(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] [path ...]\n", os.Args[0])
	flag.PrintDefaults()
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer) error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	r := bytes.NewBuffer(src)

	file, errs := parser.Parse(filename, r, parser.NewScope(nil))
	if len(errs) > 0 {
		for _, err := range errs {
			fmt.Fprintln(os.Stderr, err)
		}
		return fmt.Errorf("%d parsing errors", len(errs))
	}

	modified, errs := findModules(file)
	if len(errs) > 0 {
		for _, err := range errs {
			fmt.Fprintln(os.Stderr, err)
		}
		fmt.Fprintln(os.Stderr, "continuing...")
	}

	if modified {
		res, err := parser.Print(file)
		if err != nil {
			return err
		}

		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0644)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s bpfmt/%s\n", filename, filename)
			out.Write(data)
		}

		if !*list && !*write && !*doDiff {
			_, err = out.Write(res)
		}
	}

	return err
}

func findModules(file *parser.File) (modified bool, errs []error) {

	for _, def := range file.Defs {
		if module, ok := def.(*parser.Module); ok {
			for _, prop := range module.Properties {
				if prop.Name == "name" && prop.Value.Type() == parser.StringType {
					if targetedModule(prop.Value.Eval().(*parser.String).Value) {
						m, newErrs := processModule(module, prop.Name, file)
						errs = append(errs, newErrs...)
						modified = modified || m
					}
				}
			}
		}
	}

	return modified, errs
}

func processModule(module *parser.Module, moduleName string,
	file *parser.File) (modified bool, errs []error) {

	for _, prop := range module.Properties {
		if prop.Name == *parameter {
			modified, errs = processParameter(prop.Value, *parameter, moduleName, file)
			return
		}
	}

	prop := parser.Property{Name: *parameter, Value: &parser.List{}}
	modified, errs = processParameter(prop.Value, *parameter, moduleName, file)

	if modified {
		module.Properties = append(module.Properties, &prop)
	}

	return modified, errs
}

func processParameter(value parser.Expression, paramName, moduleName string,
	file *parser.File) (modified bool, errs []error) {
	if _, ok := value.(*parser.Variable); ok {
		return false, []error{fmt.Errorf("parameter %s in module %s is a variable, unsupported",
			paramName, moduleName)}
	}

	if _, ok := value.(*parser.Operator); ok {
		return false, []error{fmt.Errorf("parameter %s in module %s is an expression, unsupported",
			paramName, moduleName)}
	}

	list, ok := value.(*parser.List)
	if !ok {
		return false, []error{fmt.Errorf("expected parameter %s in module %s to be list, found %s",
			paramName, moduleName, value.Type().String())}
	}

	wasSorted := parser.ListIsSorted(list)

	for _, a := range addIdents.idents {
		m := parser.AddStringToList(list, a)
		modified = modified || m
	}

	for _, r := range removeIdents.idents {
		m := parser.RemoveStringFromList(list, r)
		modified = modified || m
	}

	if (wasSorted || *sortLists) && modified {
		parser.SortList(file, list)
	}

	return modified, nil
}

func targetedModule(name string) bool {
	if targetedModules.all {
		return true
	}
	for _, m := range targetedModules.idents {
		if m == name {
			return true
		}
	}

	return false
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && f.Name() == "Blueprints" {
		err = processFile(path, nil, os.Stdout)
	}
	if err != nil {
		report(err)
	}
	return nil
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func main() {
	defer func() { os.Exit(exitCode) }()

	flag.Parse()

	if flag.NArg() == 0 {
		if *write {
			report(fmt.Errorf("error: cannot use -w with standard input"))
			return
		}
		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
			report(err)
		}
		return
	}

	if len(targetedModules.idents) == 0 {
		report(fmt.Errorf("-m parameter is required"))
		return
	}

	if len(addIdents.idents) == 0 && len(removeIdents.idents) == 0 {
		report(fmt.Errorf("-a or -r parameter is required"))
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout); err != nil {
				report(err)
			}
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err error) {
	f1, err := ioutil.TempFile("", "bpfmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "bpfmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-uw", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return

}

type identSet struct {
	idents []string
	all    bool
}

func (m *identSet) String() string {
	return strings.Join(m.idents, ",")
}

func (m *identSet) Set(s string) error {
	m.idents = strings.FieldsFunc(s, func(c rune) bool {
		return unicode.IsSpace(c) || c == ','
	})
	if len(m.idents) == 1 && m.idents[0] == "*" {
		m.all = true
	}
	return nil
}

func (m *identSet) Get() interface{} {
	return m.idents
}
<commit_msg>bpmodify: fix os.Exit() shouldn't shadow panic()<commit_after>// Mostly copied from Go's src/cmd/gofmt:
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/google/blueprint/parser"
)

var (
	// main operation modes
	list            = flag.Bool("l", false, "list files that would be modified by bpmodify")
	write           = flag.Bool("w", false, "write result to (source) file instead of stdout")
	doDiff          = flag.Bool("d", false, "display diffs instead of rewriting files")
	sortLists       = flag.Bool("s", false, "sort touched lists, even if they were unsorted")
	parameter       = flag.String("parameter", "deps", "name of parameter to modify on each module")
	targetedModules = new(identSet)
	addIdents       = new(identSet)
	removeIdents    = new(identSet)
)

func init() {
	flag.Var(targetedModules, "m", "comma or whitespace separated list of modules on which to operate")
	flag.Var(addIdents, "a", "comma or whitespace separated list of identifiers to add")
	flag.Var(removeIdents, "r", "comma or whitespace separated list of identifiers to remove")
	flag.Usage = usage
}

var (
	exitCode = 0
)

func report(err error) {
	fmt.Fprintln(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] [path ...]\n", os.Args[0])
	flag.PrintDefaults()
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer) error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	r := bytes.NewBuffer(src)

	file, errs := parser.Parse(filename, r, parser.NewScope(nil))
	if len(errs) > 0 {
		for _, err := range errs {
			fmt.Fprintln(os.Stderr, err)
		}
		return fmt.Errorf("%d parsing errors", len(errs))
	}

	modified, errs := findModules(file)
	if len(errs) > 0 {
		for _, err := range errs {
			fmt.Fprintln(os.Stderr, err)
		}
		fmt.Fprintln(os.Stderr, "continuing...")
	}

	if modified {
		res, err := parser.Print(file)
		if err != nil {
			return err
		}

		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0644)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s bpfmt/%s\n", filename, filename)
			out.Write(data)
		}

		if !*list && !*write && !*doDiff {
			_, err = out.Write(res)
		}
	}

	return err
}

func findModules(file *parser.File) (modified bool, errs []error) {

	for _, def := range file.Defs {
		if module, ok := def.(*parser.Module); ok {
			for _, prop := range module.Properties {
				if prop.Name == "name" && prop.Value.Type() == parser.StringType {
					if targetedModule(prop.Value.Eval().(*parser.String).Value) {
						m, newErrs := processModule(module, prop.Name, file)
						errs = append(errs, newErrs...)
						modified = modified || m
					}
				}
			}
		}
	}

	return modified, errs
}

func processModule(module *parser.Module, moduleName string,
	file *parser.File) (modified bool, errs []error) {

	for _, prop := range module.Properties {
		if prop.Name == *parameter {
			modified, errs = processParameter(prop.Value, *parameter, moduleName, file)
			return
		}
	}

	prop := parser.Property{Name: *parameter, Value: &parser.List{}}
	modified, errs = processParameter(prop.Value, *parameter, moduleName, file)

	if modified {
		module.Properties = append(module.Properties, &prop)
	}

	return modified, errs
}

func processParameter(value parser.Expression, paramName, moduleName string,
	file *parser.File) (modified bool, errs []error) {
	if _, ok := value.(*parser.Variable); ok {
		return false, []error{fmt.Errorf("parameter %s in module %s is a variable, unsupported",
			paramName, moduleName)}
	}

	if _, ok := value.(*parser.Operator); ok {
		return false, []error{fmt.Errorf("parameter %s in module %s is an expression, unsupported",
			paramName, moduleName)}
	}

	list, ok := value.(*parser.List)
	if !ok {
		return false, []error{fmt.Errorf("expected parameter %s in module %s to be list, found %s",
			paramName, moduleName, value.Type().String())}
	}

	wasSorted := parser.ListIsSorted(list)

	for _, a := range addIdents.idents {
		m := parser.AddStringToList(list, a)
		modified = modified || m
	}

	for _, r := range removeIdents.idents {
		m := parser.RemoveStringFromList(list, r)
		modified = modified || m
	}

	if (wasSorted || *sortLists) && modified {
		parser.SortList(file, list)
	}

	return modified, nil
}

func targetedModule(name string) bool {
	if targetedModules.all {
		return true
	}
	for _, m := range targetedModules.idents {
		if m == name {
			return true
		}
	}

	return false
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && f.Name() == "Blueprints" {
		err = processFile(path, nil, os.Stdout)
	}
	if err != nil {
		report(err)
	}
	return nil
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func main() {
	defer func() {
		if err := recover(); err != nil {
			report(fmt.Errorf("error: %s", err))
		}
		os.Exit(exitCode)
	}()

	flag.Parse()

	if flag.NArg() == 0 {
		if *write {
			report(fmt.Errorf("error: cannot use -w with standard input"))
			return
		}
		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
			report(err)
		}
		return
	}

	if len(targetedModules.idents) == 0 {
		report(fmt.Errorf("-m parameter is required"))
		return
	}

	if len(addIdents.idents) == 0 && len(removeIdents.idents) == 0 {
		report(fmt.Errorf("-a or -r parameter is required"))
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout); err != nil {
				report(err)
			}
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err error) {
	f1, err := ioutil.TempFile("", "bpfmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "bpfmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-uw", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return

}

type identSet struct {
	idents []string
	all    bool
}

func (m *identSet) String() string {
	return strings.Join(m.idents, ",")
}

func (m *identSet) Set(s string) error {
	m.idents = strings.FieldsFunc(s, func(c rune) bool {
		return unicode.IsSpace(c) || c == ','
	})
	if len(m.idents) == 1 && m.idents[0] == "*" {
		m.all = true
	}
	return nil
}

func (m *identSet) Get() interface{} {
	return m.idents
}
<|endoftext|>


<commit_before>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package brotli

import "io"
import "bufio"

// The bitReader preserves the property that it will never read more bytes than
// is necessary. However, this feature dramatically hurts performance because
// every byte needs to be obtained through a ReadByte method call.
// Furthermore, the decoding of variable length codes in ReadSymbol, often
// requires multiple passes before it knows the exact bit-length of the code.
//
// Thus, to improve performance, if the underlying byteReader is a bufio.Reader,
// then the bitReader will use the Peek and Discard methods to fill the internal
// bit buffer with as many bits as possible, allowing the TryReadBits and
// TryReadSymbol methods to often succeed on the first try.

type byteReader interface {
	io.Reader
	io.ByteReader
}

type bitReader struct {
	rd      byteReader
	bufBits uint64 // Buffer to hold some bits
	numBits uint   // Number of valid bits in bufBits
	offset  int64  // Number of bytes read from the underlying io.Reader

	// These fields are only used if rd is a bufio.Reader.
	bufRd       *bufio.Reader
	bufPeek     []byte // Buffer for the Peek data
	idxPeek     int    // The current index into the Peek buffer
	discardBits int    // Number of bits to discard from bufio.Reader
	fedBits     uint   // Number of bits fed in last call to FeedBits

	// Local copy of decoders to reduce memory allocations.
	prefix prefixDecoder
}

func (br *bitReader) Init(r io.Reader) {
	*br = bitReader{prefix: br.prefix}
	if rr, ok := r.(byteReader); ok {
		br.rd = rr
	} else {
		br.rd = bufio.NewReader(r)
	}
	if brd, ok := br.rd.(*bufio.Reader); ok {
		br.bufRd = brd
	}
}

// FlushOffset updates the read offset of the underlying byteReader.
// If the byteReader is a bufio.Reader, then this calls Discard to update the
// read offset.
func (br *bitReader) FlushOffset() int64 {
	if br.bufRd == nil {
		return br.offset
	}

	// Update the number of total bits to discard.
	br.discardBits += int(br.fedBits - br.numBits)
	br.fedBits = br.numBits

	// Discard some bytes to update read offset.
	nd := (br.discardBits + 7) / 8 // Round up to nearest byte
	nd, _ = br.bufRd.Discard(nd)
	br.discardBits -= nd * 8 // -7..0
	br.offset += int64(nd)

	// These are invalid after Discard.
	br.bufPeek = nil
	br.idxPeek = 0

	return br.offset
}

// FeedBits ensures that at least nb bits exist in the bit buffer.
// If the underlying byteReader is a bufio.Reader, then this will fill the
// bit buffer with as many bits as possible, relying on Peek and Discard to
// properly advance the read offset. Otherwise, it will use ReadByte to fill the
// buffer with just the right number of bits.
func (br *bitReader) FeedBits(nb uint) {
	if br.bufRd != nil {
		br.discardBits += int(br.fedBits - br.numBits)
		for br.numBits <= 56 {
			if len(br.bufPeek) <= br.idxPeek {
				br.fedBits = br.numBits // Don't discard bits just added
				br.FlushOffset()

				var err error
				cntPeek := 8 // Minimum Peek amount to make progress
				if br.bufRd.Buffered() > cntPeek {
					cntPeek = br.bufRd.Buffered()
				}
				br.bufPeek, err = br.bufRd.Peek(cntPeek)

				br.idxPeek = int(br.numBits / 8)
				if len(br.bufPeek) <= br.idxPeek {
					if br.numBits >= nb {
						break
					}
					if err == io.EOF {
						err = io.ErrUnexpectedEOF
					}
					panic(err)
				}
			}
			br.bufBits |= uint64(br.bufPeek[br.idxPeek]) << br.numBits
			br.numBits += 8
			br.idxPeek++
		}
		br.fedBits = br.numBits
	} else {
		for br.numBits < nb {
			c, err := br.rd.ReadByte()
			if err != nil {
				if err == io.EOF {
					err = io.ErrUnexpectedEOF
				}
				panic(err)
			}
			br.bufBits |= uint64(c) << br.numBits
			br.numBits += 8
			br.offset++
		}
	}
}

// Read reads up to len(buf) bytes into buf.
func (br *bitReader) Read(buf []byte) (cnt int, err error) {
	if br.numBits%8 != 0 {
		return 0, Error("non-aligned bit buffer")
	}
	if br.numBits > 0 {
		for cnt = 0; len(buf) > cnt && br.numBits > 0; cnt++ {
			buf[cnt] = byte(br.bufBits)
			br.bufBits >>= 8
			br.numBits -= 8
		}
	} else {
		br.FlushOffset()
		cnt, err = br.rd.Read(buf)
		br.offset += int64(cnt)
	}
	return cnt, err
}

// TryReadBits attempts to read nb bits using the contents of the bit buffer
// alone. It returns the value and whether it succeeded.
//
// This method is designed to be inlined for performance reasons.
func (br *bitReader) TryReadBits(nb uint) (uint, bool) {
	if br.numBits < nb {
		return 0, false
	}
	val := uint(br.bufBits & uint64(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val, true
}

// ReadBits reads nb bits in LSB order from the underlying reader.
func (br *bitReader) ReadBits(nb uint) uint {
	br.FeedBits(nb)
	val := uint(br.bufBits & uint64(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment.
func (br *bitReader) ReadPads() uint {
	nb := br.numBits % 8
	val := uint(br.bufBits & uint64(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// TryReadSymbol attempts to decode the next symbol using the contents of the
// bit buffer alone. It returns the decoded symbol and whether it succeeded.
//
// This method is designed to be inlined for performance reasons.
func (br *bitReader) TryReadSymbol(pd *prefixDecoder) (uint, bool) {
	if br.numBits < uint(pd.minBits) || len(pd.chunks) == 0 {
		return 0, false
	}
	chunk := pd.chunks[uint32(br.bufBits)&pd.chunkMask]
	nb := uint(chunk & prefixCountMask)
	if nb > br.numBits || nb > uint(pd.chunkBits) {
		return 0, false
	}
	br.bufBits >>= nb
	br.numBits -= nb
	return uint(chunk >> prefixCountBits), true
}

// ReadSymbol reads the next prefix symbol using the provided prefixDecoder.
func (br *bitReader) ReadSymbol(pd *prefixDecoder) uint {
	if len(pd.chunks) == 0 {
		panic(ErrCorrupt) // Decode with empty tree
	}

	nb := uint(pd.minBits)
	for {
		br.FeedBits(nb)
		chunk := pd.chunks[uint32(br.bufBits)&pd.chunkMask]
		nb = uint(chunk & prefixCountMask)
		if nb > uint(pd.chunkBits) {
			linkIdx := chunk >> prefixCountBits
			chunk = pd.links[linkIdx][uint32(br.bufBits>>pd.chunkBits)&pd.linkMask]
			nb = uint(chunk & prefixCountMask)
		}
		if nb <= br.numBits {
			br.bufBits >>= nb
			br.numBits -= nb
			return uint(chunk >> prefixCountBits)
		}
	}
}

// ReadOffset reads an offset value using the provided rangesCodes indexed by
// the given symbol.
func (br *bitReader) ReadOffset(sym uint, rcs []rangeCode) uint {
	rc := rcs[sym]
	return uint(rc.base) + br.ReadBits(uint(rc.bits))
}

// ReadPrefixCode reads the prefix definition from the stream and initializes
// the provided prefixDecoder. The value maxSyms is the alphabet size of the
// prefix code being generated. The actual number of representable symbols
// will be between 1 and maxSyms, inclusively.
func (br *bitReader) ReadPrefixCode(pd *prefixDecoder, maxSyms uint) {
	hskip := br.ReadBits(2)
	if hskip == 1 {
		br.readSimplePrefixCode(pd, maxSyms)
	} else {
		br.readComplexPrefixCode(pd, maxSyms, hskip)
	}
}

// readSimplePrefixCode reads the prefix code according to RFC section 3.4.
func (br *bitReader) readSimplePrefixCode(pd *prefixDecoder, maxSyms uint) {
	var codes [4]prefixCode
	nsym := int(br.ReadBits(2)) + 1
	clen := neededBits(uint32(maxSyms))
	for i := 0; i < nsym; i++ {
		codes[i].sym = uint32(br.ReadBits(clen))
	}

	var copyLens = func(lens []uint) {
		for i := 0; i < nsym; i++ {
			codes[i].len = uint32(lens[i])
		}
	}
	var compareSwap = func(i, j int) {
		if codes[i].sym > codes[j].sym {
			codes[i], codes[j] = codes[j], codes[i]
		}
	}

	switch nsym {
	case 1:
		copyLens(simpleLens1[:])
	case 2:
		copyLens(simpleLens2[:])
		compareSwap(0, 1)
	case 3:
		copyLens(simpleLens3[:])
		compareSwap(0, 1)
		compareSwap(0, 2)
		compareSwap(1, 2)
	case 4:
		if tsel := br.ReadBits(1) == 1; !tsel {
			copyLens(simpleLens4a[:])
		} else {
			copyLens(simpleLens4b[:])
		}
		compareSwap(0, 1)
		compareSwap(2, 3)
		compareSwap(0, 2)
		compareSwap(1, 3)
		compareSwap(1, 2)
	}
	if uint(codes[nsym-1].sym) >= maxSyms {
		panic(ErrCorrupt) // Symbol goes beyond range of alphabet
	}
	pd.Init(codes[:nsym], true) // Must have 1..4 symbols
}

// readComplexPrefixCode reads the prefix code according to RFC section 3.5.
func (br *bitReader) readComplexPrefixCode(pd *prefixDecoder, maxSyms, hskip uint) {
	// Read the code-lengths prefix table.
	var codeCLensArr [len(complexLens)]prefixCode // Sorted, but may have holes
	sum := 32
	for _, sym := range complexLens[hskip:] {
		clen := br.ReadSymbol(&decCLens)
		if clen > 0 {
			codeCLensArr[sym] = prefixCode{sym: uint32(sym), len: uint32(clen)}
			if sum -= 32 >> clen; sum <= 0 {
				break
			}
		}
	}
	codeCLens := codeCLensArr[:0] // Compact the array to have no holes
	for _, c := range codeCLensArr {
		if c.len > 0 {
			codeCLens = append(codeCLens, c)
		}
	}
	if len(codeCLens) < 1 {
		panic(ErrCorrupt)
	}
	br.prefix.Init(codeCLens, true) // Must have 1..len(complexLens) symbols

	// Use code-lengths table to decode rest of prefix table.
	var codesArr [maxNumAlphabetSyms]prefixCode
	var sym, repSymLast, repCntLast, clenLast uint = 0, 0, 0, 8
	codes := codesArr[:0]
	for sym, sum = 0, 32768; sym < maxSyms && sum > 0; {
		clen := br.ReadSymbol(&br.prefix)
		if clen < 16 {
			// Literal bit-length symbol used.
			if clen > 0 {
				codes = append(codes, prefixCode{sym: uint32(sym), len: uint32(clen)})
				clenLast = clen
				sum -= 32768 >> clen
			}
			repSymLast = 0 // Reset last repeater symbol
			sym++
		} else {
			// Repeater symbol used.
			//	16: Repeat previous non-zero code-length
			//	17: Repeat code length of zero

			repSym := clen // Rename clen for better clarity
			if repSym != repSymLast {
				repCntLast = 0
				repSymLast = repSym
			}

			nb := repSym - 14          // 2..3 bits
			rep := br.ReadBits(nb) + 3 // 3..6 or 3..10
			if repCntLast > 0 {
				rep += (repCntLast - 2) << nb // Modify previous repeat count
			}
			repDiff := rep - repCntLast // Always positive
			repCntLast = rep

			if repSym == 16 {
				clen := clenLast
				for symEnd := sym + repDiff; sym < symEnd; sym++ {
					codes = append(codes, prefixCode{sym: uint32(sym), len: uint32(clen)})
				}
				sum -= int(repDiff) * (32768 >> clen)
			} else {
				sym += repDiff
			}
		}
	}
	if len(codes) < 2 || sym > maxSyms {
		panic(ErrCorrupt)
	}
	pd.Init(codes, true) // Must have 2..maxSyms symbols
}
<commit_msg>brotli: speed up decoding by using range instead of indexes<commit_after>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package brotli

import "io"
import "bufio"

// The bitReader preserves the property that it will never read more bytes than
// is necessary. However, this feature dramatically hurts performance because
// every byte needs to be obtained through a ReadByte method call.
// Furthermore, the decoding of variable length codes in ReadSymbol, often
// requires multiple passes before it knows the exact bit-length of the code.
//
// Thus, to improve performance, if the underlying byteReader is a bufio.Reader,
// then the bitReader will use the Peek and Discard methods to fill the internal
// bit buffer with as many bits as possible, allowing the TryReadBits and
// TryReadSymbol methods to often succeed on the first try.

type byteReader interface {
	io.Reader
	io.ByteReader
}

type bitReader struct {
	rd      byteReader
	bufBits uint64 // Buffer to hold some bits
	numBits uint   // Number of valid bits in bufBits
	offset  int64  // Number of bytes read from the underlying io.Reader

	// These fields are only used if rd is a bufio.Reader.
	bufRd       *bufio.Reader
	bufPeek     []byte // Buffer for the Peek data
	discardBits int    // Number of bits to discard from bufio.Reader
	fedBits     uint   // Number of bits fed in last call to FeedBits

	// Local copy of decoders to reduce memory allocations.
	prefix prefixDecoder
}

func (br *bitReader) Init(r io.Reader) {
	*br = bitReader{prefix: br.prefix}
	if rr, ok := r.(byteReader); ok {
		br.rd = rr
	} else {
		br.rd = bufio.NewReader(r)
	}
	if brd, ok := br.rd.(*bufio.Reader); ok {
		br.bufRd = brd
	}
}

// FlushOffset updates the read offset of the underlying byteReader.
// If the byteReader is a bufio.Reader, then this calls Discard to update the
// read offset.
func (br *bitReader) FlushOffset() int64 {
	if br.bufRd == nil {
		return br.offset
	}

	// Update the number of total bits to discard.
	br.discardBits += int(br.fedBits - br.numBits)
	br.fedBits = br.numBits

	// Discard some bytes to update read offset.
	nd := (br.discardBits + 7) / 8 // Round up to nearest byte
	nd, _ = br.bufRd.Discard(nd)
	br.discardBits -= nd * 8 // -7..0
	br.offset += int64(nd)

	// These are invalid after Discard.
	br.bufPeek = nil
	return br.offset
}

// FeedBits ensures that at least nb bits exist in the bit buffer.
// If the underlying byteReader is a bufio.Reader, then this will fill the
// bit buffer with as many bits as possible, relying on Peek and Discard to
// properly advance the read offset. Otherwise, it will use ReadByte to fill the
// buffer with just the right number of bits.
func (br *bitReader) FeedBits(nb uint) {
	if br.bufRd != nil {
		br.discardBits += int(br.fedBits - br.numBits)
		for {
			if len(br.bufPeek) == 0 {
				br.fedBits = br.numBits // Don't discard bits just added
				br.FlushOffset()

				var err error
				cntPeek := 8 // Minimum Peek amount to make progress
				if br.bufRd.Buffered() > cntPeek {
					cntPeek = br.bufRd.Buffered()
				}
				br.bufPeek, err = br.bufRd.Peek(cntPeek)
				br.bufPeek = br.bufPeek[int(br.numBits/8):] // Skip buffered bits
				if len(br.bufPeek) == 0 {
					if br.numBits >= nb {
						break
					}
					if err == io.EOF {
						err = io.ErrUnexpectedEOF
					}
					panic(err)
				}
			}
			cnt := int(64-br.numBits) / 8
			if cnt > len(br.bufPeek) {
				cnt = len(br.bufPeek)
			}
			for _, c := range br.bufPeek[:cnt] {
				br.bufBits |= uint64(c) << br.numBits
				br.numBits += 8
			}
			br.bufPeek = br.bufPeek[cnt:]
			if br.numBits > 56 {
				break
			}
		}
		br.fedBits = br.numBits
	} else {
		for br.numBits < nb {
			c, err := br.rd.ReadByte()
			if err != nil {
				if err == io.EOF {
					err = io.ErrUnexpectedEOF
				}
				panic(err)
			}
			br.bufBits |= uint64(c) << br.numBits
			br.numBits += 8
			br.offset++
		}
	}
}

// Read reads up to len(buf) bytes into buf.
func (br *bitReader) Read(buf []byte) (cnt int, err error) {
	if br.numBits%8 != 0 {
		return 0, Error("non-aligned bit buffer")
	}
	if br.numBits > 0 {
		for cnt = 0; len(buf) > cnt && br.numBits > 0; cnt++ {
			buf[cnt] = byte(br.bufBits)
			br.bufBits >>= 8
			br.numBits -= 8
		}
	} else {
		br.FlushOffset()
		cnt, err = br.rd.Read(buf)
		br.offset += int64(cnt)
	}
	return cnt, err
}

// TryReadBits attempts to read nb bits using the contents of the bit buffer
// alone. It returns the value and whether it succeeded.
//
// This method is designed to be inlined for performance reasons.
func (br *bitReader) TryReadBits(nb uint) (uint, bool) {
	if br.numBits < nb {
		return 0, false
	}
	val := uint(br.bufBits & uint64(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val, true
}

// ReadBits reads nb bits in LSB order from the underlying reader.
func (br *bitReader) ReadBits(nb uint) uint {
	br.FeedBits(nb)
	val := uint(br.bufBits & uint64(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment.
func (br *bitReader) ReadPads() uint {
	nb := br.numBits % 8
	val := uint(br.bufBits & uint64(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// TryReadSymbol attempts to decode the next symbol using the contents of the
// bit buffer alone. It returns the decoded symbol and whether it succeeded.
//
// This method is designed to be inlined for performance reasons.
func (br *bitReader) TryReadSymbol(pd *prefixDecoder) (uint, bool) {
	if br.numBits < uint(pd.minBits) || len(pd.chunks) == 0 {
		return 0, false
	}
	chunk := pd.chunks[uint32(br.bufBits)&pd.chunkMask]
	nb := uint(chunk & prefixCountMask)
	if nb > br.numBits || nb > uint(pd.chunkBits) {
		return 0, false
	}
	br.bufBits >>= nb
	br.numBits -= nb
	return uint(chunk >> prefixCountBits), true
}

// ReadSymbol reads the next prefix symbol using the provided prefixDecoder.
func (br *bitReader) ReadSymbol(pd *prefixDecoder) uint {
	if len(pd.chunks) == 0 {
		panic(ErrCorrupt) // Decode with empty tree
	}

	nb := uint(pd.minBits)
	for {
		br.FeedBits(nb)
		chunk := pd.chunks[uint32(br.bufBits)&pd.chunkMask]
		nb = uint(chunk & prefixCountMask)
		if nb > uint(pd.chunkBits) {
			linkIdx := chunk >> prefixCountBits
			chunk = pd.links[linkIdx][uint32(br.bufBits>>pd.chunkBits)&pd.linkMask]
			nb = uint(chunk & prefixCountMask)
		}
		if nb <= br.numBits {
			br.bufBits >>= nb
			br.numBits -= nb
			return uint(chunk >> prefixCountBits)
		}
	}
}

// ReadOffset reads an offset value using the provided rangesCodes indexed by
// the given symbol.
func (br *bitReader) ReadOffset(sym uint, rcs []rangeCode) uint {
	rc := rcs[sym]
	return uint(rc.base) + br.ReadBits(uint(rc.bits))
}

// ReadPrefixCode reads the prefix definition from the stream and initializes
// the provided prefixDecoder. The value maxSyms is the alphabet size of the
// prefix code being generated. The actual number of representable symbols
// will be between 1 and maxSyms, inclusively.
func (br *bitReader) ReadPrefixCode(pd *prefixDecoder, maxSyms uint) {
	hskip := br.ReadBits(2)
	if hskip == 1 {
		br.readSimplePrefixCode(pd, maxSyms)
	} else {
		br.readComplexPrefixCode(pd, maxSyms, hskip)
	}
}

// readSimplePrefixCode reads the prefix code according to RFC section 3.4.
func (br *bitReader) readSimplePrefixCode(pd *prefixDecoder, maxSyms uint) {
	var codes [4]prefixCode
	nsym := int(br.ReadBits(2)) + 1
	clen := neededBits(uint32(maxSyms))
	for i := 0; i < nsym; i++ {
		codes[i].sym = uint32(br.ReadBits(clen))
	}

	var copyLens = func(lens []uint) {
		for i := 0; i < nsym; i++ {
			codes[i].len = uint32(lens[i])
		}
	}
	var compareSwap = func(i, j int) {
		if codes[i].sym > codes[j].sym {
			codes[i], codes[j] = codes[j], codes[i]
		}
	}

	switch nsym {
	case 1:
		copyLens(simpleLens1[:])
	case 2:
		copyLens(simpleLens2[:])
		compareSwap(0, 1)
	case 3:
		copyLens(simpleLens3[:])
		compareSwap(0, 1)
		compareSwap(0, 2)
		compareSwap(1, 2)
	case 4:
		if tsel := br.ReadBits(1) == 1; !tsel {
			copyLens(simpleLens4a[:])
		} else {
			copyLens(simpleLens4b[:])
		}
		compareSwap(0, 1)
		compareSwap(2, 3)
		compareSwap(0, 2)
		compareSwap(1, 3)
		compareSwap(1, 2)
	}
	if uint(codes[nsym-1].sym) >= maxSyms {
		panic(ErrCorrupt) // Symbol goes beyond range of alphabet
	}
	pd.Init(codes[:nsym], true) // Must have 1..4 symbols
}

// readComplexPrefixCode reads the prefix code according to RFC section 3.5.
func (br *bitReader) readComplexPrefixCode(pd *prefixDecoder, maxSyms, hskip uint) {
	// Read the code-lengths prefix table.
	var codeCLensArr [len(complexLens)]prefixCode // Sorted, but may have holes
	sum := 32
	for _, sym := range complexLens[hskip:] {
		clen := br.ReadSymbol(&decCLens)
		if clen > 0 {
			codeCLensArr[sym] = prefixCode{sym: uint32(sym), len: uint32(clen)}
			if sum -= 32 >> clen; sum <= 0 {
				break
			}
		}
	}
	codeCLens := codeCLensArr[:0] // Compact the array to have no holes
	for _, c := range codeCLensArr {
		if c.len > 0 {
			codeCLens = append(codeCLens, c)
		}
	}
	if len(codeCLens) < 1 {
		panic(ErrCorrupt)
	}
	br.prefix.Init(codeCLens, true) // Must have 1..len(complexLens) symbols

	// Use code-lengths table to decode rest of prefix table.
	var codesArr [maxNumAlphabetSyms]prefixCode
	var sym, repSymLast, repCntLast, clenLast uint = 0, 0, 0, 8
	codes := codesArr[:0]
	for sym, sum = 0, 32768; sym < maxSyms && sum > 0; {
		clen := br.ReadSymbol(&br.prefix)
		if clen < 16 {
			// Literal bit-length symbol used.
			if clen > 0 {
				codes = append(codes, prefixCode{sym: uint32(sym), len: uint32(clen)})
				clenLast = clen
				sum -= 32768 >> clen
			}
			repSymLast = 0 // Reset last repeater symbol
			sym++
		} else {
			// Repeater symbol used.
			//	16: Repeat previous non-zero code-length
			//	17: Repeat code length of zero

			repSym := clen // Rename clen for better clarity
			if repSym != repSymLast {
				repCntLast = 0
				repSymLast = repSym
			}

			nb := repSym - 14          // 2..3 bits
			rep := br.ReadBits(nb) + 3 // 3..6 or 3..10
			if repCntLast > 0 {
				rep += (repCntLast - 2) << nb // Modify previous repeat count
			}
			repDiff := rep - repCntLast // Always positive
			repCntLast = rep

			if repSym == 16 {
				clen := clenLast
				for symEnd := sym + repDiff; sym < symEnd; sym++ {
					codes = append(codes, prefixCode{sym: uint32(sym), len: uint32(clen)})
				}
				sum -= int(repDiff) * (32768 >> clen)
			} else {
				sym += repDiff
			}
		}
	}
	if len(codes) < 2 || sym > maxSyms {
		panic(ErrCorrupt)
	}
	pd.Init(codes, true) // Must have 2..maxSyms symbols
}
<|endoftext|>


<commit_before>package main

const Version = "1.0.0"
<commit_msg>update version<commit_after>package main

const Version = "1.1.0"
<|endoftext|>