<commit_before>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"syscall"
	"time"
	"zabbix"
)

var (
	argClientCount = flag.Int("client", 200, "number of concurrent clients")
	argThreadCount = flag.Int("threads", runtime.NumCPU()*4, "number of threads")
	argClinetName  = flag.String("client-format", "client-%d", "format of client name")
	argPacketSize  = flag.Int("packet-size", 400, "count of metric in packet")
	argMetricName  = flag.String("metric-format", "metric-%d", "format of metric name in packet")
	argPacketDelay = flag.Duration("packet-delay", 100*time.Millisecond, "delay of send packet")
	argSendTimeout = flag.Duration("packet-send-timeout", 10*time.Millisecond, "packet send timeout")
	argZabbix      = flag.String("zabbix", "127.0.0.1:10051", "address of zabbix server")
	argMaxMetrics  = flag.Int("max-metrics", 0, "max number of metrics each client sends")
	argMaxSeconds  = flag.Int("max-time", 0, "max duration of benchmark test in seconds")

	errorChannel     = make(chan error, 10)
	completedChannel = make(chan int, 10)
	signalChannel    = make(chan os.Signal, 1)

	mutex               = &sync.Mutex{}
	counter, total, sec = 0, 0, 1
)

func main() {

	signal.Notify(signalChannel, os.Interrupt)
	signal.Notify(signalChannel, syscall.SIGTERM)

	if !flag.Parsed() {
		flag.Parse()
	}

	runtime.GOMAXPROCS(*argThreadCount)
	for i := 0; i < *argClientCount; i++ {
		go StartClient(i)
	}

	report_and_exit := func() {
		speed := 0
		if sec > 0 {
			speed = int(total / sec)
		}
		fmt.Printf("\n-----------------------------\n")
		fmt.Printf("Total processed: %d (%d metric/s)\n", total, speed)
		os.Exit(0)
	}

	fmt.Printf("Start %d clients with packet size %d metric and delay between packets %v\n", *argClientCount, *argPacketSize, *argPacketDelay)
	ticker := time.Tick(time.Second)
	for {
		select {
		case <-ticker:
			mutex.Lock()
			fmt.Printf("progress %d s, %d metric/s\n", sec, counter)
			sec += 1
			counter = 0
			if *argMaxMetrics > 0 && total > *argMaxMetrics {
				report_and_exit()
			}
			if *argMaxSeconds > 0 && sec > *argMaxSeconds {
				report_and_exit()
			}
			mutex.Unlock()
		case count := <-completedChannel:
			mutex.Lock()
			counter += count
			total += count
			mutex.Unlock()
		case err := <-errorChannel:
			fmt.Fprintf(os.Stderr, "Error write metric:\t%s\n", err.Error())
		case <-signalChannel:
			report_and_exit()
		}
	}

}

// client of zabbix server
type client struct {
	id     int
	host   string
	sender *zabbix.Sender
}

// generate and send zabbix packet
func (c *client) send() error {
	now := time.Now().Unix()
	metrics := make([]*zabbix.Metric, 0)
	for i := 0; i < *argPacketSize; i++ {
		metrics = append(metrics, zabbix.NewMetric(c.host, fmt.Sprintf(*argMetricName, i), fmt.Sprintf("%d", i), now))
	}
	return c.sender.Send(zabbix.NewPacket(metrics, now))
}

func StartClient(id int) {
	time.Sleep((time.Duration(rand.Float64()*100) * *argPacketDelay) / 100)
	c := &client{
		id:     id,
		host:   fmt.Sprintf(*argClinetName, id),
		sender: zabbix.NewSender(*argZabbix),
	}
	ticker := time.Tick(*argPacketDelay)
	for {
		select {
		case <-ticker:
			if err := c.send(); err != nil {
				errorChannel <- err
			} else {
				completedChannel <- *argPacketSize
			}
		}
	}
}
<commit_msg>duration in seconds<commit_after>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"syscall"
	"time"
	"zabbix"
)

var (
	argClientCount = flag.Int("client", 200, "number of concurrent clients")
	argThreadCount = flag.Int("threads", runtime.NumCPU()*4, "number of threads")
	argClinetName  = flag.String("client-format", "client-%d", "format of client name")
	argPacketSize  = flag.Int("packet-size", 400, "count of metric in packet")
	argMetricName  = flag.String("metric-format", "metric-%d", "format of metric name in packet")
	argPacketDelay = flag.Duration("packet-delay", 100*time.Millisecond, "delay of send packet")
	argSendTimeout = flag.Duration("packet-send-timeout", 10*time.Millisecond, "packet send timeout")
	argZabbix      = flag.String("zabbix", "127.0.0.1:10051", "address of zabbix server")
	argMaxMetrics  = flag.Int("max-metrics", 0, "max number of metrics each client sends")
	argMaxDuration = flag.Duration("max-duration", 0, "max duration of benchmark test")

	errorChannel     = make(chan error, 10)
	completedChannel = make(chan int, 10)
	signalChannel    = make(chan os.Signal, 1)

	mutex               = &sync.Mutex{}
	counter, total, sec = 0, 0, 1
)

func main() {

	signal.Notify(signalChannel, os.Interrupt)
	signal.Notify(signalChannel, syscall.SIGTERM)

	if !flag.Parsed() {
		flag.Parse()
	}

	runtime.GOMAXPROCS(*argThreadCount)
	for i := 0; i < *argClientCount; i++ {
		go StartClient(i)
	}

	report_and_exit := func() {
		speed := 0
		if sec > 0 {
			speed = int(total / sec)
		}
		fmt.Printf("\n-----------------------------\n")
		fmt.Printf("Total processed: %d (%d metric/s)\n", total, speed)
		os.Exit(0)
	}

	fmt.Printf("Start %d clients with packet size %d metric and delay between packets %v\n", *argClientCount, *argPacketSize, *argPacketDelay)
	ticker := time.Tick(time.Second)
	for {
		select {
		case <-ticker:
			mutex.Lock()
			fmt.Printf("progress %d s, %d metric/s\n", sec, counter)
			sec += 1
			counter = 0
			if *argMaxMetrics > 0 && total > *argMaxMetrics {
				report_and_exit()
			}
			if *argMaxDuration > 0 && time.Duration(sec)*time.Second > *argMaxDuration {
				report_and_exit()
			}
			mutex.Unlock()
		case count := <-completedChannel:
			mutex.Lock()
			counter += count
			total += count
			mutex.Unlock()
		case err := <-errorChannel:
			fmt.Fprintf(os.Stderr, "Error write metric:\t%s\n", err.Error())
		case <-signalChannel:
			report_and_exit()
		}
	}

}

// client of zabbix server
type client struct {
	id     int
	host   string
	sender *zabbix.Sender
}

// generate and send zabbix packet
func (c *client) send() error {
	now := time.Now().Unix()
	metrics := make([]*zabbix.Metric, 0)
	for i := 0; i < *argPacketSize; i++ {
		metrics = append(metrics, zabbix.NewMetric(c.host, fmt.Sprintf(*argMetricName, i), fmt.Sprintf("%d", i), now))
	}
	return c.sender.Send(zabbix.NewPacket(metrics, now))
}

func StartClient(id int) {
	time.Sleep((time.Duration(rand.Float64()*100) * *argPacketDelay) / 100)
	c := &client{
		id:     id,
		host:   fmt.Sprintf(*argClinetName, id),
		sender: zabbix.NewSender(*argZabbix),
	}
	ticker := time.Tick(*argPacketDelay)
	for {
		select {
		case <-ticker:
			if err := c.send(); err != nil {
				errorChannel <- err
			} else {
				completedChannel <- *argPacketSize
			}
		}
	}
}
<|endoftext|>


<commit_before>// main.go
package main

import (
	"fmt"
	"log"
	"os/exec"
	"strconv"
	"time"
)

const (
	SLEEP_TIME          = 3 * time.Second
	KEYBOARD_BUFER_SIZE = 10000
)

func main() {

	keyboard_id := 14
	cmd := exec.Command("xinput", "test", strconv.Itoa(keyboard_id))

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	buf := make([]byte, KEYBOARD_BUFER_SIZE)
	for {
		n, err := stdout.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		// processing buf here
		fmt.Println(len(buf), n)
		time.Sleep(5 * time.Second)
	}
}
<commit_msg>Added sleep constant<commit_after>// main.go
package main

import (
	"fmt"
	"log"
	"os/exec"
	"strconv"
	"time"
)

const (
	SLEEP_TIME          = 3 * time.Second
	KEYBOARD_BUFER_SIZE = 10000
)

func main() {

	keyboard_id := 14
	cmd := exec.Command("xinput", "test", strconv.Itoa(keyboard_id))

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	buf := make([]byte, KEYBOARD_BUFER_SIZE)
	for {
		n, err := stdout.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		// processing buf here
		fmt.Println(n)
		time.Sleep(SLEEP_TIME)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
)

func main() {
    fmt.Println("abc")
    fmt.Println("efg")
    fmt.Println("hij")
}
<commit_msg>增加klm<commit_after>package main

import (
    "fmt"
)

func main() {
    fmt.Println("abc")
    fmt.Println("efg")
    fmt.Println("hij")
    fmt.Println("klm")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/elazarl/goproxy"
)

func isBaidu() goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		log.Print("Host:" + req.URL.Host)
		log.Print("path:" + req.URL.Path)
		log.Print("schema:" + req.URL.Scheme)
		return strings.Contains(req.URL.Host, "oms.com")
	}
}

func main() {
	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = false
	proxy.OnRequest(isBaidu()).DoFunc(func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		log.Print("url:" + r.URL.String())
		res, err := http.Get("http://127.0.0.1:8080" + r.URL.Path)
		if err != nil {
			fmt.Println(err.Error())
		}
		return r, res
	})
	log.Fatal(http.ListenAndServe(":8081", proxy))
}
<commit_msg>update<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"strings"

	"golang.org/x/sys/windows/registry"

	"github.com/elazarl/goproxy"
)

func isBaidu() goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		log.Print("Host:" + req.URL.Host)
		log.Print("path:" + req.URL.Path)
		log.Print("schema:" + req.URL.Scheme)
		return strings.Contains(req.URL.Host, "oms.com")
	}
}

func updateSystemProxySetting() {
	k, err := registry.OpenKey(registry.CURRENT_USER, `Software\Microsoft\Windows\CurrentVersion\Internet Settings`, registry.ALL_ACCESS)
	if err != nil {
		log.Fatal(err)
	}
	defer k.Close()

	k.SetStringValue("ProxyServer", "127.0.0.1:720")
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	updateSystemProxySetting()
	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = false
	proxy.OnRequest(isBaidu()).DoFunc(func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		log.Print("url:" + r.URL.String())
		res, err := http.Get("http://127.0.0.1:8080" + r.URL.Path)
		if err != nil {
			fmt.Println(err.Error())
		}
		return r, res
	})
	log.Fatal(http.ListenAndServe(":720", proxy))
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bzip2

import "sort"

// A huffmanTree is a binary tree which is navigated, bit-by-bit to reach a
// symbol.
type huffmanTree struct {
	// nodes contains all the non-leaf nodes in the tree. nodes[0] is the
	// root of the tree and nextNode contains the index of the next element
	// of nodes to use when the tree is being constructed.
	nodes    []huffmanNode
	nextNode int
}

// A huffmanNode is a node in the tree. left and right contain indexes into the
// nodes slice of the tree. If left or right is invalidNodeValue then the child
// is a left node and its value is in leftValue/rightValue.
//
// The symbols are uint16s because bzip2 encodes not only MTF indexes in the
// tree, but also two magic values for run-length encoding and an EOF symbol.
// Thus there are more than 256 possible symbols.
type huffmanNode struct {
	left, right           uint16
	leftValue, rightValue uint16
}

// invalidNodeValue is an invalid index which marks a leaf node in the tree.
const invalidNodeValue = 0xffff

// Decode reads bits from the given bitReader and navigates the tree until a
// symbol is found.
func (t *huffmanTree) Decode(br *bitReader) (v uint16) {
	nodeIndex := uint16(0) // node 0 is the root of the tree.

	for {
		node := &t.nodes[nodeIndex]
		bit, ok := br.TryReadBit()
		if !ok && br.ReadBit() {
			bit = 1
		}
		// bzip2 encodes left as a true bit.
		if bit != 0 {
			// left
			if node.left == invalidNodeValue {
				return node.leftValue
			}
			nodeIndex = node.left
		} else {
			// right
			if node.right == invalidNodeValue {
				return node.rightValue
			}
			nodeIndex = node.right
		}
	}
}

// newHuffmanTree builds a Huffman tree from a slice containing the code
// lengths of each symbol. The maximum code length is 32 bits.
func newHuffmanTree(lengths []uint8) (huffmanTree, error) {
	// There are many possible trees that assign the same code length to
	// each symbol (consider reflecting a tree down the middle, for
	// example). Since the code length assignments determine the
	// efficiency of the tree, each of these trees is equally good. In
	// order to minimize the amount of information needed to build a tree
	// bzip2 uses a canonical tree so that it can be reconstructed given
	// only the code length assignments.

	if len(lengths) < 2 {
		panic("newHuffmanTree: too few symbols")
	}

	var t huffmanTree

	// First we sort the code length assignments by ascending code length,
	// using the symbol value to break ties.
	pairs := huffmanSymbolLengthPairs(make([]huffmanSymbolLengthPair, len(lengths)))
	for i, length := range lengths {
		pairs[i].value = uint16(i)
		pairs[i].length = length
	}

	sort.Sort(pairs)

	// Now we assign codes to the symbols, starting with the longest code.
	// We keep the codes packed into a uint32, at the most-significant end.
	// So branches are taken from the MSB downwards. This makes it easy to
	// sort them later.
	code := uint32(0)
	length := uint8(32)

	codes := huffmanCodes(make([]huffmanCode, len(lengths)))
	for i := len(pairs) - 1; i >= 0; i-- {
		if length > pairs[i].length {
			// If the code length decreases we shift in order to
			// zero any bits beyond the end of the code.
			length >>= 32 - pairs[i].length
			length <<= 32 - pairs[i].length
			length = pairs[i].length
		}
		codes[i].code = code
		codes[i].codeLen = length
		codes[i].value = pairs[i].value
		// We need to 'increment' the code, which means treating |code|
		// like a |length| bit number.
		code += 1 << (32 - length)
	}

	// Now we can sort by the code so that the left half of each branch are
	// grouped together, recursively.
	sort.Sort(codes)

	t.nodes = make([]huffmanNode, len(codes))
	_, err := buildHuffmanNode(&t, codes, 0)
	return t, err
}

// huffmanSymbolLengthPair contains a symbol and its code length.
type huffmanSymbolLengthPair struct {
	value  uint16
	length uint8
}

// huffmanSymbolLengthPair is used to provide an interface for sorting.
type huffmanSymbolLengthPairs []huffmanSymbolLengthPair

func (h huffmanSymbolLengthPairs) Len() int {
	return len(h)
}

func (h huffmanSymbolLengthPairs) Less(i, j int) bool {
	if h[i].length < h[j].length {
		return true
	}
	if h[i].length > h[j].length {
		return false
	}
	if h[i].value < h[j].value {
		return true
	}
	return false
}

func (h huffmanSymbolLengthPairs) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}

// huffmanCode contains a symbol, its code and code length.
type huffmanCode struct {
	code    uint32
	codeLen uint8
	value   uint16
}

// huffmanCodes is used to provide an interface for sorting.
type huffmanCodes []huffmanCode

func (n huffmanCodes) Len() int {
	return len(n)
}

func (n huffmanCodes) Less(i, j int) bool {
	return n[i].code < n[j].code
}

func (n huffmanCodes) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

// buildHuffmanNode takes a slice of sorted huffmanCodes and builds a node in
// the Huffman tree at the given level. It returns the index of the newly
// constructed node.
func buildHuffmanNode(t *huffmanTree, codes []huffmanCode, level uint32) (nodeIndex uint16, err error) {
	test := uint32(1) << (31 - level)

	// We have to search the list of codes to find the divide between the left and right sides.
	firstRightIndex := len(codes)
	for i, code := range codes {
		if code.code&test != 0 {
			firstRightIndex = i
			break
		}
	}

	left := codes[:firstRightIndex]
	right := codes[firstRightIndex:]

	if len(left) == 0 || len(right) == 0 {
		return 0, StructuralError("superfluous level in Huffman tree")
	}

	nodeIndex = uint16(t.nextNode)
	node := &t.nodes[t.nextNode]
	t.nextNode++

	if len(left) == 1 {
		// leaf node
		node.left = invalidNodeValue
		node.leftValue = left[0].value
	} else {
		node.left, err = buildHuffmanNode(t, left, level+1)
	}

	if err != nil {
		return
	}

	if len(right) == 1 {
		// leaf node
		node.right = invalidNodeValue
		node.rightValue = right[0].value
	} else {
		node.right, err = buildHuffmanNode(t, right, level+1)
	}

	return
}
<commit_msg>compress/bzip2: support superfluous Huffman levels.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bzip2

import "sort"

// A huffmanTree is a binary tree which is navigated, bit-by-bit to reach a
// symbol.
type huffmanTree struct {
	// nodes contains all the non-leaf nodes in the tree. nodes[0] is the
	// root of the tree and nextNode contains the index of the next element
	// of nodes to use when the tree is being constructed.
	nodes    []huffmanNode
	nextNode int
}

// A huffmanNode is a node in the tree. left and right contain indexes into the
// nodes slice of the tree. If left or right is invalidNodeValue then the child
// is a left node and its value is in leftValue/rightValue.
//
// The symbols are uint16s because bzip2 encodes not only MTF indexes in the
// tree, but also two magic values for run-length encoding and an EOF symbol.
// Thus there are more than 256 possible symbols.
type huffmanNode struct {
	left, right           uint16
	leftValue, rightValue uint16
}

// invalidNodeValue is an invalid index which marks a leaf node in the tree.
const invalidNodeValue = 0xffff

// Decode reads bits from the given bitReader and navigates the tree until a
// symbol is found.
func (t *huffmanTree) Decode(br *bitReader) (v uint16) {
	nodeIndex := uint16(0) // node 0 is the root of the tree.

	for {
		node := &t.nodes[nodeIndex]
		bit, ok := br.TryReadBit()
		if !ok && br.ReadBit() {
			bit = 1
		}
		// bzip2 encodes left as a true bit.
		if bit != 0 {
			// left
			if node.left == invalidNodeValue {
				return node.leftValue
			}
			nodeIndex = node.left
		} else {
			// right
			if node.right == invalidNodeValue {
				return node.rightValue
			}
			nodeIndex = node.right
		}
	}
}

// newHuffmanTree builds a Huffman tree from a slice containing the code
// lengths of each symbol. The maximum code length is 32 bits.
func newHuffmanTree(lengths []uint8) (huffmanTree, error) {
	// There are many possible trees that assign the same code length to
	// each symbol (consider reflecting a tree down the middle, for
	// example). Since the code length assignments determine the
	// efficiency of the tree, each of these trees is equally good. In
	// order to minimize the amount of information needed to build a tree
	// bzip2 uses a canonical tree so that it can be reconstructed given
	// only the code length assignments.

	if len(lengths) < 2 {
		panic("newHuffmanTree: too few symbols")
	}

	var t huffmanTree

	// First we sort the code length assignments by ascending code length,
	// using the symbol value to break ties.
	pairs := huffmanSymbolLengthPairs(make([]huffmanSymbolLengthPair, len(lengths)))
	for i, length := range lengths {
		pairs[i].value = uint16(i)
		pairs[i].length = length
	}

	sort.Sort(pairs)

	// Now we assign codes to the symbols, starting with the longest code.
	// We keep the codes packed into a uint32, at the most-significant end.
	// So branches are taken from the MSB downwards. This makes it easy to
	// sort them later.
	code := uint32(0)
	length := uint8(32)

	codes := huffmanCodes(make([]huffmanCode, len(lengths)))
	for i := len(pairs) - 1; i >= 0; i-- {
		if length > pairs[i].length {
			// If the code length decreases we shift in order to
			// zero any bits beyond the end of the code.
			length >>= 32 - pairs[i].length
			length <<= 32 - pairs[i].length
			length = pairs[i].length
		}
		codes[i].code = code
		codes[i].codeLen = length
		codes[i].value = pairs[i].value
		// We need to 'increment' the code, which means treating |code|
		// like a |length| bit number.
		code += 1 << (32 - length)
	}

	// Now we can sort by the code so that the left half of each branch are
	// grouped together, recursively.
	sort.Sort(codes)

	t.nodes = make([]huffmanNode, len(codes))
	_, err := buildHuffmanNode(&t, codes, 0)
	return t, err
}

// huffmanSymbolLengthPair contains a symbol and its code length.
type huffmanSymbolLengthPair struct {
	value  uint16
	length uint8
}

// huffmanSymbolLengthPair is used to provide an interface for sorting.
type huffmanSymbolLengthPairs []huffmanSymbolLengthPair

func (h huffmanSymbolLengthPairs) Len() int {
	return len(h)
}

func (h huffmanSymbolLengthPairs) Less(i, j int) bool {
	if h[i].length < h[j].length {
		return true
	}
	if h[i].length > h[j].length {
		return false
	}
	if h[i].value < h[j].value {
		return true
	}
	return false
}

func (h huffmanSymbolLengthPairs) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}

// huffmanCode contains a symbol, its code and code length.
type huffmanCode struct {
	code    uint32
	codeLen uint8
	value   uint16
}

// huffmanCodes is used to provide an interface for sorting.
type huffmanCodes []huffmanCode

func (n huffmanCodes) Len() int {
	return len(n)
}

func (n huffmanCodes) Less(i, j int) bool {
	return n[i].code < n[j].code
}

func (n huffmanCodes) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

// buildHuffmanNode takes a slice of sorted huffmanCodes and builds a node in
// the Huffman tree at the given level. It returns the index of the newly
// constructed node.
func buildHuffmanNode(t *huffmanTree, codes []huffmanCode, level uint32) (nodeIndex uint16, err error) {
	test := uint32(1) << (31 - level)

	// We have to search the list of codes to find the divide between the left and right sides.
	firstRightIndex := len(codes)
	for i, code := range codes {
		if code.code&test != 0 {
			firstRightIndex = i
			break
		}
	}

	left := codes[:firstRightIndex]
	right := codes[firstRightIndex:]

	if len(left) == 0 || len(right) == 0 {
		// There is a superfluous level in the Huffman tree indicating
		// a bug in the encoder. However, this bug has been observed in
		// the wild so we handle it.

		// If this function was called recursively then we know that
		// len(codes) >= 2 because, otherwise, we would have hit the
		// "leaf node" case, below, and not recursed.
		//
		// However, for the initial call it's possible that len(codes)
		// is zero or one. Both cases are invalid because a zero length
		// tree cannot encode anything and a length-1 tree can only
		// encode EOF and so is superfluous. We reject both.
		if len(codes) < 2 {
			return 0, StructuralError("empty Huffman tree")
		}

		// In this case the recursion doesn't always reduce the length
		// of codes so we need to ensure termination via another
		// mechanism.
		if level == 31 {
			// Since len(codes) >= 2 the only way that the values
			// can match at all 32 bits is if they are equal, which
			// is invalid. This ensures that we never enter
			// infinite recursion.
			return 0, StructuralError("equal symbols in Huffman tree")
		}

		if len(left) == 0 {
			return buildHuffmanNode(t, right, level+1)
		}
		return buildHuffmanNode(t, left, level+1)
	}

	nodeIndex = uint16(t.nextNode)
	node := &t.nodes[t.nextNode]
	t.nextNode++

	if len(left) == 1 {
		// leaf node
		node.left = invalidNodeValue
		node.leftValue = left[0].value
	} else {
		node.left, err = buildHuffmanNode(t, left, level+1)
	}

	if err != nil {
		return
	}

	if len(right) == 1 {
		// leaf node
		node.right = invalidNodeValue
		node.rightValue = right[0].value
	} else {
		node.right, err = buildHuffmanNode(t, right, level+1)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements translation between
// unsigned integer values and byte sequences.
package binary

import (
	"math"
	"io"
	"os"
	"reflect"
)

// A ByteOrder specifies how to convert byte sequences into
// 16-, 32-, or 64-bit unsigned integers.
type ByteOrder interface {
	Uint16(b []byte) uint16
	Uint32(b []byte) uint32
	Uint64(b []byte) uint64
	PutUint16([]byte, uint16)
	PutUint32([]byte, uint32)
	PutUint64([]byte, uint64)
	String() string
}

// This is byte instead of struct{} so that it can be compared,
// allowing, e.g., order == binary.LittleEndian.
type unused byte

var LittleEndian ByteOrder = littleEndian(0)
var BigEndian ByteOrder = bigEndian(0)

type littleEndian unused

func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 }

func (littleEndian) PutUint16(b []byte, v uint16) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
}

func (littleEndian) Uint32(b []byte) uint32 {
	return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}

func (littleEndian) PutUint32(b []byte, v uint32) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
	b[2] = byte(v >> 16)
	b[3] = byte(v >> 24)
}

func (littleEndian) Uint64(b []byte) uint64 {
	return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
		uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}

func (littleEndian) PutUint64(b []byte, v uint64) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
	b[2] = byte(v >> 16)
	b[3] = byte(v >> 24)
	b[4] = byte(v >> 32)
	b[5] = byte(v >> 40)
	b[6] = byte(v >> 48)
	b[7] = byte(v >> 56)
}

func (littleEndian) String() string { return "LittleEndian" }

func (littleEndian) GoString() string { return "binary.LittleEndian" }

type bigEndian unused

func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 }

func (bigEndian) PutUint16(b []byte, v uint16) {
	b[0] = byte(v >> 8)
	b[1] = byte(v)
}

func (bigEndian) Uint32(b []byte) uint32 {
	return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
}

func (bigEndian) PutUint32(b []byte, v uint32) {
	b[0] = byte(v >> 24)
	b[1] = byte(v >> 16)
	b[2] = byte(v >> 8)
	b[3] = byte(v)
}

func (bigEndian) Uint64(b []byte) uint64 {
	return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 |
		uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56
}

func (bigEndian) PutUint64(b []byte, v uint64) {
	b[0] = byte(v >> 56)
	b[1] = byte(v >> 48)
	b[2] = byte(v >> 40)
	b[3] = byte(v >> 32)
	b[4] = byte(v >> 24)
	b[5] = byte(v >> 16)
	b[6] = byte(v >> 8)
	b[7] = byte(v)
}

func (bigEndian) String() string { return "BigEndian" }

func (bigEndian) GoString() string { return "binary.BigEndian" }

// Read reads structured binary data from r into data.
// Data must be a pointer to a fixed-size value or a slice
// of fixed-size values.
// A fixed-size value is either a fixed-size integer
// (int8, uint8, int16, uint16, ...) or an array or struct
// containing only fixed-size values.  Bytes read from
// r are decoded using the specified byte order and written
// to successive fields of the data.
func Read(r io.Reader, order ByteOrder, data interface{}) os.Error {
	var v reflect.Value
	switch d := reflect.NewValue(data).(type) {
	case *reflect.PtrValue:
		v = d.Elem()
	case *reflect.SliceValue:
		v = d
	default:
		return os.NewError("binary.Read: invalid type " + v.Type().String())
	}
	size := TotalSize(v)
	if size < 0 {
		return os.NewError("binary.Read: invalid type " + v.Type().String())
	}
	d := &decoder{order: order, buf: make([]byte, size)}
	if _, err := io.ReadFull(r, d.buf); err != nil {
		return err
	}
	d.value(v)
	return nil
}

// Write writes the binary representation of data into w.
// Data must be a fixed-size value or a pointer to
// a fixed-size value.
// A fixed-size value is either a fixed-size integer
// (int8, uint8, int16, uint16, ...) or an array or struct
// containing only fixed-size values.  Bytes written to
// w are encoded using the specified byte order and read
// from successive fields of the data.
func Write(w io.Writer, order ByteOrder, data interface{}) os.Error {
	v := reflect.Indirect(reflect.NewValue(data))
	size := TotalSize(v)
	if size < 0 {
		return os.NewError("binary.Write: invalid type " + v.Type().String())
	}
	buf := make([]byte, size)
	e := &encoder{order: order, buf: buf}
	e.value(v)
	_, err := w.Write(buf)
	return err
}

func TotalSize(v reflect.Value) int {
	if sv, ok := v.(*reflect.SliceValue); ok {
		elem := sizeof(v.Type().(*reflect.SliceType).Elem())
		if elem < 0 {
			return -1
		}
		return sv.Len() * elem
	}
	return sizeof(v.Type())
}

func sizeof(v reflect.Type) int {
	switch t := v.(type) {
	case *reflect.ArrayType:
		n := sizeof(t.Elem())
		if n < 0 {
			return -1
		}
		return t.Len() * n

	case *reflect.StructType:
		sum := 0
		for i, n := 0, t.NumField(); i < n; i++ {
			s := sizeof(t.Field(i).Type)
			if s < 0 {
				return -1
			}
			sum += s
		}
		return sum

	case *reflect.Uint8Type:
		return 1
	case *reflect.Uint16Type:
		return 2
	case *reflect.Uint32Type:
		return 4
	case *reflect.Uint64Type:
		return 8
	case *reflect.Int8Type:
		return 1
	case *reflect.Int16Type:
		return 2
	case *reflect.Int32Type:
		return 4
	case *reflect.Int64Type:
		return 8
	case *reflect.Float32Type:
		return 4
	case *reflect.Float64Type:
		return 8
	}
	return -1
}

type decoder struct {
	order ByteOrder
	buf   []byte
}

type encoder struct {
	order ByteOrder
	buf   []byte
}

func (d *decoder) uint8() uint8 {
	x := d.buf[0]
	d.buf = d.buf[1:]
	return x
}

func (e *encoder) uint8(x uint8) {
	e.buf[0] = x
	e.buf = e.buf[1:]
}

func (d *decoder) uint16() uint16 {
	x := d.order.Uint16(d.buf[0:2])
	d.buf = d.buf[2:]
	return x
}

func (e *encoder) uint16(x uint16) {
	e.order.PutUint16(e.buf[0:2], x)
	e.buf = e.buf[2:]
}

func (d *decoder) uint32() uint32 {
	x := d.order.Uint32(d.buf[0:4])
	d.buf = d.buf[4:]
	return x
}

func (e *encoder) uint32(x uint32) {
	e.order.PutUint32(e.buf[0:4], x)
	e.buf = e.buf[4:]
}

func (d *decoder) uint64() uint64 {
	x := d.order.Uint64(d.buf[0:8])
	d.buf = d.buf[8:]
	return x
}

func (e *encoder) uint64(x uint64) {
	e.order.PutUint64(e.buf[0:8], x)
	e.buf = e.buf[8:]
}

func (d *decoder) int8() int8 { return int8(d.uint8()) }

func (e *encoder) int8(x int8) { e.uint8(uint8(x)) }

func (d *decoder) int16() int16 { return int16(d.uint16()) }

func (e *encoder) int16(x int16) { e.uint16(uint16(x)) }

func (d *decoder) int32() int32 { return int32(d.uint32()) }

func (e *encoder) int32(x int32) { e.uint32(uint32(x)) }

func (d *decoder) int64() int64 { return int64(d.uint64()) }

func (e *encoder) int64(x int64) { e.uint64(uint64(x)) }

func (d *decoder) value(v reflect.Value) {
	switch v := v.(type) {
	case *reflect.ArrayValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			d.value(v.Elem(i))
		}
	case *reflect.StructValue:
		l := v.NumField()
		for i := 0; i < l; i++ {
			d.value(v.Field(i))
		}

	case *reflect.SliceValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			d.value(v.Elem(i))
		}

	case *reflect.Uint8Value:
		v.Set(d.uint8())
	case *reflect.Uint16Value:
		v.Set(d.uint16())
	case *reflect.Uint32Value:
		v.Set(d.uint32())
	case *reflect.Uint64Value:
		v.Set(d.uint64())
	case *reflect.Int8Value:
		v.Set(d.int8())
	case *reflect.Int16Value:
		v.Set(d.int16())
	case *reflect.Int32Value:
		v.Set(d.int32())
	case *reflect.Int64Value:
		v.Set(d.int64())
	case *reflect.Float32Value:
		v.Set(math.Float32frombits(d.uint32()))
	case *reflect.Float64Value:
		v.Set(math.Float64frombits(d.uint64()))
	}
}

func (e *encoder) value(v reflect.Value) {
	switch v := v.(type) {
	case *reflect.ArrayValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			e.value(v.Elem(i))
		}
	case *reflect.StructValue:
		l := v.NumField()
		for i := 0; i < l; i++ {
			e.value(v.Field(i))
		}
	case *reflect.SliceValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			e.value(v.Elem(i))
		}

	case *reflect.Uint8Value:
		e.uint8(v.Get())
	case *reflect.Uint16Value:
		e.uint16(v.Get())
	case *reflect.Uint32Value:
		e.uint32(v.Get())
	case *reflect.Uint64Value:
		e.uint64(v.Get())
	case *reflect.Int8Value:
		e.int8(v.Get())
	case *reflect.Int16Value:
		e.int16(v.Get())
	case *reflect.Int32Value:
		e.int32(v.Get())
	case *reflect.Int64Value:
		e.int64(v.Get())
	case *reflect.Float32Value:
		e.uint32(math.Float32bits(v.Get()))
	case *reflect.Float64Value:
		e.uint64(math.Float64bits(v.Get()))
	}
}
<commit_msg>encoding/binary: fix error reporting bug<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements translation between
// unsigned integer values and byte sequences.
package binary

import (
	"math"
	"io"
	"os"
	"reflect"
)

// A ByteOrder specifies how to convert byte sequences into
// 16-, 32-, or 64-bit unsigned integers.
type ByteOrder interface {
	Uint16(b []byte) uint16
	Uint32(b []byte) uint32
	Uint64(b []byte) uint64
	PutUint16([]byte, uint16)
	PutUint32([]byte, uint32)
	PutUint64([]byte, uint64)
	String() string
}

// This is byte instead of struct{} so that it can be compared,
// allowing, e.g., order == binary.LittleEndian.
type unused byte

var LittleEndian ByteOrder = littleEndian(0)
var BigEndian ByteOrder = bigEndian(0)

type littleEndian unused

func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 }

func (littleEndian) PutUint16(b []byte, v uint16) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
}

func (littleEndian) Uint32(b []byte) uint32 {
	return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}

func (littleEndian) PutUint32(b []byte, v uint32) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
	b[2] = byte(v >> 16)
	b[3] = byte(v >> 24)
}

func (littleEndian) Uint64(b []byte) uint64 {
	return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
		uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}

func (littleEndian) PutUint64(b []byte, v uint64) {
	b[0] = byte(v)
	b[1] = byte(v >> 8)
	b[2] = byte(v >> 16)
	b[3] = byte(v >> 24)
	b[4] = byte(v >> 32)
	b[5] = byte(v >> 40)
	b[6] = byte(v >> 48)
	b[7] = byte(v >> 56)
}

func (littleEndian) String() string { return "LittleEndian" }

func (littleEndian) GoString() string { return "binary.LittleEndian" }

type bigEndian unused

func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 }

func (bigEndian) PutUint16(b []byte, v uint16) {
	b[0] = byte(v >> 8)
	b[1] = byte(v)
}

func (bigEndian) Uint32(b []byte) uint32 {
	return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
}

func (bigEndian) PutUint32(b []byte, v uint32) {
	b[0] = byte(v >> 24)
	b[1] = byte(v >> 16)
	b[2] = byte(v >> 8)
	b[3] = byte(v)
}

func (bigEndian) Uint64(b []byte) uint64 {
	return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 |
		uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56
}

func (bigEndian) PutUint64(b []byte, v uint64) {
	b[0] = byte(v >> 56)
	b[1] = byte(v >> 48)
	b[2] = byte(v >> 40)
	b[3] = byte(v >> 32)
	b[4] = byte(v >> 24)
	b[5] = byte(v >> 16)
	b[6] = byte(v >> 8)
	b[7] = byte(v)
}

func (bigEndian) String() string { return "BigEndian" }

func (bigEndian) GoString() string { return "binary.BigEndian" }

// Read reads structured binary data from r into data.
// Data must be a pointer to a fixed-size value or a slice
// of fixed-size values.
// A fixed-size value is either a fixed-size integer
// (int8, uint8, int16, uint16, ...) or an array or struct
// containing only fixed-size values.  Bytes read from
// r are decoded using the specified byte order and written
// to successive fields of the data.
func Read(r io.Reader, order ByteOrder, data interface{}) os.Error {
	var v reflect.Value
	switch d := reflect.NewValue(data).(type) {
	case *reflect.PtrValue:
		v = d.Elem()
	case *reflect.SliceValue:
		v = d
	default:
		return os.NewError("binary.Read: invalid type " + d.Type().String())
	}
	size := TotalSize(v)
	if size < 0 {
		return os.NewError("binary.Read: invalid type " + v.Type().String())
	}
	d := &decoder{order: order, buf: make([]byte, size)}
	if _, err := io.ReadFull(r, d.buf); err != nil {
		return err
	}
	d.value(v)
	return nil
}

// Write writes the binary representation of data into w.
// Data must be a fixed-size value or a pointer to
// a fixed-size value.
// A fixed-size value is either a fixed-size integer
// (int8, uint8, int16, uint16, ...) or an array or struct
// containing only fixed-size values.  Bytes written to
// w are encoded using the specified byte order and read
// from successive fields of the data.
func Write(w io.Writer, order ByteOrder, data interface{}) os.Error {
	v := reflect.Indirect(reflect.NewValue(data))
	size := TotalSize(v)
	if size < 0 {
		return os.NewError("binary.Write: invalid type " + v.Type().String())
	}
	buf := make([]byte, size)
	e := &encoder{order: order, buf: buf}
	e.value(v)
	_, err := w.Write(buf)
	return err
}

func TotalSize(v reflect.Value) int {
	if sv, ok := v.(*reflect.SliceValue); ok {
		elem := sizeof(v.Type().(*reflect.SliceType).Elem())
		if elem < 0 {
			return -1
		}
		return sv.Len() * elem
	}
	return sizeof(v.Type())
}

func sizeof(v reflect.Type) int {
	switch t := v.(type) {
	case *reflect.ArrayType:
		n := sizeof(t.Elem())
		if n < 0 {
			return -1
		}
		return t.Len() * n

	case *reflect.StructType:
		sum := 0
		for i, n := 0, t.NumField(); i < n; i++ {
			s := sizeof(t.Field(i).Type)
			if s < 0 {
				return -1
			}
			sum += s
		}
		return sum

	case *reflect.Uint8Type:
		return 1
	case *reflect.Uint16Type:
		return 2
	case *reflect.Uint32Type:
		return 4
	case *reflect.Uint64Type:
		return 8
	case *reflect.Int8Type:
		return 1
	case *reflect.Int16Type:
		return 2
	case *reflect.Int32Type:
		return 4
	case *reflect.Int64Type:
		return 8
	case *reflect.Float32Type:
		return 4
	case *reflect.Float64Type:
		return 8
	}
	return -1
}

type decoder struct {
	order ByteOrder
	buf   []byte
}

type encoder struct {
	order ByteOrder
	buf   []byte
}

func (d *decoder) uint8() uint8 {
	x := d.buf[0]
	d.buf = d.buf[1:]
	return x
}

func (e *encoder) uint8(x uint8) {
	e.buf[0] = x
	e.buf = e.buf[1:]
}

func (d *decoder) uint16() uint16 {
	x := d.order.Uint16(d.buf[0:2])
	d.buf = d.buf[2:]
	return x
}

func (e *encoder) uint16(x uint16) {
	e.order.PutUint16(e.buf[0:2], x)
	e.buf = e.buf[2:]
}

func (d *decoder) uint32() uint32 {
	x := d.order.Uint32(d.buf[0:4])
	d.buf = d.buf[4:]
	return x
}

func (e *encoder) uint32(x uint32) {
	e.order.PutUint32(e.buf[0:4], x)
	e.buf = e.buf[4:]
}

func (d *decoder) uint64() uint64 {
	x := d.order.Uint64(d.buf[0:8])
	d.buf = d.buf[8:]
	return x
}

func (e *encoder) uint64(x uint64) {
	e.order.PutUint64(e.buf[0:8], x)
	e.buf = e.buf[8:]
}

func (d *decoder) int8() int8 { return int8(d.uint8()) }

func (e *encoder) int8(x int8) { e.uint8(uint8(x)) }

func (d *decoder) int16() int16 { return int16(d.uint16()) }

func (e *encoder) int16(x int16) { e.uint16(uint16(x)) }

func (d *decoder) int32() int32 { return int32(d.uint32()) }

func (e *encoder) int32(x int32) { e.uint32(uint32(x)) }

func (d *decoder) int64() int64 { return int64(d.uint64()) }

func (e *encoder) int64(x int64) { e.uint64(uint64(x)) }

func (d *decoder) value(v reflect.Value) {
	switch v := v.(type) {
	case *reflect.ArrayValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			d.value(v.Elem(i))
		}
	case *reflect.StructValue:
		l := v.NumField()
		for i := 0; i < l; i++ {
			d.value(v.Field(i))
		}

	case *reflect.SliceValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			d.value(v.Elem(i))
		}

	case *reflect.Uint8Value:
		v.Set(d.uint8())
	case *reflect.Uint16Value:
		v.Set(d.uint16())
	case *reflect.Uint32Value:
		v.Set(d.uint32())
	case *reflect.Uint64Value:
		v.Set(d.uint64())
	case *reflect.Int8Value:
		v.Set(d.int8())
	case *reflect.Int16Value:
		v.Set(d.int16())
	case *reflect.Int32Value:
		v.Set(d.int32())
	case *reflect.Int64Value:
		v.Set(d.int64())
	case *reflect.Float32Value:
		v.Set(math.Float32frombits(d.uint32()))
	case *reflect.Float64Value:
		v.Set(math.Float64frombits(d.uint64()))
	}
}

func (e *encoder) value(v reflect.Value) {
	switch v := v.(type) {
	case *reflect.ArrayValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			e.value(v.Elem(i))
		}
	case *reflect.StructValue:
		l := v.NumField()
		for i := 0; i < l; i++ {
			e.value(v.Field(i))
		}
	case *reflect.SliceValue:
		l := v.Len()
		for i := 0; i < l; i++ {
			e.value(v.Elem(i))
		}

	case *reflect.Uint8Value:
		e.uint8(v.Get())
	case *reflect.Uint16Value:
		e.uint16(v.Get())
	case *reflect.Uint32Value:
		e.uint32(v.Get())
	case *reflect.Uint64Value:
		e.uint64(v.Get())
	case *reflect.Int8Value:
		e.int8(v.Get())
	case *reflect.Int16Value:
		e.int16(v.Get())
	case *reflect.Int32Value:
		e.int32(v.Get())
	case *reflect.Int64Value:
		e.int64(v.Get())
	case *reflect.Float32Value:
		e.uint32(math.Float32bits(v.Get()))
	case *reflect.Float64Value:
		e.uint64(math.Float64bits(v.Get()))
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"go/build"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

func runTest(t *testing.T, path string) {
	exitCode = 0

	*recursive = false
	if suffix := ".go"; strings.HasSuffix(path, suffix) {
		// single file
		path = filepath.Join(runtime.GOROOT(), "src/pkg", path)
		path, file := filepath.Split(path)
		*pkgName = file[:len(file)-len(suffix)]
		processFiles([]string{path}, true)
	} else {
		// package directory
		// TODO(gri) gotype should use the build package instead
		ctxt := build.Default
		ctxt.CgoEnabled = false
		pkg, err := ctxt.Import(path, "", 0)
		if err != nil {
			t.Errorf("build.Import error for path = %s: %s", path, err)
			return
		}
		// TODO(gri) there ought to be a more direct way using the build package...
		files := make([]string, len(pkg.GoFiles))
		for i, file := range pkg.GoFiles {
			files[i] = filepath.Join(pkg.Dir, file)
		}
		*pkgName = pkg.Name
		processFiles(files, true)
	}

	if exitCode != 0 {
		t.Errorf("processing %s failed: exitCode = %d", path, exitCode)
	}
}

var tests = []string{
	// individual files
	"exp/gotype/testdata/test1.go",

	// directories
	// Note: Packages that don't typecheck yet are commented out.
	// Unless there is a comment next to the commented out packages,
	// the package doesn't typecheck due to errors in the shift
	// expression checker.
	"archive/tar",
	"archive/zip",

	"bufio",
	"bytes",

	// "compress/bzip2",
	"compress/flate",
	"compress/gzip",
	// "compress/lzw",
	"compress/zlib",

	"container/heap",
	"container/list",
	"container/ring",

	"crypto",
	"crypto/aes",
	"crypto/cipher",
	"crypto/des",
	"crypto/dsa",
	"crypto/ecdsa",
	"crypto/elliptic",
	"crypto/hmac",
	"crypto/md5",
	// "crypto/rand",
	"crypto/rc4",
	// "crypto/rsa", // intermittent failure: /home/gri/go2/src/pkg/crypto/rsa/pkcs1v15.go:21:27: undeclared name: io
	"crypto/sha1",
	"crypto/sha256",
	"crypto/sha512",
	"crypto/subtle",
	"crypto/tls",
	"crypto/x509",
	"crypto/x509/pkix",

	"database/sql",
	"database/sql/driver",

	// "debug/dwarf",
	"debug/elf",
	"debug/gosym",
	"debug/macho",
	"debug/pe",

	"encoding/ascii85",
	// "encoding/asn1",
	"encoding/base32",
	"encoding/base64",
	"encoding/binary",
	"encoding/csv",
	"encoding/gob",
	"encoding/hex",
	"encoding/json",
	"encoding/pem",
	"encoding/xml",

	"errors",
	"expvar",
	"flag",
	"fmt",

	"exp/gotype",

	"go/ast",
	"go/build",
	"go/doc",
	"go/format",
	"go/parser",
	"go/printer",
	"go/scanner",
	// "go/token",
	"go/types",

	"hash/adler32",
	"hash/crc32",
	"hash/crc64",
	"hash/fnv",

	"image",
	"image/color",
	"image/draw",
	"image/gif",
	// "image/jpeg",
	"image/png",

	"index/suffixarray",

	"io",
	"io/ioutil",

	"log",
	"log/syslog",

	// "math",
	"math/big",
	"math/cmplx",
	"math/rand",

	"mime",
	"mime/multipart",

	// "net",
	"net/http",
	"net/http/cgi",
	"net/http/fcgi",
	"net/http/httptest",
	"net/http/httputil",
	"net/http/pprof",
	"net/mail",
	"net/rpc",
	"net/rpc/jsonrpc",
	"net/smtp",
	// "net/textproto",
	"net/url",

	"path",
	"path/filepath",

	"reflect",

	"regexp",
	"regexp/syntax",

	// "runtime",
	"runtime/cgo",
	"runtime/debug",
	"runtime/pprof",

	"sort",
	// "strconv",
	"strings",

	"sync",
	"sync/atomic",

	// "syscall",

	"testing",
	"testing/iotest",
	"testing/quick",

	// "text/scanner",
	"text/tabwriter",
	"text/template",
	"text/template/parse",

	"time",
	"unicode",
	"unicode/utf16",
	"unicode/utf8",
}

func Test(t *testing.T) {
	for _, test := range tests {
		runTest(t, test)
	}
}
<commit_msg>exp/gotype: fix build breakage due to https://code.google.com/p/go/source/detail?r=ca5e5de48173<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"go/build"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

func runTest(t *testing.T, path string) {
	exitCode = 0

	*recursive = false
	if suffix := ".go"; strings.HasSuffix(path, suffix) {
		// single file
		path = filepath.Join(runtime.GOROOT(), "src/pkg", path)
		path, file := filepath.Split(path)
		*pkgName = file[:len(file)-len(suffix)]
		processFiles([]string{path}, true)
	} else {
		// package directory
		// TODO(gri) gotype should use the build package instead
		ctxt := build.Default
		ctxt.CgoEnabled = false
		pkg, err := ctxt.Import(path, "", 0)
		if err != nil {
			t.Errorf("build.Import error for path = %s: %s", path, err)
			return
		}
		// TODO(gri) there ought to be a more direct way using the build package...
		files := make([]string, len(pkg.GoFiles))
		for i, file := range pkg.GoFiles {
			files[i] = filepath.Join(pkg.Dir, file)
		}
		*pkgName = pkg.Name
		processFiles(files, true)
	}

	if exitCode != 0 {
		t.Errorf("processing %s failed: exitCode = %d", path, exitCode)
	}
}

var tests = []string{
	// individual files
	"exp/gotype/testdata/test1.go",

	// directories
	// Note: Packages that don't typecheck yet are commented out.
	// Unless there is a comment next to the commented out packages,
	// the package doesn't typecheck due to errors in the shift
	// expression checker.
	"archive/tar",
	"archive/zip",

	"bufio",
	"bytes",

	// "compress/bzip2",
	"compress/flate",
	"compress/gzip",
	// "compress/lzw",
	"compress/zlib",

	"container/heap",
	"container/list",
	"container/ring",

	"crypto",
	"crypto/aes",
	"crypto/cipher",
	"crypto/des",
	"crypto/dsa",
	"crypto/ecdsa",
	"crypto/elliptic",
	"crypto/hmac",
	"crypto/md5",
	// "crypto/rand",
	"crypto/rc4",
	// "crypto/rsa", // intermittent failure: /home/gri/go2/src/pkg/crypto/rsa/pkcs1v15.go:21:27: undeclared name: io
	"crypto/sha1",
	"crypto/sha256",
	"crypto/sha512",
	"crypto/subtle",
	"crypto/tls",
	"crypto/x509",
	"crypto/x509/pkix",

	"database/sql",
	"database/sql/driver",

	// "debug/dwarf",
	"debug/elf",
	"debug/gosym",
	"debug/macho",
	"debug/pe",

	"encoding/ascii85",
	// "encoding/asn1",
	"encoding/base32",
	"encoding/base64",
	"encoding/binary",
	"encoding/csv",
	"encoding/gob",
	"encoding/hex",
	"encoding/json",
	"encoding/pem",
	"encoding/xml",

	"errors",
	"expvar",
	"flag",
	"fmt",

	"exp/gotype",

	"go/ast",
	"go/build",
	"go/doc",
	"go/format",
	"go/parser",
	"go/printer",
	"go/scanner",
	// "go/token",
	"go/types",

	"hash/adler32",
	"hash/crc32",
	"hash/crc64",
	"hash/fnv",

	"image",
	"image/color",
	"image/draw",
	"image/gif",
	// "image/jpeg",
	"image/png",

	"index/suffixarray",

	"io",
	"io/ioutil",

	"log",
	"log/syslog",

	// "math",
	//"math/big",
	"math/cmplx",
	"math/rand",

	"mime",
	"mime/multipart",

	// "net",
	"net/http",
	"net/http/cgi",
	"net/http/fcgi",
	"net/http/httptest",
	"net/http/httputil",
	"net/http/pprof",
	"net/mail",
	"net/rpc",
	"net/rpc/jsonrpc",
	"net/smtp",
	// "net/textproto",
	"net/url",

	"path",
	"path/filepath",

	"reflect",

	"regexp",
	"regexp/syntax",

	// "runtime",
	"runtime/cgo",
	"runtime/debug",
	"runtime/pprof",

	"sort",
	// "strconv",
	"strings",

	"sync",
	"sync/atomic",

	// "syscall",

	"testing",
	"testing/iotest",
	"testing/quick",

	// "text/scanner",
	"text/tabwriter",
	"text/template",
	"text/template/parse",

	"time",
	"unicode",
	"unicode/utf16",
	"unicode/utf8",
}

func Test(t *testing.T) {
	for _, test := range tests {
		runTest(t, test)
	}
}
<|endoftext|>


<commit_before>package local

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"restic/backend"
	"restic/debug"
)

// Local is a backend in a local directory.
type Local struct {
	p string
}

func paths(dir string) []string {
	return []string{
		dir,
		filepath.Join(dir, backend.Paths.Data),
		filepath.Join(dir, backend.Paths.Snapshots),
		filepath.Join(dir, backend.Paths.Index),
		filepath.Join(dir, backend.Paths.Locks),
		filepath.Join(dir, backend.Paths.Keys),
		filepath.Join(dir, backend.Paths.Temp),
	}
}

// Open opens the local backend as specified by config.
func Open(dir string) (*Local, error) {
	// test if all necessary dirs are there
	for _, d := range paths(dir) {
		if _, err := os.Stat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	return &Local{p: dir}, nil
}

// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(dir string) (*Local, error) {
	// test if config file already exists
	_, err := os.Lstat(filepath.Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp
	for _, d := range paths(dir) {
		err := os.MkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	// open backend
	return Open(dir)
}

// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
	return b.p
}

// Construct path for given Type and name.
func filename(base string, t backend.Type, name string) string {
	if t == backend.Config {
		return filepath.Join(base, "config")
	}

	return filepath.Join(dirname(base, t, name), name)
}

// Construct directory for given Type.
func dirname(base string, t backend.Type, name string) string {
	var n string
	switch t {
	case backend.Data:
		n = backend.Paths.Data
		if len(name) > 2 {
			n = filepath.Join(n, name[:2])
		}
	case backend.Snapshot:
		n = backend.Paths.Snapshots
	case backend.Index:
		n = backend.Paths.Index
	case backend.Lock:
		n = backend.Paths.Locks
	case backend.Key:
		n = backend.Paths.Keys
	}
	return filepath.Join(base, n)
}

// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
	if err := h.Valid(); err != nil {
		return 0, err
	}

	f, err := os.Open(filename(b.p, h.Type, h.Name))
	if err != nil {
		return 0, err
	}

	defer func() {
		e := f.Close()
		if err == nil && e != nil {
			err = e
		}
	}()

	if off > 0 {
		_, err = f.Seek(off, 0)
		if err != nil {
			return 0, err
		}
	}

	return io.ReadFull(f, p)
}

// writeToTempfile saves p into a tempfile in tempdir.
func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
	if err != nil {
		return "", err
	}

	n, err := tmpfile.Write(p)
	if err != nil {
		return "", err
	}

	if n != len(p) {
		return "", errors.New("not all bytes writen")
	}

	if err = tmpfile.Sync(); err != nil {
		return "", err
	}

	err = tmpfile.Close()
	if err != nil {
		return "", err
	}

	return tmpfile.Name(), nil
}

// Save stores data in the backend at the handle.
func (b *Local) Save(h backend.Handle, p []byte) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
	debug.Log("local.Save", "saved %v (%d bytes) to %v", h, len(p), tmpfile)

	filename := filename(b.p, h.Type, h.Name)

	// test if new path already exists
	if _, err := os.Stat(filename); err == nil {
		return fmt.Errorf("Rename(): file %v already exists", filename)
	}

	// create directories if necessary, ignore errors
	if h.Type == backend.Data {
		err = os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if err != nil {
			return err
		}
	}

	err = os.Rename(tmpfile, filename)
	debug.Log("local.Save", "save %v: rename %v -> %v: %v",
		h, filepath.Base(tmpfile), filepath.Base(filename), err)

	if err != nil {
		return err
	}

	// set mode to read-only
	fi, err := os.Stat(filename)
	if err != nil {
		return err
	}

	return setNewFileMode(filename, fi)
}

// Stat returns information about a blob.
func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) {
	if err := h.Valid(); err != nil {
		return backend.BlobInfo{}, err
	}

	fi, err := os.Stat(filename(b.p, h.Type, h.Name))
	if err != nil {
		return backend.BlobInfo{}, err
	}

	return backend.BlobInfo{Size: fi.Size()}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(t backend.Type, name string) (bool, error) {
	_, err := os.Stat(filename(b.p, t, name))
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// Remove removes the blob with the given name and type.
func (b *Local) Remove(t backend.Type, name string) error {
	fn := filename(b.p, t, name)

	// reset read-only flag
	err := os.Chmod(fn, 0666)
	if err != nil {
		return err
	}

	return os.Remove(fn)
}

func isFile(fi os.FileInfo) bool {
	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
}

func readdir(d string) (fileInfos []os.FileInfo, err error) {
	f, e := os.Open(d)
	if e != nil {
		return nil, e
	}

	defer func() {
		e := f.Close()
		if err == nil {
			err = e
		}
	}()

	return f.Readdir(-1)
}

// listDir returns a list of all files in d.
func listDir(d string) (filenames []string, err error) {
	fileInfos, err := readdir(d)
	if err != nil {
		return nil, err
	}

	for _, fi := range fileInfos {
		if isFile(fi) {
			filenames = append(filenames, fi.Name())
		}
	}

	return filenames, nil
}

// listDirs returns a list of all files in directories within d.
func listDirs(dir string) (filenames []string, err error) {
	fileInfos, err := readdir(dir)
	if err != nil {
		return nil, err
	}

	for _, fi := range fileInfos {
		if !fi.IsDir() {
			continue
		}

		files, err := listDir(filepath.Join(dir, fi.Name()))
		if err != nil {
			continue
		}

		filenames = append(filenames, files...)
	}

	return filenames, nil
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
	lister := listDir
	if t == backend.Data {
		lister = listDirs
	}

	ch := make(chan string)
	items, err := lister(filepath.Join(dirname(b.p, t, "")))
	if err != nil {
		close(ch)
		return ch
	}

	go func() {
		defer close(ch)
		for _, m := range items {
			if m == "" {
				continue
			}

			select {
			case ch <- m:
			case <-done:
				return
			}
		}
	}()

	return ch
}

// Delete removes the repository and all files.
func (b *Local) Delete() error {
	return os.RemoveAll(b.p)
}

// Close closes all open files.
func (b *Local) Close() error {
	// this does not need to do anything, all open files are closed within the
	// same function.
	return nil
}
<commit_msg>local backend: Drop file content from cache after write<commit_after>package local

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"restic/backend"
	"restic/debug"
	"restic/fs"
)

// Local is a backend in a local directory.
type Local struct {
	p string
}

func paths(dir string) []string {
	return []string{
		dir,
		filepath.Join(dir, backend.Paths.Data),
		filepath.Join(dir, backend.Paths.Snapshots),
		filepath.Join(dir, backend.Paths.Index),
		filepath.Join(dir, backend.Paths.Locks),
		filepath.Join(dir, backend.Paths.Keys),
		filepath.Join(dir, backend.Paths.Temp),
	}
}

// Open opens the local backend as specified by config.
func Open(dir string) (*Local, error) {
	// test if all necessary dirs are there
	for _, d := range paths(dir) {
		if _, err := os.Stat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	return &Local{p: dir}, nil
}

// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(dir string) (*Local, error) {
	// test if config file already exists
	_, err := os.Lstat(filepath.Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp
	for _, d := range paths(dir) {
		err := os.MkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	// open backend
	return Open(dir)
}

// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
	return b.p
}

// Construct path for given Type and name.
func filename(base string, t backend.Type, name string) string {
	if t == backend.Config {
		return filepath.Join(base, "config")
	}

	return filepath.Join(dirname(base, t, name), name)
}

// Construct directory for given Type.
func dirname(base string, t backend.Type, name string) string {
	var n string
	switch t {
	case backend.Data:
		n = backend.Paths.Data
		if len(name) > 2 {
			n = filepath.Join(n, name[:2])
		}
	case backend.Snapshot:
		n = backend.Paths.Snapshots
	case backend.Index:
		n = backend.Paths.Index
	case backend.Lock:
		n = backend.Paths.Locks
	case backend.Key:
		n = backend.Paths.Keys
	}
	return filepath.Join(base, n)
}

// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
	if err := h.Valid(); err != nil {
		return 0, err
	}

	f, err := os.Open(filename(b.p, h.Type, h.Name))
	if err != nil {
		return 0, err
	}

	defer func() {
		e := f.Close()
		if err == nil && e != nil {
			err = e
		}
	}()

	if off > 0 {
		_, err = f.Seek(off, 0)
		if err != nil {
			return 0, err
		}
	}

	return io.ReadFull(f, p)
}

// writeToTempfile saves p into a tempfile in tempdir.
func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
	if err != nil {
		return "", err
	}

	n, err := tmpfile.Write(p)
	if err != nil {
		return "", err
	}

	if n != len(p) {
		return "", errors.New("not all bytes writen")
	}

	if err = tmpfile.Sync(); err != nil {
		return "", err
	}

	err = fs.ClearCache(tmpfile)
	if err != nil {
		return "", err
	}

	err = tmpfile.Close()
	if err != nil {
		return "", err
	}

	return tmpfile.Name(), nil
}

// Save stores data in the backend at the handle.
func (b *Local) Save(h backend.Handle, p []byte) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
	debug.Log("local.Save", "saved %v (%d bytes) to %v", h, len(p), tmpfile)

	filename := filename(b.p, h.Type, h.Name)

	// test if new path already exists
	if _, err := os.Stat(filename); err == nil {
		return fmt.Errorf("Rename(): file %v already exists", filename)
	}

	// create directories if necessary, ignore errors
	if h.Type == backend.Data {
		err = os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if err != nil {
			return err
		}
	}

	err = os.Rename(tmpfile, filename)
	debug.Log("local.Save", "save %v: rename %v -> %v: %v",
		h, filepath.Base(tmpfile), filepath.Base(filename), err)

	if err != nil {
		return err
	}

	// set mode to read-only
	fi, err := os.Stat(filename)
	if err != nil {
		return err
	}

	return setNewFileMode(filename, fi)
}

// Stat returns information about a blob.
func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) {
	if err := h.Valid(); err != nil {
		return backend.BlobInfo{}, err
	}

	fi, err := os.Stat(filename(b.p, h.Type, h.Name))
	if err != nil {
		return backend.BlobInfo{}, err
	}

	return backend.BlobInfo{Size: fi.Size()}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(t backend.Type, name string) (bool, error) {
	_, err := os.Stat(filename(b.p, t, name))
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// Remove removes the blob with the given name and type.
func (b *Local) Remove(t backend.Type, name string) error {
	fn := filename(b.p, t, name)

	// reset read-only flag
	err := os.Chmod(fn, 0666)
	if err != nil {
		return err
	}

	return os.Remove(fn)
}

func isFile(fi os.FileInfo) bool {
	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
}

func readdir(d string) (fileInfos []os.FileInfo, err error) {
	f, e := os.Open(d)
	if e != nil {
		return nil, e
	}

	defer func() {
		e := f.Close()
		if err == nil {
			err = e
		}
	}()

	return f.Readdir(-1)
}

// listDir returns a list of all files in d.
func listDir(d string) (filenames []string, err error) {
	fileInfos, err := readdir(d)
	if err != nil {
		return nil, err
	}

	for _, fi := range fileInfos {
		if isFile(fi) {
			filenames = append(filenames, fi.Name())
		}
	}

	return filenames, nil
}

// listDirs returns a list of all files in directories within d.
func listDirs(dir string) (filenames []string, err error) {
	fileInfos, err := readdir(dir)
	if err != nil {
		return nil, err
	}

	for _, fi := range fileInfos {
		if !fi.IsDir() {
			continue
		}

		files, err := listDir(filepath.Join(dir, fi.Name()))
		if err != nil {
			continue
		}

		filenames = append(filenames, files...)
	}

	return filenames, nil
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
	lister := listDir
	if t == backend.Data {
		lister = listDirs
	}

	ch := make(chan string)
	items, err := lister(filepath.Join(dirname(b.p, t, "")))
	if err != nil {
		close(ch)
		return ch
	}

	go func() {
		defer close(ch)
		for _, m := range items {
			if m == "" {
				continue
			}

			select {
			case ch <- m:
			case <-done:
				return
			}
		}
	}()

	return ch
}

// Delete removes the repository and all files.
func (b *Local) Delete() error {
	return os.RemoveAll(b.p)
}

// Close closes all open files.
func (b *Local) Close() error {
	// this does not need to do anything, all open files are closed within the
	// same function.
	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"text/tabwriter"
	"time"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/version"
	pfscmds "github.com/pachyderm/pachyderm/src/server/pfs/cmds"
	ppscmds "github.com/pachyderm/pachyderm/src/server/pps/cmds"
	"github.com/spf13/cobra"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/proto/version"
	"golang.org/x/net/context"
)

func PachctlCmd(address string) (*cobra.Command, error) {
	rootCmd := &cobra.Command{
		Use: os.Args[0],
		Long: `Access the Pachyderm API.

Envronment variables:
  ADDRESS=0.0.0.0:30650, the server to connect to.
`,
	}
	pfsCmds := pfscmds.Cmds(address)
	for _, cmd := range pfsCmds {
		rootCmd.AddCommand(cmd)
	}
	ppsCmds, err := ppscmds.Cmds(address)
	if err != nil {
		return nil, err
	}
	for _, cmd := range ppsCmds {
		rootCmd.AddCommand(cmd)
	}

	version := &cobra.Command{
		Use:   "version",
		Short: "Return version information.",
		Long:  "Return version information.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			printVersionHeader(writer)
			printVersion(writer, "pachctl", version.Version)
			writer.Flush()

			versionClient, err := getVersionAPIClient(address)
			if err != nil {
				return err
			}
			ctx, _ := context.WithTimeout(context.Background(), time.Second)
			version, err := versionClient.GetVersion(ctx, &google_protobuf.Empty{})

			if err != nil {
				fmt.Fprintf(writer, "pachd\tUNKNOWN: Error %v\n", err)
				return writer.Flush()
			}

			printVersion(writer, "pachd", version)
			return writer.Flush()
		}),
	}
	deleteAll := &cobra.Command{
		Use:   "delete-all",
		Short: "Delete everything.",
		Long: `Delete all repos, commits, files, pipelines and jobs.
This resets the cluster to its initial state.`,
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			fmt.Printf("Are you sure you want to delete all repos, commits, files, pipelines and jobs? yN\n")
			r := bufio.NewReader(os.Stdin)
			bytes, err := r.ReadBytes('\n')
			if err != nil {
				return err
			}
			if bytes[0] == 'y' || bytes[0] == 'Y' {
				return client.DeleteAll()
			}
			return nil
		}),
	}
	rootCmd.AddCommand(version)
	rootCmd.AddCommand(deleteAll)
	return rootCmd, nil
}

func getVersionAPIClient(address string) (protoversion.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return protoversion.NewAPIClient(clientConn), nil
}

func printVersionHeader(w io.Writer) {
	fmt.Fprintf(w, "COMPONENT\tVERSION\t\n")
}

func printVersion(w io.Writer, component string, v *protoversion.Version) {
	fmt.Fprintf(w, "%s\t%s\t\n", component, version.PrettyPrintVersion(v))
}
<commit_msg>Make pachd version error more descriptive<commit_after>package cmd

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"text/tabwriter"
	"time"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/version"
	pfscmds "github.com/pachyderm/pachyderm/src/server/pfs/cmds"
	ppscmds "github.com/pachyderm/pachyderm/src/server/pps/cmds"
	"github.com/spf13/cobra"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/proto/version"
	"golang.org/x/net/context"
)

func PachctlCmd(address string) (*cobra.Command, error) {
	rootCmd := &cobra.Command{
		Use: os.Args[0],
		Long: `Access the Pachyderm API.

Envronment variables:
  ADDRESS=0.0.0.0:30650, the server to connect to.
`,
	}
	pfsCmds := pfscmds.Cmds(address)
	for _, cmd := range pfsCmds {
		rootCmd.AddCommand(cmd)
	}
	ppsCmds, err := ppscmds.Cmds(address)
	if err != nil {
		return nil, sanitizeErr(err)
	}
	for _, cmd := range ppsCmds {
		rootCmd.AddCommand(cmd)
	}

	version := &cobra.Command{
		Use:   "version",
		Short: "Return version information.",
		Long:  "Return version information.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			printVersionHeader(writer)
			printVersion(writer, "pachctl", version.Version)
			writer.Flush()

			versionClient, err := getVersionAPIClient(address)
			if err != nil {
				return sanitizeErr(err)
			}
			ctx, _ := context.WithTimeout(context.Background(), time.Second)
			version, err := versionClient.GetVersion(ctx, &google_protobuf.Empty{})

			if err != nil {
				fmt.Fprintf(writer, "pachd\t(version unknown) : error connecting to pachd server: %v\n", sanitizeErr(err))
				return writer.Flush()
			}

			printVersion(writer, "pachd", version)
			return writer.Flush()
		}),
	}
	deleteAll := &cobra.Command{
		Use:   "delete-all",
		Short: "Delete everything.",
		Long: `Delete all repos, commits, files, pipelines and jobs.
This resets the cluster to its initial state.`,
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return sanitizeErr(err)
			}
			fmt.Printf("Are you sure you want to delete all repos, commits, files, pipelines and jobs? yN\n")
			r := bufio.NewReader(os.Stdin)
			bytes, err := r.ReadBytes('\n')
			if err != nil {
				return err
			}
			if bytes[0] == 'y' || bytes[0] == 'Y' {
				return client.DeleteAll()
			}
			return nil
		}),
	}
	rootCmd.AddCommand(version)
	rootCmd.AddCommand(deleteAll)
	return rootCmd, nil
}

func getVersionAPIClient(address string) (protoversion.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return protoversion.NewAPIClient(clientConn), nil
}

func printVersionHeader(w io.Writer) {
	fmt.Fprintf(w, "COMPONENT\tVERSION\t\n")
}

func printVersion(w io.Writer, component string, v *protoversion.Version) {
	fmt.Fprintf(w, "%s\t%s\t\n", component, version.PrettyPrintVersion(v))
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<|endoftext|>


<commit_before>package metrics

import (
	"fmt"
	"time"

	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	db "github.com/pachyderm/pachyderm/src/server/pfs/db"

	"github.com/dancannon/gorethink"
	"go.pedge.io/lion/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

type Reporter struct {
	clusterID  string
	kubeClient *kube.Client
	dbClient   *gorethink.Session
	pfsDbName  string
	ppsDbName  string
}

func NewReporter(clusterID string, kubeClient *kube.Client, address string, pfsDbName string, ppsDbName string) (*Reporter, error) {
	dbClient, err := db.DbConnect(address)
	if err != nil {
		return nil, fmt.Errorf("Error connected to DB when reporting metrics: %v\n", err)
	}
	return &Reporter{
		clusterID:  clusterID,
		kubeClient: kubeClient,
		dbClient:   dbClient,
		pfsDbName:  pfsDbName,
		ppsDbName:  ppsDbName,
	}, nil
}

// If we're not reporting metrics, incrementing should do nothing
var metricsEnabled = false

// Segment API allows for map[string]interface{} for a single user's traits
// But we only care about things that are countable for the moment
// map userID -> action name -> count
type countableActions map[string]interface{}
type countableUserActions map[string]countableActions

type incrementUserAction struct {
	action string
	user   string
}

var userActions = make(countableUserActions)
var incrementActionChannel = make(chan *incrementUserAction, 0)

//IncrementUserAction updates a counter per user per action for an API method by name
func IncrementUserAction(ctx context.Context, action string) {
	if !metricsEnabled {
		return
	}
	fmt.Printf("!!! trying to increment user actionw ctx: [%v]\n", ctx)
	md, ok := metadata.FromContext(ctx)
	fmt.Printf("!!! metadata: %v\n", md)
	if ok && md["userid"] != nil && len(md["userid"]) > 0 {
		userID := md["userid"][0]
		fmt.Printf("!!! incrementing user action: %v, %v\n", userID, action)
		incrementActionChannel <- &incrementUserAction{
			action: action,
			user:   userID,
		}
		fmt.Printf("!!! incremented user action!\n")
	}
}

func (r *Reporter) dbMetrics(metrics *Metrics) {
	cursor, err := gorethink.Object(
		"Repos",
		gorethink.DB(r.pfsDbName).Table("Repos").Count(),
		"Commits",
		gorethink.DB(r.pfsDbName).Table("Commits").Count(),
		"ArchivedCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Archived": true,
			},
		).Count(),
		"CancelledCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Cancelled": true,
			},
		).Count(),
		"Files",
		gorethink.DB(r.pfsDbName).Table("Diffs").Group("Path").Ungroup().Count(),
		"Jobs",
		gorethink.DB(r.ppsDbName).Table("JobInfos").Count(),
		"Pipelines",
		gorethink.DB(r.ppsDbName).Table("PipelineInfos").Count(),
	).Run(r.dbClient)
	if err != nil {
		protolion.Errorf("Error Fetching Metrics:%+v", err)
	}
	cursor.One(&metrics)
}

// ReportMetrics blocks and reports metrics every 15 seconds
func (r *Reporter) ReportMetrics() {
	metricsEnabled = true
	reportingTicker := time.NewTicker(time.Second * 15)
	for {
		select {
		case incrementAction := <-incrementActionChannel:
			fmt.Printf("incrementing action in map!\n")
			if userActions[incrementAction.user] == nil {
				userActions[incrementAction.user] = make(countableActions)
			}
			val, _ := userActions[incrementAction.user][incrementAction.action]
			userActions[incrementAction.user][incrementAction.action] = val.(uint64) + uint64(1)
			break
		case <-reportingTicker.C:
			fmt.Printf("!!! TICK - reporting to segment\n")
			r.reportToSegment()
			fmt.Printf("!!! kicked off segment reporting\n")
		}
	}
}

func (r *Reporter) reportToSegment() {
	fmt.Printf("!!! Reporting to segment, user actions: [%v]\n", userActions)
	if len(userActions) > 0 {
		batchOfUserActions := make(countableUserActions)
		// copy the existing stats into a new object so we can make the segment
		// request asynchronously
		for user, actions := range userActions {
			singleUserActions := make(countableActions)
			for name, count := range actions {
				singleUserActions[name] = count
			}
			batchOfUserActions[user] = singleUserActions
			go identifyUser(user)
		}
		go r.reportUserMetrics(batchOfUserActions)
		userActions = make(countableUserActions)
	}
	go r.reportClusterMetrics()
}

func (r *Reporter) reportUserMetrics(batchOfUserActions countableUserActions) {
	if len(batchOfUserActions) > 0 {
		reportUserMetricsToSegment(batchOfUserActions)
	}
}

func (r *Reporter) reportClusterMetrics() {
	metrics := &Metrics{}
	r.dbMetrics(metrics)
	externalMetrics(r.kubeClient, metrics)
	metrics.ID = r.clusterID
	metrics.PodID = uuid.NewWithoutDashes()
	metrics.Version = version.PrettyPrintVersion(version.Version)
	reportClusterMetricsToSegment(metrics)
}
<commit_msg>If non initialized, interface{} is nil, so need to handle this case<commit_after>package metrics

import (
	"fmt"
	"time"

	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	db "github.com/pachyderm/pachyderm/src/server/pfs/db"

	"github.com/dancannon/gorethink"
	"go.pedge.io/lion/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

type Reporter struct {
	clusterID  string
	kubeClient *kube.Client
	dbClient   *gorethink.Session
	pfsDbName  string
	ppsDbName  string
}

func NewReporter(clusterID string, kubeClient *kube.Client, address string, pfsDbName string, ppsDbName string) (*Reporter, error) {
	dbClient, err := db.DbConnect(address)
	if err != nil {
		return nil, fmt.Errorf("Error connected to DB when reporting metrics: %v\n", err)
	}
	return &Reporter{
		clusterID:  clusterID,
		kubeClient: kubeClient,
		dbClient:   dbClient,
		pfsDbName:  pfsDbName,
		ppsDbName:  ppsDbName,
	}, nil
}

// If we're not reporting metrics, incrementing should do nothing
var metricsEnabled = false

// Segment API allows for map[string]interface{} for a single user's traits
// But we only care about things that are countable for the moment
// map userID -> action name -> count
type countableActions map[string]interface{}
type countableUserActions map[string]countableActions

type incrementUserAction struct {
	action string
	user   string
}

var userActions = make(countableUserActions)
var incrementActionChannel = make(chan *incrementUserAction, 0)

//IncrementUserAction updates a counter per user per action for an API method by name
func IncrementUserAction(ctx context.Context, action string) {
	if !metricsEnabled {
		return
	}
	fmt.Printf("!!! trying to increment user actionw ctx: [%v]\n", ctx)
	md, ok := metadata.FromContext(ctx)
	fmt.Printf("!!! metadata: %v\n", md)
	if ok && md["userid"] != nil && len(md["userid"]) > 0 {
		userID := md["userid"][0]
		fmt.Printf("!!! incrementing user action: %v, %v\n", userID, action)
		incrementActionChannel <- &incrementUserAction{
			action: action,
			user:   userID,
		}
		fmt.Printf("!!! incremented user action!\n")
	}
}

func (r *Reporter) dbMetrics(metrics *Metrics) {
	cursor, err := gorethink.Object(
		"Repos",
		gorethink.DB(r.pfsDbName).Table("Repos").Count(),
		"Commits",
		gorethink.DB(r.pfsDbName).Table("Commits").Count(),
		"ArchivedCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Archived": true,
			},
		).Count(),
		"CancelledCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Cancelled": true,
			},
		).Count(),
		"Files",
		gorethink.DB(r.pfsDbName).Table("Diffs").Group("Path").Ungroup().Count(),
		"Jobs",
		gorethink.DB(r.ppsDbName).Table("JobInfos").Count(),
		"Pipelines",
		gorethink.DB(r.ppsDbName).Table("PipelineInfos").Count(),
	).Run(r.dbClient)
	if err != nil {
		protolion.Errorf("Error Fetching Metrics:%+v", err)
	}
	cursor.One(&metrics)
}

// ReportMetrics blocks and reports metrics every 15 seconds
func (r *Reporter) ReportMetrics() {
	metricsEnabled = true
	reportingTicker := time.NewTicker(time.Second * 15)
	for {
		select {
		case incrementAction := <-incrementActionChannel:
			fmt.Printf("incrementing action in map!\n")
			if userActions[incrementAction.user] == nil {
				userActions[incrementAction.user] = make(countableActions)
			}
			val, ok := userActions[incrementAction.user][incrementAction.action]
			if !ok {
				val = uint64(0)
			}
			userActions[incrementAction.user][incrementAction.action] = val.(uint64) + uint64(1)
			break
		case <-reportingTicker.C:
			fmt.Printf("!!! TICK - reporting to segment\n")
			r.reportToSegment()
			fmt.Printf("!!! kicked off segment reporting\n")
		}
	}
}

func (r *Reporter) reportToSegment() {
	fmt.Printf("!!! Reporting to segment, user actions: [%v]\n", userActions)
	if len(userActions) > 0 {
		batchOfUserActions := make(countableUserActions)
		// copy the existing stats into a new object so we can make the segment
		// request asynchronously
		for user, actions := range userActions {
			singleUserActions := make(countableActions)
			for name, count := range actions {
				singleUserActions[name] = count
			}
			batchOfUserActions[user] = singleUserActions
			go identifyUser(user)
		}
		go r.reportUserMetrics(batchOfUserActions)
		userActions = make(countableUserActions)
	}
	go r.reportClusterMetrics()
}

func (r *Reporter) reportUserMetrics(batchOfUserActions countableUserActions) {
	if len(batchOfUserActions) > 0 {
		reportUserMetricsToSegment(batchOfUserActions)
	}
}

func (r *Reporter) reportClusterMetrics() {
	metrics := &Metrics{}
	r.dbMetrics(metrics)
	externalMetrics(r.kubeClient, metrics)
	metrics.ID = r.clusterID
	metrics.PodID = uuid.NewWithoutDashes()
	metrics.Version = version.PrettyPrintVersion(version.Version)
	reportClusterMetricsToSegment(metrics)
}
<|endoftext|>


<commit_before>//
// nntp.go
//
package main

import (
  "bufio"
  "bytes"
  "io/ioutil"
  "log"
  "net"
  "strings"
)

type ConnectionInfo struct {
  mode string
  newsgroup string
  allowsPosting bool
  supportsStream bool
}

type NNTPConnection struct {
  conn net.Conn
  reader *bufio.Reader
  inbound bool
  debug bool
  info *ConnectionInfo
  policy *FeedPolicy
  send chan *NNTPMessage
}

func (self *NNTPConnection) HandleOutbound(d *NNTPDaemon) {
  var err error
  line := self.ReadLine()
  self.info.allowsPosting = strings.HasPrefix(line, "200 ")
  // they allow posting
  // send capabilities command
  err = self.SendLine("CAPABILITIES")

  // get capabilites
  for {
    line = strings.ToLower(self.ReadLine())
    if line == ".\r\n" {
      // done reading capabilities
      break
    }
    if line == "streaming\r\n" {
      self.info.supportsStream = true
    } else if line == "postihavestreaming\r\n" {
      self.info.supportsStream = true
    }
  }

  // if they support streaming and allow posting continue
  // otherwise quit
  if ! self.info.supportsStream || ! self.info.allowsPosting {
    if self.debug {
      log.Println(self.info.supportsStream, self.info.allowsPosting)
    }

    self.Quit()
    return
  }
  err = self.SendLine("MODE STREAM")
  if err != nil {
    return
  }
  line = self.ReadLine()
  if strings.HasPrefix(line, "203 ") {
    self.info.mode = "stream"
    log.Println("streaming mode activated")
  } else {
    self.Quit()
    return
  }

  if d.sync_on_start {
    d.store.IterateAllArticles(func(messageID string) bool {
      msg := d.store.GetMessage(messageID, false)
      if msg != nil {
        err = self.sendMessage(msg, d)
      }
      return err != nil
    })
  }

  // mainloop
  for  {
    if err != nil {
      // error from previous
      break
    }
    // poll for new message
    message := <- self.send
    err = self.sendMessage(message, d)
    if err != nil {
      log.Println(err)
      break
    }
  }
}

func (self *NNTPConnection) sendMessage(message *NNTPMessage, d *NNTPDaemon) error {
  var err error
  var line string
  // check if we allow it
  if self.policy == nil {
    // we have no policy so reject
    return nil
  }
  if ! self.policy.AllowsNewsgroup(message.Newsgroup) {
    log.Println("not federating article", message.MessageID, "beause it's in", message.Newsgroup)
    return nil
  }
  if ! self.policy.FederateNewsgroup(message.Newsgroup) {
    log.Println("dont federate article", message.messageID, "disallowed by feed policy")
    return nil
  }
  // send check
  err = self.SendLine("CHECK "+message.MessageID)
  line = self.ReadLine()
  if strings.HasPrefix(line, "238 ") {
    // accepted
    // send it
    err = self.SendLine("TAKETHIS "+message.MessageID)
    if err != nil {
      log.Println("error in outfeed", err)
      return  err
    }
    // load file
    data, err := ioutil.ReadFile(d.store.GetFilename(message.MessageID))
    if err != nil {
      log.Fatal("failed to read article", message.MessageID)
      self.Quit()
      return err
    }
    // split into lines
    parts := bytes.Split(data,[]byte{'\n'})
    // for each line send it
    for idx := range parts {
      ba := parts[idx]
      err = self.SendBytes(ba)
      err = self.Send("\r\n")
    }
    // send delimiter
    err = self.SendLine(".")
    if err != nil {
      log.Println("failed to send")
      self.Quit()
      return err
    }
    // check for success / fail
    line := self.ReadLine()
    if strings.HasPrefix(line, "239 ") {
      log.Println("Article", message.MessageID, "sent")
    } else {
      log.Println("Article", message.MessageID, "failed to send", line)
    }
    // done
    return nil
  } else if strings.HasPrefix(line, "435 ") {
    // already have it
    if self.debug {
      log.Println(message.MessageID, "already owned")
    }
  } else if strings.HasPrefix(line, "437 ") {
    // article banned
    log.Println(message.MessageID, "was banned")
  }
  if err != nil {
    self.Quit()
    log.Println("failure in outfeed", err)
    return err
  }
  return nil
}

// handle inbound connection
func (self *NNTPConnection) HandleInbound(d *NNTPDaemon) {
  var err error
  self.info.mode = "STREAM"
  log.Println("Incoming nntp connection from", self.conn.RemoteAddr())
  // send welcome
  self.SendLine("200 ayy lmao we are SRNd2, posting allowed")
  for {
    if err != nil {
      log.Println("failure in infeed", err)
      self.Quit()
      return
    }
    line := self.ReadLine()
      if len(line) == 0 {
      break
    }
    // parse line

    _line := strings.Replace(line, "\n", "", -1)
    _line = strings.Replace(_line, "\r", "", -1)
    commands := strings.Split(_line, " ")
    cmd := strings.ToUpper(commands[0])

    // capabilities command
    if cmd == "CAPABILITIES" {
      self.sendCapabilities()
    } else if cmd == "MODE" { // mode switch
      if len(commands) == 2 {
        mode := strings.ToUpper(commands[1])
        if mode == "READER" {
          self.SendLine("501 no reader mode")
        } else if mode == "STREAM" {
          self.info.mode = mode
          self.SendLine("203 stream as desired")
        } else {
          self.SendLine("501 unknown mode")
        }
      } else {
        self.SendLine("500 syntax error")
      }
    } else if self.info.mode == "STREAM" { // we are in stream mode
      if cmd == "TAKETHIS" {
        if len(commands) == 2 {
          article := commands[1]
          if ValidMessageID(article) {
            file := d.store.OpenFile(article)
            var rewrote_path bool
            for {
              line := self.ReadLine()
              // unexpected close
              if len(line) == 0 {
                log.Fatal(self.conn.RemoteAddr(), "unexpectedly closed connection")
              }
              if ! rewrote_path && strings.HasPrefix(line, "Path: ") {
                line = "Path: " + d.instance_name + "!" + line[6:]
              }
              // done reading
              if line == ".\r\n" {
                break
              } else {
                line = strings.Replace(line, "\r", "", -1)
                file.Write([]byte(line))
              }
            }
            file.Close()
            // the send was good
            // tell them
            self.SendLine("239 "+article)
            log.Println(self.conn.RemoteAddr(), "got article", article)
            d.infeed <- article
          }
        }
      }
      if cmd == "CHECK" {
        if len(commands) == 2 {
          if ! ValidMessageID(commands[1]) {
            self.SendLine("501 bad message id")
            continue
          }
          article := commands[1]
          if d.store.HasArticle(article) {
            self.SendLine("435 "+commands[1]+" we have this article")
          } else {
            self.SendLine("238 "+commands[1]+" we want this article please give it")
          }
        }
      }
    }
  }
  self.Close()
}

func (self *NNTPConnection) sendCapabilities() {
  self.SendLine("101 we can do stuff")
  self.SendLine("VERSION 2")
  self.SendLine("IMPLEMENTATION srndv2 better than SRNd")
  self.SendLine("STREAMING")
  self.SendLine(".")
}

func (self *NNTPConnection) Quit() {
  if ! self.inbound {
    self.SendLine("QUIT")
    _ = self.ReadLine()
  }
  self.Close()
}

func (self *NNTPConnection) ReadLine() string {
  line, err := self.reader.ReadString('\n')
  if err != nil {
    return ""
  }
  //line = strings.Replace(line, "\n", "", -1)
  //line = strings.Replace(line, "\r", "", -1)
  if self.debug {
    log.Println(self.conn.RemoteAddr(), "recv line", line)
  }
  return line
}

// send a line
func (self *NNTPConnection) SendLine(line string) error {
  if self.debug {
    log.Println(self.conn.RemoteAddr(), "send line", line)
  }
  return self.Send(line+"\r\n")
}

// send data
func (self *NNTPConnection) Send(data string) error {
  _, err := self.conn.Write([]byte(data))
  return err
}

// send data
func (self *NNTPConnection) SendBytes(data []byte) error {
  _ , err := self.conn.Write(data)
  return err
}

// close the connection
func (self *NNTPConnection) Close() {
  err := self.conn.Close()
  if err != nil {
    log.Println(self.conn.RemoteAddr(), err)
  }
  log.Println(self.conn.RemoteAddr(), "Closed Connection")
}
<commit_msg>variable name was wrong<commit_after>//
// nntp.go
//
package main

import (
  "bufio"
  "bytes"
  "io/ioutil"
  "log"
  "net"
  "strings"
)

type ConnectionInfo struct {
  mode string
  newsgroup string
  allowsPosting bool
  supportsStream bool
}

type NNTPConnection struct {
  conn net.Conn
  reader *bufio.Reader
  inbound bool
  debug bool
  info *ConnectionInfo
  policy *FeedPolicy
  send chan *NNTPMessage
}

func (self *NNTPConnection) HandleOutbound(d *NNTPDaemon) {
  var err error
  line := self.ReadLine()
  self.info.allowsPosting = strings.HasPrefix(line, "200 ")
  // they allow posting
  // send capabilities command
  err = self.SendLine("CAPABILITIES")

  // get capabilites
  for {
    line = strings.ToLower(self.ReadLine())
    if line == ".\r\n" {
      // done reading capabilities
      break
    }
    if line == "streaming\r\n" {
      self.info.supportsStream = true
    } else if line == "postihavestreaming\r\n" {
      self.info.supportsStream = true
    }
  }

  // if they support streaming and allow posting continue
  // otherwise quit
  if ! self.info.supportsStream || ! self.info.allowsPosting {
    if self.debug {
      log.Println(self.info.supportsStream, self.info.allowsPosting)
    }

    self.Quit()
    return
  }
  err = self.SendLine("MODE STREAM")
  if err != nil {
    return
  }
  line = self.ReadLine()
  if strings.HasPrefix(line, "203 ") {
    self.info.mode = "stream"
    log.Println("streaming mode activated")
  } else {
    self.Quit()
    return
  }

  if d.sync_on_start {
    d.store.IterateAllArticles(func(messageID string) bool {
      msg := d.store.GetMessage(messageID, false)
      if msg != nil {
        err = self.sendMessage(msg, d)
      }
      return err != nil
    })
  }

  // mainloop
  for  {
    if err != nil {
      // error from previous
      break
    }
    // poll for new message
    message := <- self.send
    err = self.sendMessage(message, d)
    if err != nil {
      log.Println(err)
      break
    }
  }
}

func (self *NNTPConnection) sendMessage(message *NNTPMessage, d *NNTPDaemon) error {
  var err error
  var line string
  // check if we allow it
  if self.policy == nil {
    // we have no policy so reject
    return nil
  }
  if ! self.policy.AllowsNewsgroup(message.Newsgroup) {
    log.Println("not federating article", message.MessageID, "beause it's in", message.Newsgroup)
    return nil
  }
  if ! self.policy.FederateNewsgroup(message.Newsgroup) {
    log.Println("dont federate article", message.MessageID, "disallowed by feed policy")
    return nil
  }
  // send check
  err = self.SendLine("CHECK "+message.MessageID)
  line = self.ReadLine()
  if strings.HasPrefix(line, "238 ") {
    // accepted
    // send it
    err = self.SendLine("TAKETHIS "+message.MessageID)
    if err != nil {
      log.Println("error in outfeed", err)
      return  err
    }
    // load file
    data, err := ioutil.ReadFile(d.store.GetFilename(message.MessageID))
    if err != nil {
      log.Fatal("failed to read article", message.MessageID)
      self.Quit()
      return err
    }
    // split into lines
    parts := bytes.Split(data,[]byte{'\n'})
    // for each line send it
    for idx := range parts {
      ba := parts[idx]
      err = self.SendBytes(ba)
      err = self.Send("\r\n")
    }
    // send delimiter
    err = self.SendLine(".")
    if err != nil {
      log.Println("failed to send")
      self.Quit()
      return err
    }
    // check for success / fail
    line := self.ReadLine()
    if strings.HasPrefix(line, "239 ") {
      log.Println("Article", message.MessageID, "sent")
    } else {
      log.Println("Article", message.MessageID, "failed to send", line)
    }
    // done
    return nil
  } else if strings.HasPrefix(line, "435 ") {
    // already have it
    if self.debug {
      log.Println(message.MessageID, "already owned")
    }
  } else if strings.HasPrefix(line, "437 ") {
    // article banned
    log.Println(message.MessageID, "was banned")
  }
  if err != nil {
    self.Quit()
    log.Println("failure in outfeed", err)
    return err
  }
  return nil
}

// handle inbound connection
func (self *NNTPConnection) HandleInbound(d *NNTPDaemon) {
  var err error
  self.info.mode = "STREAM"
  log.Println("Incoming nntp connection from", self.conn.RemoteAddr())
  // send welcome
  self.SendLine("200 ayy lmao we are SRNd2, posting allowed")
  for {
    if err != nil {
      log.Println("failure in infeed", err)
      self.Quit()
      return
    }
    line := self.ReadLine()
      if len(line) == 0 {
      break
    }
    // parse line

    _line := strings.Replace(line, "\n", "", -1)
    _line = strings.Replace(_line, "\r", "", -1)
    commands := strings.Split(_line, " ")
    cmd := strings.ToUpper(commands[0])

    // capabilities command
    if cmd == "CAPABILITIES" {
      self.sendCapabilities()
    } else if cmd == "MODE" { // mode switch
      if len(commands) == 2 {
        mode := strings.ToUpper(commands[1])
        if mode == "READER" {
          self.SendLine("501 no reader mode")
        } else if mode == "STREAM" {
          self.info.mode = mode
          self.SendLine("203 stream as desired")
        } else {
          self.SendLine("501 unknown mode")
        }
      } else {
        self.SendLine("500 syntax error")
      }
    } else if self.info.mode == "STREAM" { // we are in stream mode
      if cmd == "TAKETHIS" {
        if len(commands) == 2 {
          article := commands[1]
          if ValidMessageID(article) {
            file := d.store.OpenFile(article)
            var rewrote_path bool
            for {
              line := self.ReadLine()
              // unexpected close
              if len(line) == 0 {
                log.Fatal(self.conn.RemoteAddr(), "unexpectedly closed connection")
              }
              if ! rewrote_path && strings.HasPrefix(line, "Path: ") {
                line = "Path: " + d.instance_name + "!" + line[6:]
              }
              // done reading
              if line == ".\r\n" {
                break
              } else {
                line = strings.Replace(line, "\r", "", -1)
                file.Write([]byte(line))
              }
            }
            file.Close()
            // the send was good
            // tell them
            self.SendLine("239 "+article)
            log.Println(self.conn.RemoteAddr(), "got article", article)
            d.infeed <- article
          }
        }
      }
      if cmd == "CHECK" {
        if len(commands) == 2 {
          if ! ValidMessageID(commands[1]) {
            self.SendLine("501 bad message id")
            continue
          }
          article := commands[1]
          if d.store.HasArticle(article) {
            self.SendLine("435 "+commands[1]+" we have this article")
          } else {
            self.SendLine("238 "+commands[1]+" we want this article please give it")
          }
        }
      }
    }
  }
  self.Close()
}

func (self *NNTPConnection) sendCapabilities() {
  self.SendLine("101 we can do stuff")
  self.SendLine("VERSION 2")
  self.SendLine("IMPLEMENTATION srndv2 better than SRNd")
  self.SendLine("STREAMING")
  self.SendLine(".")
}

func (self *NNTPConnection) Quit() {
  if ! self.inbound {
    self.SendLine("QUIT")
    _ = self.ReadLine()
  }
  self.Close()
}

func (self *NNTPConnection) ReadLine() string {
  line, err := self.reader.ReadString('\n')
  if err != nil {
    return ""
  }
  //line = strings.Replace(line, "\n", "", -1)
  //line = strings.Replace(line, "\r", "", -1)
  if self.debug {
    log.Println(self.conn.RemoteAddr(), "recv line", line)
  }
  return line
}

// send a line
func (self *NNTPConnection) SendLine(line string) error {
  if self.debug {
    log.Println(self.conn.RemoteAddr(), "send line", line)
  }
  return self.Send(line+"\r\n")
}

// send data
func (self *NNTPConnection) Send(data string) error {
  _, err := self.conn.Write([]byte(data))
  return err
}

// send data
func (self *NNTPConnection) SendBytes(data []byte) error {
  _ , err := self.conn.Write(data)
  return err
}

// close the connection
func (self *NNTPConnection) Close() {
  err := self.conn.Close()
  if err != nil {
    log.Println(self.conn.RemoteAddr(), err)
  }
  log.Println(self.conn.RemoteAddr(), "Closed Connection")
}
<|endoftext|>