<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bufio

import (
	"bytes"
	"errors"
	"io"
	"unicode/utf8"
)

// Scanner provides a convenient interface for reading data such as
// a file of newline-delimited lines of text. Successive calls to
// the Scan method will step through the 'tokens' of a file, skipping
// the bytes between the tokens. The specification of a token is
// defined by a split function of type SplitFunc; the default split
// function breaks the input into lines with line termination stripped. Split
// functions are defined in this package for scanning a file into
// lines, bytes, UTF-8-encoded runes, and space-delimited words. The
// client may instead provide a custom split function.
//
// Scanning stops unrecoverably at EOF, the first I/O error, or a token too
// large to fit in the buffer. When a scan stops, the reader may have
// advanced arbitrarily far past the last token. Programs that need more
// control over error handling or large tokens, or must run sequential scans
// on a reader, should use bufio.Reader instead.
//
type Scanner struct {
	r            io.Reader // The reader provided by the client.
	split        SplitFunc // The function to split the tokens.
	maxTokenSize int       // Maximum size of a token; modified by tests.
	token        []byte    // Last token returned by split.
	buf          []byte    // Buffer used as argument to split.
	start        int       // First non-processed byte in buf.
	end          int       // End of data in buf.
	err          error     // Sticky error.
	empties      int       // Count of successive empty tokens.
}

// SplitFunc is the signature of the split function used to tokenize the
// input. The arguments are an initial substring of the remaining unprocessed
// data and a flag, atEOF, that reports whether the Reader has no more data
// to give. The return values are the number of bytes to advance the input
// and the next token to return to the user, plus an error, if any. If the
// data does not yet hold a complete token, for instance if it has no newline
// while scanning lines, SplitFunc can return (0, nil, nil) to signal the
// Scanner to read more data into the slice and try again with a longer slice
// starting at the same point in the input.
//
// If the returned error is non-nil, scanning stops and the error
// is returned to the client.
//
// The function is never called with an empty data slice unless atEOF
// is true. If atEOF is true, however, data may be non-empty and,
// as always, holds unprocessed text.
type SplitFunc func(data []byte, atEOF bool) (advance int, token []byte, err error)

// Errors returned by Scanner.
var (
	ErrTooLong         = errors.New("bufio.Scanner: token too long")
	ErrNegativeAdvance = errors.New("bufio.Scanner: SplitFunc returns negative advance count")
	ErrAdvanceTooFar   = errors.New("bufio.Scanner: SplitFunc returns advance count beyond input")
)

const (
	// MaxScanTokenSize is the maximum size used to buffer a token.
	// The actual maximum token size may be smaller as the buffer
	// may need to include, for instance, a newline.
	MaxScanTokenSize = 64 * 1024
)

// NewScanner returns a new Scanner to read from r.
// The split function defaults to ScanLines.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		r:            r,
		split:        ScanLines,
		maxTokenSize: MaxScanTokenSize,
		buf:          make([]byte, 4096), // Plausible starting size; needn't be large.
	}
}

// Err returns the first non-EOF error that was encountered by the Scanner.
func (s *Scanner) Err() error {
	if s.err == io.EOF {
		return nil
	}
	return s.err
}

// Bytes returns the most recent token generated by a call to Scan.
// The underlying array may point to data that will be overwritten
// by a subsequent call to Scan. It does no allocation.
func (s *Scanner) Bytes() []byte {
	return s.token
}

// Text returns the most recent token generated by a call to Scan
// as a newly allocated string holding its bytes.
func (s *Scanner) Text() string {
	return string(s.token)
}

// Scan advances the Scanner to the next token, which will then be
// available through the Bytes or Text method. It returns false when the
// scan stops, either by reaching the end of the input or an error.
// After Scan returns false, the Err method will return any error that
// occurred during scanning, except that if it was io.EOF, Err
// will return nil.
// Scan panics if the split function returns 100 empty tokens without
// advancing the input. This is a common error mode for scanners.
func (s *Scanner) Scan() bool {
	// Loop until we have a token.
	for {
		// See if we can get a token with what we already have.
		// If we've run out of data but have an error, give the split function
		// a chance to recover any remaining, possibly empty token.
		if s.end > s.start || s.err != nil {
			advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
			if err != nil {
				s.setErr(err)
				return false
			}
			if !s.advance(advance) {
				return false
			}
			s.token = token
			if token != nil {
				if s.err == nil || advance > 0 {
					s.empties = 0
				} else {
					// Returning tokens not advancing input at EOF.
					s.empties++
					if s.empties > 100 {
						panic("bufio.Scan: 100 empty tokens without progressing")
					}
				}
				return true
			}
		}
		// We cannot generate a token with what we are holding.
		// If we've already hit EOF or an I/O error, we are done.
		if s.err != nil {
			// Shut it down.
			s.start = 0
			s.end = 0
			return false
		}
		// Must read more data.
		// First, shift data to beginning of buffer if there's lots of empty space
		// or space is needed.
		if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) {
			copy(s.buf, s.buf[s.start:s.end])
			s.end -= s.start
			s.start = 0
		}
		// Is the buffer full? If so, resize.
		if s.end == len(s.buf) {
			if len(s.buf) >= s.maxTokenSize {
				s.setErr(ErrTooLong)
				return false
			}
			newSize := len(s.buf) * 2
			if newSize > s.maxTokenSize {
				newSize = s.maxTokenSize
			}
			newBuf := make([]byte, newSize)
			copy(newBuf, s.buf[s.start:s.end])
			s.buf = newBuf
			s.end -= s.start
			s.start = 0
			continue
		}
		// Finally we can read some input. Make sure we don't get stuck with
		// a misbehaving Reader. Officially we don't need to do this, but let's
		// be extra careful: Scanner is for safe, simple jobs.
		for loop := 0; ; {
			n, err := s.r.Read(s.buf[s.end:len(s.buf)])
			s.end += n
			if err != nil {
				s.setErr(err)
				break
			}
			if n > 0 {
				s.empties = 0
				break
			}
			loop++
			if loop > maxConsecutiveEmptyReads {
				s.setErr(io.ErrNoProgress)
				break
			}
		}
	}
}

// advance consumes n bytes of the buffer. It reports whether the advance was legal.
func (s *Scanner) advance(n int) bool {
	if n < 0 {
		s.setErr(ErrNegativeAdvance)
		return false
	}
	if n > s.end-s.start {
		s.setErr(ErrAdvanceTooFar)
		return false
	}
	s.start += n
	return true
}

// setErr records the first error encountered.
func (s *Scanner) setErr(err error) {
	if s.err == nil || s.err == io.EOF {
		s.err = err
	}
}

// Split sets the split function for the Scanner. If called, it must be
// called before Scan. The default split function is ScanLines.
func (s *Scanner) Split(split SplitFunc) {
	s.split = split
}

// Split functions

// ScanBytes is a split function for a Scanner that returns each byte as a token.
func ScanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	return 1, data[0:1], nil
}

var errorRune = []byte(string(utf8.RuneError))

// ScanRunes is a split function for a Scanner that returns each
// UTF-8-encoded rune as a token. The sequence of runes returned is
// equivalent to that from a range loop over the input as a string, which
// means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd".
// Because of the Scan interface, this makes it impossible for the client to
// distinguish correctly encoded replacement runes from encoding errors.
func ScanRunes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Fast path 1: ASCII.
	if data[0] < utf8.RuneSelf {
		return 1, data[0:1], nil
	}

	// Fast path 2: Correct UTF-8 decode without error.
	_, width := utf8.DecodeRune(data)
	if width > 1 {
		// It's a valid encoding. Width cannot be one for a correctly encoded
		// non-ASCII rune.
		return width, data[0:width], nil
	}

	// We know it's an error: we have width==1 and implicitly r==utf8.RuneError.
	// Is the error because there wasn't a full rune to be decoded?
	// FullRune distinguishes correctly between erroneous and incomplete encodings.
	if !atEOF && !utf8.FullRune(data) {
		// Incomplete; get more bytes.
		return 0, nil, nil
	}

	// We have a real UTF-8 encoding error. Return a properly encoded error rune
	// but advance only one byte. This matches the behavior of a range loop over
	// an incorrectly encoded string.
	return 1, errorRune, nil
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// ScanLines is a split function for a Scanner that returns each line of
// text, stripped of any trailing end-of-line marker. The returned line may
// be empty. The end-of-line marker is one optional carriage return followed
// by one mandatory newline. In regular expression notation, it is `\r?\n`.
// The last non-empty line of input will be returned even if it has no
// newline.
func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil
}

// isSpace reports whether the character is a Unicode white space character.
// We avoid dependency on the unicode package, but check validity of the implementation
// in the tests.
func isSpace(r rune) bool {
	if r <= '\u00FF' {
		// Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs.
		switch r {
		case ' ', '\t', '\n', '\v', '\f', '\r':
			return true
		case '\u0085', '\u00A0':
			return true
		}
		return false
	}
	// High-valued ones.
	if '\u2000' <= r && r <= '\u200a' {
		return true
	}
	switch r {
	case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
		return true
	}
	return false
}

// ScanWords is a split function for a Scanner that returns each
// space-separated word of text, with surrounding spaces deleted. It will
// never return an empty string. The definition of space is set by
// unicode.IsSpace.
func ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !isSpace(r) {
			break
		}
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isSpace(r) {
			return i + width, data[start:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return start, nil, nil
}
<commit_msg>FIX: Added MaxScanTokenSize as argument to New<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scan

import (
	"bytes"
	"errors"
	"io"
	"unicode/utf8"
)

// Scanner provides a convenient interface for reading data such as
// a file of newline-delimited lines of text. Successive calls to
// the Scan method will step through the 'tokens' of a file, skipping
// the bytes between the tokens. The specification of a token is
// defined by a split function of type SplitFunc; the default split
// function breaks the input into lines with line termination stripped. Split
// functions are defined in this package for scanning a file into
// lines, bytes, UTF-8-encoded runes, and space-delimited words. The
// client may instead provide a custom split function.
//
// Scanning stops unrecoverably at EOF, the first I/O error, or a token too
// large to fit in the buffer. When a scan stops, the reader may have
// advanced arbitrarily far past the last token. Programs that need more
// control over error handling or large tokens, or must run sequential scans
// on a reader, should use bufio.Reader instead.
//
type Scanner struct {
	r            io.Reader // The reader provided by the client.
	split        SplitFunc // The function to split the tokens.
	maxTokenSize int       // Maximum size of a token; modified by tests.
	token        []byte    // Last token returned by split.
	buf          []byte    // Buffer used as argument to split.
	start        int       // First non-processed byte in buf.
	end          int       // End of data in buf.
	err          error     // Sticky error.
	empties      int       // Count of successive empty tokens.
}

// SplitFunc is the signature of the split function used to tokenize the
// input. The arguments are an initial substring of the remaining unprocessed
// data and a flag, atEOF, that reports whether the Reader has no more data
// to give. The return values are the number of bytes to advance the input
// and the next token to return to the user, plus an error, if any. If the
// data does not yet hold a complete token, for instance if it has no newline
// while scanning lines, SplitFunc can return (0, nil, nil) to signal the
// Scanner to read more data into the slice and try again with a longer slice
// starting at the same point in the input.
//
// If the returned error is non-nil, scanning stops and the error
// is returned to the client.
//
// The function is never called with an empty data slice unless atEOF
// is true. If atEOF is true, however, data may be non-empty and,
// as always, holds unprocessed text.
type SplitFunc func(data []byte, atEOF bool) (advance int, token []byte, err error)

// Errors returned by Scanner.
var (
	ErrTooLong         = errors.New("bufio.Scanner: token too long")
	ErrNegativeAdvance = errors.New("bufio.Scanner: SplitFunc returns negative advance count")
	ErrAdvanceTooFar   = errors.New("bufio.Scanner: SplitFunc returns advance count beyond input")
)

const (
	// MaxScanTokenSize is the maximum size used to buffer a token.
	// The actual maximum token size may be smaller as the buffer
	// may need to include, for instance, a newline.
	MaxScanTokenSize         = 64 * 1024
	maxConsecutiveEmptyReads = 100
)

// NewScanner returns a new Scanner to read from r.
// The split function defaults to ScanLines.
func New(r io.Reader, maxScanTokenSize int) *Scanner {
	return &Scanner{
		r:            r,
		split:        ScanLines,
		maxTokenSize: maxScanTokenSize,
		buf:          make([]byte, 4096), // Plausible starting size; needn't be large.
	}
}

// Err returns the first non-EOF error that was encountered by the Scanner.
func (s *Scanner) Err() error {
	if s.err == io.EOF {
		return nil
	}
	return s.err
}

// Bytes returns the most recent token generated by a call to Scan.
// The underlying array may point to data that will be overwritten
// by a subsequent call to Scan. It does no allocation.
func (s *Scanner) Bytes() []byte {
	return s.token
}

// Text returns the most recent token generated by a call to Scan
// as a newly allocated string holding its bytes.
func (s *Scanner) Text() string {
	return string(s.token)
}

// Scan advances the Scanner to the next token, which will then be
// available through the Bytes or Text method. It returns false when the
// scan stops, either by reaching the end of the input or an error.
// After Scan returns false, the Err method will return any error that
// occurred during scanning, except that if it was io.EOF, Err
// will return nil.
// Scan panics if the split function returns 100 empty tokens without
// advancing the input. This is a common error mode for scanners.
func (s *Scanner) Scan() bool {
	// Loop until we have a token.
	for {
		// See if we can get a token with what we already have.
		// If we've run out of data but have an error, give the split function
		// a chance to recover any remaining, possibly empty token.
		if s.end > s.start || s.err != nil {
			advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
			if err != nil {
				s.setErr(err)
				return false
			}
			if !s.advance(advance) {
				return false
			}
			s.token = token
			if token != nil {
				if s.err == nil || advance > 0 {
					s.empties = 0
				} else {
					// Returning tokens not advancing input at EOF.
					s.empties++
					if s.empties > 100 {
						panic("bufio.Scan: 100 empty tokens without progressing")
					}
				}
				return true
			}
		}
		// We cannot generate a token with what we are holding.
		// If we've already hit EOF or an I/O error, we are done.
		if s.err != nil {
			// Shut it down.
			s.start = 0
			s.end = 0
			return false
		}
		// Must read more data.
		// First, shift data to beginning of buffer if there's lots of empty space
		// or space is needed.
		if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) {
			copy(s.buf, s.buf[s.start:s.end])
			s.end -= s.start
			s.start = 0
		}
		// Is the buffer full? If so, resize.
		if s.end == len(s.buf) {
			if len(s.buf) >= s.maxTokenSize {
				s.setErr(ErrTooLong)
				return false
			}
			newSize := len(s.buf) * 2
			if newSize > s.maxTokenSize {
				newSize = s.maxTokenSize
			}
			newBuf := make([]byte, newSize)
			copy(newBuf, s.buf[s.start:s.end])
			s.buf = newBuf
			s.end -= s.start
			s.start = 0
			continue
		}
		// Finally we can read some input. Make sure we don't get stuck with
		// a misbehaving Reader. Officially we don't need to do this, but let's
		// be extra careful: Scanner is for safe, simple jobs.
		for loop := 0; ; {
			n, err := s.r.Read(s.buf[s.end:len(s.buf)])
			s.end += n
			if err != nil {
				s.setErr(err)
				break
			}
			if n > 0 {
				s.empties = 0
				break
			}
			loop++
			if loop > maxConsecutiveEmptyReads {
				s.setErr(io.ErrNoProgress)
				break
			}
		}
	}
}

// advance consumes n bytes of the buffer. It reports whether the advance was legal.
func (s *Scanner) advance(n int) bool {
	if n < 0 {
		s.setErr(ErrNegativeAdvance)
		return false
	}
	if n > s.end-s.start {
		s.setErr(ErrAdvanceTooFar)
		return false
	}
	s.start += n
	return true
}

// setErr records the first error encountered.
func (s *Scanner) setErr(err error) {
	if s.err == nil || s.err == io.EOF {
		s.err = err
	}
}

// Split sets the split function for the Scanner. If called, it must be
// called before Scan. The default split function is ScanLines.
func (s *Scanner) Split(split SplitFunc) {
	s.split = split
}

// Split functions

// ScanBytes is a split function for a Scanner that returns each byte as a token.
func ScanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	return 1, data[0:1], nil
}

var errorRune = []byte(string(utf8.RuneError))

// ScanRunes is a split function for a Scanner that returns each
// UTF-8-encoded rune as a token. The sequence of runes returned is
// equivalent to that from a range loop over the input as a string, which
// means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd".
// Because of the Scan interface, this makes it impossible for the client to
// distinguish correctly encoded replacement runes from encoding errors.
func ScanRunes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Fast path 1: ASCII.
	if data[0] < utf8.RuneSelf {
		return 1, data[0:1], nil
	}

	// Fast path 2: Correct UTF-8 decode without error.
	_, width := utf8.DecodeRune(data)
	if width > 1 {
		// It's a valid encoding. Width cannot be one for a correctly encoded
		// non-ASCII rune.
		return width, data[0:width], nil
	}

	// We know it's an error: we have width==1 and implicitly r==utf8.RuneError.
	// Is the error because there wasn't a full rune to be decoded?
	// FullRune distinguishes correctly between erroneous and incomplete encodings.
	if !atEOF && !utf8.FullRune(data) {
		// Incomplete; get more bytes.
		return 0, nil, nil
	}

	// We have a real UTF-8 encoding error. Return a properly encoded error rune
	// but advance only one byte. This matches the behavior of a range loop over
	// an incorrectly encoded string.
	return 1, errorRune, nil
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// ScanLines is a split function for a Scanner that returns each line of
// text, stripped of any trailing end-of-line marker. The returned line may
// be empty. The end-of-line marker is one optional carriage return followed
// by one mandatory newline. In regular expression notation, it is `\r?\n`.
// The last non-empty line of input will be returned even if it has no
// newline.
func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil
}

// isSpace reports whether the character is a Unicode white space character.
// We avoid dependency on the unicode package, but check validity of the implementation
// in the tests.
func isSpace(r rune) bool {
	if r <= '\u00FF' {
		// Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs.
		switch r {
		case ' ', '\t', '\n', '\v', '\f', '\r':
			return true
		case '\u0085', '\u00A0':
			return true
		}
		return false
	}
	// High-valued ones.
	if '\u2000' <= r && r <= '\u200a' {
		return true
	}
	switch r {
	case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
		return true
	}
	return false
}

// ScanWords is a split function for a Scanner that returns each
// space-separated word of text, with surrounding spaces deleted. It will
// never return an empty string. The definition of space is set by
// unicode.IsSpace.
func ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !isSpace(r) {
			break
		}
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isSpace(r) {
			return i + width, data[start:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return start, nil, nil
}
<|endoftext|>


<commit_before>package sugarcane

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"io"
	"os"
)

// Prepare encodes struct into byte.Buffer.
func prepare(p interface{}) (bytes.Buffer, error) {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(p)
	if err != nil {
		return buf, err
	}
	return buf, nil
}

// Write writes byte buffer to file f.
func Write(f *os.File, buf bytes.Buffer) error {
	w := bufio.NewWriter(f)
	_, err := w.Write(buf.Bytes())
	if err != nil {
		return err
	}
	w.Flush()
	return nil
}

// Open opens a file for writing.
func Open(filename string) (*os.File, error) {
	w, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
	if err != nil {
		return w, err
	}
	return w, nil
}

// Read reads a file
func Read(filename string) (*bytes.Buffer, error) {
	f, err := os.OpenFile(filename, os.O_RDONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	buf := make([]byte, fi.Size()) // make the buffer as big as the file
	r := bufio.NewReader(f)
	for {
		n, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return nil, err
		}
		if n == 0 {
			break
		}
	}
	return bytes.NewBuffer(buf), nil
}

// Insert prepares structure for disk saving.
func Insert(p interface{}, f *os.File) error {
	binary, err := prepare(p)
	if err != nil {
		return err
	}
	Write(f, binary)
	return nil
}

// Scan returns a first structure from byte buffer and decodes it according given structure.
func Scan(p interface{}, data *bytes.Buffer) error {
	dec := gob.NewDecoder(data)
	err := dec.Decode(p)
	if err != nil {
		return err
	}
	return err
}
<commit_msg>function renaming - write doesn't need to be exported<commit_after>package sugarcane

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"io"
	"os"
)

// Prepare encodes struct into byte.Buffer.
func prepare(p interface{}) (bytes.Buffer, error) {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(p)
	if err != nil {
		return buf, err
	}
	return buf, nil
}

// write writes byte buffer to file f.
func write(f *os.File, buf bytes.Buffer) error {
	w := bufio.NewWriter(f)
	_, err := w.Write(buf.Bytes())
	if err != nil {
		return err
	}
	w.Flush()
	return nil
}

// Open opens a file for writing.
func Open(filename string) (*os.File, error) {
	w, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
	if err != nil {
		return w, err
	}
	return w, nil
}

// Read reads a file
func Read(filename string) (*bytes.Buffer, error) {
	f, err := os.OpenFile(filename, os.O_RDONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	buf := make([]byte, fi.Size()) // make the buffer as big as the file
	r := bufio.NewReader(f)
	for {
		n, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return nil, err
		}
		if n == 0 {
			break
		}
	}
	return bytes.NewBuffer(buf), nil
}

// Insert prepares structure for disk saving.
func Insert(p interface{}, f *os.File) error {
	binary, err := prepare(p)
	if err != nil {
		return err
	}
	write(f, binary)
	return nil
}

// Scan returns a first structure from byte buffer and decodes it according given structure.
func Scan(p interface{}, data *bytes.Buffer) error {
	dec := gob.NewDecoder(data)
	err := dec.Decode(p)
	if err != nil {
		return err
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	_ "time"
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"github.com/gorilla/sessions"
	"io"
	"net/http"
	"encoding/json"
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"runtime"
	"errors"
	"html/template"
)

var templates = template.Must(template.ParseFiles("index.html"))

const MESSAGE_QUEUE_SIZE = 10

var authKey = []byte("somesecretauth")
var store sessions.Store

var pool *Pool
var clients map[int64]*Client

var db *sql.DB

type Pool struct {
	in  chan *Client
	out chan *Room
}

type Client struct {
	id      int64
	in      chan string
	out     chan string
	retChan chan *Room
}

type Room struct {
	// id      int64
	client1 *Client
	client2 *Client
}

func (p *Pool) Pair() {
	for {
		c1, c2 := <-p.in, <-p.in

		fmt.Println("match found for ", c1.id, " and ", c2.id)

		// b := make([]byte, 8)
		// n, err := io.ReadFull(rand.Reader, b)
		// if err != nil || n != 8 {
		// 	return
		// }
		// crId, _ := binary.Varint(b)

		// fmt.Print

		room := &Room{c1, c2}

		c1.in, c2.in = c2.out, c1.out

		c1.retChan <- room
		c2.retChan <- room
	}
}

func newPool() *Pool {
	pool := &Pool{
		in:  make(chan *Client),
		out: make(chan *Room),
	}

	go pool.Pair()

	return pool
}

func UIDFromSession(w http.ResponseWriter, r *http.Request) (int64, error) {
	session, _ := store.Get(r, "session")
	userid := session.Values["userid"]

	fmt.Println(session.Values)
	if userid == nil {
		return 0, errors.New("no cookie set")
	}
	return userid.(int64), nil
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, _ = sql.Open("mysql", "root:pass@/suitup")
	defer db.Close()

	store = sessions.NewCookieStore(authKey)

	pool = newPool()
	clients = make(map[int64]*Client)

	http.HandleFunc("/", mainHandle)

	http.HandleFunc("/login", login)

	http.HandleFunc("/message/check", checkMessage)
	http.HandleFunc("/message/send", sendMessage)

	http.HandleFunc("/chatroom/join", joinChatRoom)
	http.HandleFunc("/chatroom/leave", leaveChatRoom)

	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("/home/suitup/hackmit/assets/"))))
	http.ListenAndServe(":8080", nil)
}

type IdQuery struct {
    Id            int64      `json:"id"`

}

func mainHandle(w http.ResponseWriter, r *http.Request) {
	templates.ExecuteTemplate(w, "index.html", nil)
}

func joinChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

  	fmt.Println("uid: ", uid)
	retChan := make(chan *Room)
	client := &Client{
		id:      uid,
		in:      nil,
		out:     make(chan string, MESSAGE_QUEUE_SIZE),
		retChan: retChan,
	}
	clients[uid] = client
	pool.in <- client

	fmt.Println("added ", uid, " to queue")
	chatroom := <- retChan

	fmt.Fprint(w, "{\"status\":\"success\",\"crid\":", chatroom.id, "}")
}

func leaveChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, _ := UIDFromSession(w, r)
	fmt.Fprint(w, uid)
}

func sendMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println(uid)

	message := r.FormValue("s")

	// message := r.PostFormValue("message")

	client := clients[uid]

	if client != nil {
		client.out <- message
		fmt.Fprint(w, "{\"status\":\"success\"}-", message)
	} else {
		fmt.Fprint(w, "{\"status\":\"failure\"}-", message)
	}
}

func checkMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	client := clients[uid]

	if client != nil {
		fmt.Println("client found")
		select {
		case message, ok := <- clients[uid].in:
			fmt.Println("message pulled from channel")
			if ok {
				fmt.Fprint(w, message)
			} else {
				fmt.Fprint(w, "")
			}
		default:
			fmt.Println("")
			fmt.Fprint(w, "")
		}

	} else {
		fmt.Println("client not found")
		fmt.Fprint(w, "")
	}
}

func login(w http.ResponseWriter, r *http.Request) {
	inputToken := r.FormValue("access_token")
	if len(inputToken) != 0 {
		uid := GetMe(inputToken)

		// row := db.QueryRow("SELECT id FROM users")
		row := db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
		iq := new(IdQuery)
		err := row.Scan(&iq.Id)

		if err != nil {
			_, err = db.Exec("insert into users (facebook_id, username, email, level, points) values (?, ?, ?, 0, 0)", uid, "", "")
			if err != nil {
				fmt.Fprint(w, "{\"status\":\"failure\"}")
				return
			} else {
				row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
				err = row.Scan(&iq.Id)
				if err != nil {
					fmt.Fprint(w, "{\"status\":\"failure\"}")
					return
				}
			}

		}

		fmt.Println("session-id: ", iq.Id)

		session, _ := store.Get(r, "session")
		session.Values["userid"] = iq.Id
		session.Save(r, w)

		fmt.Println(session.Values)

		fmt.Fprint(w, "{\"status\":\"success\"}")

	// 	if err == nil {
	// 		fmt.Fprint(w, "{\"status\":\"success\",\"uid\":", iq.Id, "}")
	// 	} else {
	// 		_, err = db.Exec("insert into users (facebook_id, username, email, level, points) values (?, ?, ?, 0, 0)", uid, "", "")
	// 		if err == nil {
	// 			row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
	// 			err = row.Scan(&iq.Id)
	// 			if err == nil {
	// 				fmt.Fprint(w, "{\"status\":\"success\"},\"uid\":", iq.Id, "}")
	// 			} else {
	// 				fmt.Fprint(w, "{\"status\":\"failure\"}")
	// 			}
	// 		} else {
	// 			fmt.Fprint(w, "{\"status\":\"failure\"}")
	// 		}
	// 	}
	// } else {
	// 	fmt.Fprint(w, "{\"status\":\"failure\"}")
	}
}

func readHttpBody(response *http.Response) string {
	bodyBuffer := make([]byte, 1000)
	var str string

	count, err := response.Body.Read(bodyBuffer)

	for ; count > 0; count, err = response.Body.Read(bodyBuffer) {

		if err != nil {

		}

		str += string(bodyBuffer[:count])
	}

	return str

}

func getUncachedResponse(uri string) (*http.Response, error) {
	request, err := http.NewRequest("GET", uri, nil)

	if err == nil {
		request.Header.Add("Cache-Control", "no-cache")

		client := new(http.Client)

		return client.Do(request)
	}

	if (err != nil) {
	}
	return nil, err

}

func GetMe(token string) string {
	response, err := getUncachedResponse("https://graph.facebook.com/me?access_token="+token)

	if err == nil {

		var jsonBlob interface{}

		responseBody := readHttpBody(response)

		if responseBody != "" {
			err = json.Unmarshal([]byte(responseBody), &jsonBlob)

			if err == nil {
				jsonObj := jsonBlob.(map[string]interface{})
				return jsonObj["id"].(string)
			}
		}
		return err.Error()
	}

	return err.Error()
}

func handleError(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>index.html<commit_after>package main

import (
	_ "time"
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"github.com/gorilla/sessions"
	"io"
	"net/http"
	"encoding/json"
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"runtime"
	"errors"
	"html/template"
)

var templates = template.Must(template.ParseFiles("index.html"))

const MESSAGE_QUEUE_SIZE = 10

var authKey = []byte("somesecretauth")
var store sessions.Store

var pool *Pool
var clients map[int64]*Client

var db *sql.DB

type Pool struct {
	in  chan *Client
	out chan *Room
}

type Client struct {
	id      int64
	in      chan string
	out     chan string
	retChan chan *Room
}

type Room struct {
	id      int64
	client1 *Client
	client2 *Client
}

func (p *Pool) Pair() {
	for {
		c1, c2 := <-p.in, <-p.in

		fmt.Println("match found for ", c1.id, " and ", c2.id)

		b := make([]byte, 8)
		n, err := io.ReadFull(rand.Reader, b)
		if err != nil || n != 8 {
			return
		}
		crId, _ := binary.Varint(b)

		fmt.Print

		room := &Room{c1, c2}

		c1.in, c2.in = c2.out, c1.out

		c1.retChan <- room
		c2.retChan <- room
	}
}

func newPool() *Pool {
	pool := &Pool{
		in:  make(chan *Client),
		out: make(chan *Room),
	}

	go pool.Pair()

	return pool
}

func UIDFromSession(w http.ResponseWriter, r *http.Request) (int64, error) {
	session, _ := store.Get(r, "session")
	userid := session.Values["userid"]

	fmt.Println(session.Values)
	if userid == nil {
		return 0, errors.New("no cookie set")
	}
	return userid.(int64), nil
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, _ = sql.Open("mysql", "root:pass@/suitup")
	defer db.Close()

	store = sessions.NewCookieStore(authKey)

	pool = newPool()
	clients = make(map[int64]*Client)

	http.HandleFunc("/", mainHandle)

	http.HandleFunc("/login", login)

	http.HandleFunc("/message/check", checkMessage)
	http.HandleFunc("/message/send", sendMessage)

	http.HandleFunc("/chatroom/join", joinChatRoom)
	http.HandleFunc("/chatroom/leave", leaveChatRoom)

	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("/home/suitup/hackmit/assets/"))))
	http.ListenAndServe(":8080", nil)
}

type IdQuery struct {
    Id            int64      `json:"id"`

}

func mainHandle(w http.ResponseWriter, r *http.Request) {
	templates.ExecuteTemplate(w, "index.html", nil)
}

func joinChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

  	fmt.Println("uid: ", uid)
	retChan := make(chan *Room)
	client := &Client{
		id:      uid,
		in:      nil,
		out:     make(chan string, MESSAGE_QUEUE_SIZE),
		retChan: retChan,
	}
	clients[uid] = client
	pool.in <- client

	fmt.Println("added ", uid, " to queue")
	chatroom := <- retChan

	fmt.Fprint(w, "{\"status\":\"success\",\"crid\":", chatroom.id, "}")
}

func leaveChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, _ := UIDFromSession(w, r)
	fmt.Fprint(w, uid)
}

func sendMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println(uid)

	message := r.FormValue("s")

	// message := r.PostFormValue("message")

	client := clients[uid]

	if client != nil {
		client.out <- message
		fmt.Fprint(w, "{\"status\":\"success\"}-", message)
	} else {
		fmt.Fprint(w, "{\"status\":\"failure\"}-", message)
	}
}

func checkMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	client := clients[uid]

	if client != nil {
		fmt.Println("client found")
		select {
		case message, ok := <- clients[uid].in:
			fmt.Println("message pulled from channel")
			if ok {
				fmt.Fprint(w, message)
			} else {
				fmt.Fprint(w, "")
			}
		default:
			fmt.Println("")
			fmt.Fprint(w, "")
		}

	} else {
		fmt.Println("client not found")
		fmt.Fprint(w, "")
	}
}

func login(w http.ResponseWriter, r *http.Request) {
	inputToken := r.FormValue("access_token")
	if len(inputToken) != 0 {
		uid := GetMe(inputToken)

		// row := db.QueryRow("SELECT id FROM users")
		row := db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
		iq := new(IdQuery)
		err := row.Scan(&iq.Id)

		if err != nil {
			_, err = db.Exec("insert into users (facebook_id, username, email, level, points) values (?, ?, ?, 0, 0)", uid, "", "")
			if err != nil {
				fmt.Fprint(w, "{\"status\":\"failure\"}")
				return
			} else {
				row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
				err = row.Scan(&iq.Id)
				if err != nil {
					fmt.Fprint(w, "{\"status\":\"failure\"}")
					return
				}
			}

		}

		fmt.Println("session-id: ", iq.Id)

		session, _ := store.Get(r, "session")
		session.Values["userid"] = iq.Id
		session.Save(r, w)

		fmt.Println(session.Values)

		fmt.Fprint(w, "{\"status\":\"success\"}")

	// 	if err == nil {
	// 		fmt.Fprint(w, "{\"status\":\"success\",\"uid\":", iq.Id, "}")
	// 	} else {
	// 		_, err = db.Exec("insert into users (facebook_id, username, email, level, points) values (?, ?, ?, 0, 0)", uid, "", "")
	// 		if err == nil {
	// 			row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
	// 			err = row.Scan(&iq.Id)
	// 			if err == nil {
	// 				fmt.Fprint(w, "{\"status\":\"success\"},\"uid\":", iq.Id, "}")
	// 			} else {
	// 				fmt.Fprint(w, "{\"status\":\"failure\"}")
	// 			}
	// 		} else {
	// 			fmt.Fprint(w, "{\"status\":\"failure\"}")
	// 		}
	// 	}
	// } else {
	// 	fmt.Fprint(w, "{\"status\":\"failure\"}")
	}
}

func readHttpBody(response *http.Response) string {
	bodyBuffer := make([]byte, 1000)
	var str string

	count, err := response.Body.Read(bodyBuffer)

	for ; count > 0; count, err = response.Body.Read(bodyBuffer) {

		if err != nil {

		}

		str += string(bodyBuffer[:count])
	}

	return str

}

func getUncachedResponse(uri string) (*http.Response, error) {
	request, err := http.NewRequest("GET", uri, nil)

	if err == nil {
		request.Header.Add("Cache-Control", "no-cache")

		client := new(http.Client)

		return client.Do(request)
	}

	if (err != nil) {
	}
	return nil, err

}

func GetMe(token string) string {
	response, err := getUncachedResponse("https://graph.facebook.com/me?access_token="+token)

	if err == nil {

		var jsonBlob interface{}

		responseBody := readHttpBody(response)

		if responseBody != "" {
			err = json.Unmarshal([]byte(responseBody), &jsonBlob)

			if err == nil {
				jsonObj := jsonBlob.(map[string]interface{})
				return jsonObj["id"].(string)
			}
		}
		return err.Error()
	}

	return err.Error()
}

func handleError(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>/*
* Open-Falcon
*
* Copyright (c) 2014-2018 Xiaomi, Inc. All Rights Reserved.
*
* This product is licensed to you under the Apache License, Version 2.0 (the "License").
* You may not use this product except in compliance with the License.
*
* This product may include a number of subcomponents with separate copyright notices
* and license terms. Your use of these subcomponents is subject to the terms and
* conditions of the subcomponent's license, as noted in the LICENSE file.
 */

package main

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/open-falcon/mymon/common"

	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
)

// ShowGlobalStatus execute mysql query `SHOW GLOBAL STATUS`
func ShowGlobalStatus(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	return parseMySQLStatus(conf, db, "SHOW /*!50001 GLOBAL */ STATUS")
}

// ShowGlobalVariables execute mysql query `SHOW GLOBAL VARIABLES`
func ShowGlobalVariables(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	return parseMySQLStatus(conf, db, "SHOW /*!50001 GLOBAL */ VARIABLES")
}

// ShowProcesslist execute mysql query `SHOW FULL PROCESSLIST`
func ShowProcesslist(conf *common.Config, db mysql.Conn) error {
	var note string
	rows, _, err := db.Query("SHOW FULL PROCESSLIST")
	if err != nil {
		Log.Debug("get processlist error: %+v", err)
		return err
	}
	for _, row := range rows {
		note += fmt.Sprintf("%s\t%d\t%s\t%s\t%s\t%s\t%d\t%s\t%s\n",
			time.Now().Format("2006-01-02 15:04:05"), row.Int(0), row.Str(1), row.Str(2),
			row.Str(3), row.Str(4), row.Int(5), row.Str(6), row.Str(7))
	}

	fileNameDay, fileNameOldDay := common.GetFileNameDayAndOldDay(conf, "process")
	err = Snapshot(conf, note, fileNameDay, fileNameOldDay)
	return err
}

// ShowInnodbStatus execute mysql query `SHOW SHOW /*!50000 ENGINE*/ INNODB STATUS`
func ShowInnodbStatus(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	status, _, err := db.QueryFirst("SHOW /*!50000 ENGINE*/ INNODB STATUS")
	if err != nil {
		Log.Debug("show innodb status error: %+v", err)
		return nil, err
	}
	allStatus := status.Str(2)
	fileNameDay, fileNameOldDay := common.GetFileNameDayAndOldDay(conf, "innodb")
	err = Snapshot(conf, allStatus, fileNameDay, fileNameOldDay)
	if err != nil {
		Log.Debug("write snapshot error: %+v", err)
		return nil, err
	}

	rows := strings.Split(allStatus, "\n")
	return parseInnodbStatus(conf, rows)
}

// ShowBinaryLogs execute mysql query `SHOW BINARY LOGS`
func ShowBinaryLogs(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	var sum int
	binlogFileCounts := NewMetric(conf, "binlog_file_counts")
	binlogFileSize := NewMetric(conf, "binlog_file_size")

	rows, res, err := db.Query("SHOW BINARY LOGS")
	if err != nil {
		// Received #1381 error from MySQL server: "You are not using binary logging"
		Log.Debug("SHOW BINARY LOGS Error: %s", err.Error())
		return []*MetaData{binlogFileCounts, binlogFileSize}, nil
	}

	for _, row := range rows {
		size, _ := strconv.Atoi(row.Str(res.Map("File_size")))
		sum += size
	}
	binlogFileCounts.SetValue(len(rows))
	binlogFileSize.SetValue(sum)
	return []*MetaData{binlogFileCounts, binlogFileSize}, err
}

// ShowSlaveStatus get all slave status of mysql serves
func ShowSlaveStatus(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	// Check IsSlave
	row, res, err := db.QueryFirst("SHOW SLAVE STATUS")
	if err != nil {
		IsSlave = -1
		return nil, err
	}

	if row != nil {
		IsSlave = 1
	} else {
		IsSlave = 0
	}

	Tag = GetTag(conf)
	isSlaveMetric := NewMetric(conf, "Is_slave")
	isSlaveMetric.SetValue(IsSlave)

	// be master
	if IsSlave == 0 {
		// Master_is_readonly VS master_is_read_only for version compatible, ugly
		masterReadOnly, err := ShowOtherMetric(conf, db, "Master_is_readonly")
		if err != nil {
			Log.Debug("get Master_is_readonly metric error: %+v", err)
			return nil, err
		}
		masterReadOnly2, err := ShowOtherMetric(conf, db, "master_is_read_only")
		if err != nil {
			Log.Debug("get master_is_readonly metric error: %+v", err)
			return nil, err
		}
		innodbStatsOnMetadata, err := ShowOtherMetric(conf, db, "innodb_stats_on_metadata")
		if err != nil {
			// mysql < 5.5 doesn't have this variable
			Log.Debug("get innodb_stats_on_metadata metric error: %+v", err)
			return nil, err
		}
		return []*MetaData{isSlaveMetric, masterReadOnly, masterReadOnly2, innodbStatsOnMetadata}, nil
	}

	// be slave
	ioDelay, err := ShowOtherMetric(conf, db, "io_thread_delay")
	if err != nil {
		Log.Debug("get io_thread_delay metric error: %+v", err)
		return nil, err
	}
	slaveReadOnly, err := ShowOtherMetric(conf, db, "slave_is_read_only")
	if err != nil {
		Log.Debug("get slave_is_read_only metric error: %+v", err)
		return nil, err
	}
	heartbeat, err := ShowOtherMetric(conf, db, "Heartbeats_Behind_Master")
	if err != nil {
		// mysql.heartbeat table not necessary exist if you don't care about heartbeat
		// bypass heartbeat table not exist error
		Log.Debug("Heartbeats_Behind_Master: %s", err.Error())
		err = nil
	}
	data := make([]*MetaData, len(SlaveStatus))
	for i, s := range SlaveStatus {
		data[i] = NewMetric(conf, s)
		switch s {
		case "Slave_SQL_Running", "Slave_IO_Running":
			data[i].SetValue(0)
			v := row.Str(res.Map(s))
			if v == "Yes" {
				data[i].SetValue(1)
			}
		default:
			pos := res.Map(s)
			if pos > 0 {
				v, err := row.Int64Err(pos)
				if err != nil {
					data[i].SetValue(-1)
				} else {
					data[i].SetValue(v)
				}
			}
		}
	}
	return append(data, []*MetaData{isSlaveMetric, ioDelay, slaveReadOnly, heartbeat}...), nil
}

// ShowOtherMetric all other metric will add in this func
func ShowOtherMetric(conf *common.Config, db mysql.Conn, metric string) (*MetaData, error) {
	var err error
	var row mysql.Row
	newMetaData := NewMetric(conf, metric)
	switch metric {
	case "master_is_read_only", "slave_is_read_only", "Master_is_readonly":
		newMetaData.SetValue(IsReadOnly)
	case "innodb_stats_on_metadata":
		row, _, err = db.QueryFirst("SELECT /*!50504 @@GLOBAL.innodb_stats_on_metadata,*/ -1;")
		newMetaData.SetValue(row.Int(0))
	case "io_thread_delay":
		var res mysql.Result
		row, res, err = db.QueryFirst("SHOW SLAVE STATUS")
		if bytes.Equal([]byte(row.Str(res.Map("Master_Log_File"))), []byte(row.Str(res.Map("Relay_Master_Log_File")))) {
			newMetaData.SetValue(0)
		} else {
			masterLogFile := common.GetLastNum(row.Str(res.Map("Master_Log_File")), ".")
			relayMasterLogFile := common.GetLastNum(row.Str(res.Map("Relay_Master_Log_File")), ".")
			newMetaData.SetValue(masterLogFile - relayMasterLogFile)
			if masterLogFile < 0 || relayMasterLogFile < 0 {
				newMetaData.SetValue(-1)
			}
		}
	case "Heartbeats_Behind_Master":
		row, _, err = db.QueryFirst("select ts from mysql.heartbeat limit 1")
		// when row is empty, err is nil either
		if err != nil || len(row) == 0 {
			newMetaData.SetValue(-1)
		} else {
			localTimezone, _ := time.LoadLocation("Local")
			heartbeatTimeStr := row.Str(0)
			b := strings.Replace(heartbeatTimeStr, "T", " ", 1)
			t, _ := time.ParseInLocation("2006-01-02 15:04:05", strings.Split(b, ".")[0], localTimezone)
			heartbeatTimestamp := t.Unix()
			currentTimestamp := time.Now().Unix()
			newMetaData.SetValue(currentTimestamp - heartbeatTimestamp)
		}
	}

	return newMetaData, err
}

func parseMySQLStatus(conf *common.Config, db mysql.Conn, sql string) ([]*MetaData, error) {
	rows, _, err := db.Query(sql)
	if err != nil {
		Log.Debug("sql query: %s error: %+v", sql, err)
		return nil, err
	}

	data := make([]*MetaData, len(rows))
	i := 0
	for _, row := range rows {
		k := row.Str(0)
		//TODO: many variables values "ON|OFF" but not int should be fetched, like super_read_only
		v, err := row.Int64Err(1)
		if err != nil {
			continue
		}

		data[i] = NewMetric(conf, k)
		data[i].SetValue(v)
		i++
	}
	return data[:i], nil
}

func parseInnodbSection(
	conf *common.Config, row string, section string,
	pdata *[]*MetaData, longTranTime *int) error {
	var err error
	switch section {
	case "TRANSACTIONS":
		txPrefixes := []string{"ACTIVE ", "ACTIVE (PREPARED) "}
		for _, txPrefix := range txPrefixes {
			if strings.Contains(row, txPrefix) {
				var tmpLongTransactionTime int
				secString := strings.Split(strings.Split(
					row, txPrefix)[1], " sec")[0]
				tmpLongTransactionTime, err = strconv.Atoi(secString)
				if err != nil {
					continue
				}
				if tmpLongTransactionTime > *longTranTime {
					*longTranTime = tmpLongTransactionTime
				}
				break
			}
		}
		if err != nil {
			Log.Warn(err.Error(), "Longest_transaction metric parse Error: ", row)
		}
		if strings.Contains(row, "History list length") {
			hisListLengthStr := strings.Split(row, "length ")[1]
			hisListLength, _ := strconv.Atoi(hisListLengthStr)
			HistoryListLength := NewMetric(conf, "History_list_length")
			HistoryListLength.SetValue(hisListLength)
			*pdata = append(*pdata, HistoryListLength)
		}
	case "SEMAPHORES":
		matches := regexp.MustCompile(`^Mutex spin waits\s+(\d+),\s+rounds\s+(\d+),\s+OS waits\s+(\d+)`).FindStringSubmatch(row)
		if len(matches) == 4 {
			spinWaits, _ := strconv.Atoi(matches[1])
			innodbMutexSpinWaits := NewMetric(conf, "Innodb_mutex_spin_waits")
			innodbMutexSpinWaits.SetValue(spinWaits)
			*pdata = append(*pdata, innodbMutexSpinWaits)

			spinRounds, _ := strconv.Atoi(matches[2])
			InnodbMutexSpinRounds := NewMetric(conf, "Innodb_mutex_spin_rounds")
			InnodbMutexSpinRounds.SetValue(spinRounds)
			*pdata = append(*pdata, InnodbMutexSpinRounds)

			osWaits, _ := strconv.Atoi(matches[3])
			InnodbMutexOsWaits := NewMetric(conf, "Innodb_mutex_os_waits")
			InnodbMutexOsWaits.SetValue(osWaits)
			*pdata = append(*pdata, InnodbMutexOsWaits)
		}
	}

	return err
}

func parseInnodbStatus(conf *common.Config, rows []string) ([]*MetaData, error) {
	var section string
	longTranTime := 0
	var err error
	var data []*MetaData
	for _, row := range rows {
		switch row {
		case "BACKGROUND THREAD":
			section = row
			continue
		case "DEAD LOCK ERRORS":
			section = row
			continue
		case "LATEST DETECTED DEADLOCK":
			section = row
			continue
		case "FOREIGN KEY CONSTRAINT ERRORS", "LATEST FOREIGN KEY ERROR":
			section = row
			continue
		case "SEMAPHORES":
			section = row
			continue
		case "TRANSACTIONS":
			section = row
			continue
		case "FILE I/O":
			section = row
			continue
		case "INSERT BUFFER AND ADAPTIVE HASH INDEX":
			section = row
			continue
		case "LOG":
			section = row
			continue
		case "BUFFER POOL AND MEMORY":
			section = row
			continue
		case "ROW OPERATIONS":
			section = row
			continue
		}
		err = parseInnodbSection(conf, row, section, &data, &longTranTime)
		if err != nil {
			Log.Debug("parse innodb section error: %+v", err)
			return nil, err
		}
	}
	longTranMetric := NewMetric(conf, "longest_transaction")
	longTranMetric.SetValue(longTranTime)
	data = append(data, longTranMetric)
	return data, nil
}
<commit_msg>fix:func ShowBinaryLogs return when has error<commit_after>/*
* Open-Falcon
*
* Copyright (c) 2014-2018 Xiaomi, Inc. All Rights Reserved.
*
* This product is licensed to you under the Apache License, Version 2.0 (the "License").
* You may not use this product except in compliance with the License.
*
* This product may include a number of subcomponents with separate copyright notices
* and license terms. Your use of these subcomponents is subject to the terms and
* conditions of the subcomponent's license, as noted in the LICENSE file.
 */

package main

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/open-falcon/mymon/common"

	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
)

// ShowGlobalStatus execute mysql query `SHOW GLOBAL STATUS`
func ShowGlobalStatus(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	return parseMySQLStatus(conf, db, "SHOW /*!50001 GLOBAL */ STATUS")
}

// ShowGlobalVariables execute mysql query `SHOW GLOBAL VARIABLES`
func ShowGlobalVariables(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	return parseMySQLStatus(conf, db, "SHOW /*!50001 GLOBAL */ VARIABLES")
}

// ShowProcesslist execute mysql query `SHOW FULL PROCESSLIST`
func ShowProcesslist(conf *common.Config, db mysql.Conn) error {
	var note string
	rows, _, err := db.Query("SHOW FULL PROCESSLIST")
	if err != nil {
		Log.Debug("get processlist error: %+v", err)
		return err
	}
	for _, row := range rows {
		note += fmt.Sprintf("%s\t%d\t%s\t%s\t%s\t%s\t%d\t%s\t%s\n",
			time.Now().Format("2006-01-02 15:04:05"), row.Int(0), row.Str(1), row.Str(2),
			row.Str(3), row.Str(4), row.Int(5), row.Str(6), row.Str(7))
	}

	fileNameDay, fileNameOldDay := common.GetFileNameDayAndOldDay(conf, "process")
	err = Snapshot(conf, note, fileNameDay, fileNameOldDay)
	return err
}

// ShowInnodbStatus execute mysql query `SHOW SHOW /*!50000 ENGINE*/ INNODB STATUS`
func ShowInnodbStatus(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	status, _, err := db.QueryFirst("SHOW /*!50000 ENGINE*/ INNODB STATUS")
	if err != nil {
		Log.Debug("show innodb status error: %+v", err)
		return nil, err
	}
	allStatus := status.Str(2)
	fileNameDay, fileNameOldDay := common.GetFileNameDayAndOldDay(conf, "innodb")
	err = Snapshot(conf, allStatus, fileNameDay, fileNameOldDay)
	if err != nil {
		Log.Debug("write snapshot error: %+v", err)
		return nil, err
	}

	rows := strings.Split(allStatus, "\n")
	return parseInnodbStatus(conf, rows)
}

// ShowBinaryLogs execute mysql query `SHOW BINARY LOGS`
func ShowBinaryLogs(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	var sum int
	binlogFileCounts := NewMetric(conf, "binlog_file_counts")
	binlogFileSize := NewMetric(conf, "binlog_file_size")

	rows, res, err := db.Query("SHOW BINARY LOGS")
	if err != nil {
		// Received #1381 error from MySQL server: "You are not using binary logging"
		Log.Debug("SHOW BINARY LOGS Error: %s", err.Error())
		return nil, err
	}

	for _, row := range rows {
		size, _ := strconv.Atoi(row.Str(res.Map("File_size")))
		sum += size
	}
	binlogFileCounts.SetValue(len(rows))
	binlogFileSize.SetValue(sum)
	return []*MetaData{binlogFileCounts, binlogFileSize}, err
}

// ShowSlaveStatus get all slave status of mysql serves
func ShowSlaveStatus(conf *common.Config, db mysql.Conn) ([]*MetaData, error) {
	// Check IsSlave
	row, res, err := db.QueryFirst("SHOW SLAVE STATUS")
	if err != nil {
		IsSlave = -1
		return nil, err
	}

	if row != nil {
		IsSlave = 1
	} else {
		IsSlave = 0
	}

	Tag = GetTag(conf)
	isSlaveMetric := NewMetric(conf, "Is_slave")
	isSlaveMetric.SetValue(IsSlave)

	// be master
	if IsSlave == 0 {
		// Master_is_readonly VS master_is_read_only for version compatible, ugly
		masterReadOnly, err := ShowOtherMetric(conf, db, "Master_is_readonly")
		if err != nil {
			Log.Debug("get Master_is_readonly metric error: %+v", err)
			return nil, err
		}
		masterReadOnly2, err := ShowOtherMetric(conf, db, "master_is_read_only")
		if err != nil {
			Log.Debug("get master_is_readonly metric error: %+v", err)
			return nil, err
		}
		innodbStatsOnMetadata, err := ShowOtherMetric(conf, db, "innodb_stats_on_metadata")
		if err != nil {
			// mysql < 5.5 doesn't have this variable
			Log.Debug("get innodb_stats_on_metadata metric error: %+v", err)
			return nil, err
		}
		return []*MetaData{isSlaveMetric, masterReadOnly, masterReadOnly2, innodbStatsOnMetadata}, nil
	}

	// be slave
	ioDelay, err := ShowOtherMetric(conf, db, "io_thread_delay")
	if err != nil {
		Log.Debug("get io_thread_delay metric error: %+v", err)
		return nil, err
	}
	slaveReadOnly, err := ShowOtherMetric(conf, db, "slave_is_read_only")
	if err != nil {
		Log.Debug("get slave_is_read_only metric error: %+v", err)
		return nil, err
	}
	heartbeat, err := ShowOtherMetric(conf, db, "Heartbeats_Behind_Master")
	if err != nil {
		// mysql.heartbeat table not necessary exist if you don't care about heartbeat
		// bypass heartbeat table not exist error
		Log.Debug("Heartbeats_Behind_Master: %s", err.Error())
		err = nil
	}
	data := make([]*MetaData, len(SlaveStatus))
	for i, s := range SlaveStatus {
		data[i] = NewMetric(conf, s)
		switch s {
		case "Slave_SQL_Running", "Slave_IO_Running":
			data[i].SetValue(0)
			v := row.Str(res.Map(s))
			if v == "Yes" {
				data[i].SetValue(1)
			}
		default:
			pos := res.Map(s)
			if pos > 0 {
				v, err := row.Int64Err(pos)
				if err != nil {
					data[i].SetValue(-1)
				} else {
					data[i].SetValue(v)
				}
			}
		}
	}
	return append(data, []*MetaData{isSlaveMetric, ioDelay, slaveReadOnly, heartbeat}...), nil
}

// ShowOtherMetric all other metric will add in this func
func ShowOtherMetric(conf *common.Config, db mysql.Conn, metric string) (*MetaData, error) {
	var err error
	var row mysql.Row
	newMetaData := NewMetric(conf, metric)
	switch metric {
	case "master_is_read_only", "slave_is_read_only", "Master_is_readonly":
		newMetaData.SetValue(IsReadOnly)
	case "innodb_stats_on_metadata":
		row, _, err = db.QueryFirst("SELECT /*!50504 @@GLOBAL.innodb_stats_on_metadata,*/ -1;")
		newMetaData.SetValue(row.Int(0))
	case "io_thread_delay":
		var res mysql.Result
		row, res, err = db.QueryFirst("SHOW SLAVE STATUS")
		if bytes.Equal([]byte(row.Str(res.Map("Master_Log_File"))), []byte(row.Str(res.Map("Relay_Master_Log_File")))) {
			newMetaData.SetValue(0)
		} else {
			masterLogFile := common.GetLastNum(row.Str(res.Map("Master_Log_File")), ".")
			relayMasterLogFile := common.GetLastNum(row.Str(res.Map("Relay_Master_Log_File")), ".")
			newMetaData.SetValue(masterLogFile - relayMasterLogFile)
			if masterLogFile < 0 || relayMasterLogFile < 0 {
				newMetaData.SetValue(-1)
			}
		}
	case "Heartbeats_Behind_Master":
		row, _, err = db.QueryFirst("select ts from mysql.heartbeat limit 1")
		// when row is empty, err is nil either
		if err != nil || len(row) == 0 {
			newMetaData.SetValue(-1)
		} else {
			localTimezone, _ := time.LoadLocation("Local")
			heartbeatTimeStr := row.Str(0)
			b := strings.Replace(heartbeatTimeStr, "T", " ", 1)
			t, _ := time.ParseInLocation("2006-01-02 15:04:05", strings.Split(b, ".")[0], localTimezone)
			heartbeatTimestamp := t.Unix()
			currentTimestamp := time.Now().Unix()
			newMetaData.SetValue(currentTimestamp - heartbeatTimestamp)
		}
	}

	return newMetaData, err
}

func parseMySQLStatus(conf *common.Config, db mysql.Conn, sql string) ([]*MetaData, error) {
	rows, _, err := db.Query(sql)
	if err != nil {
		Log.Debug("sql query: %s error: %+v", sql, err)
		return nil, err
	}

	data := make([]*MetaData, len(rows))
	i := 0
	for _, row := range rows {
		k := row.Str(0)
		//TODO: many variables values "ON|OFF" but not int should be fetched, like super_read_only
		v, err := row.Int64Err(1)
		if err != nil {
			continue
		}

		data[i] = NewMetric(conf, k)
		data[i].SetValue(v)
		i++
	}
	return data[:i], nil
}

func parseInnodbSection(
	conf *common.Config, row string, section string,
	pdata *[]*MetaData, longTranTime *int) error {
	var err error
	switch section {
	case "TRANSACTIONS":
		txPrefixes := []string{"ACTIVE ", "ACTIVE (PREPARED) "}
		for _, txPrefix := range txPrefixes {
			if strings.Contains(row, txPrefix) {
				var tmpLongTransactionTime int
				secString := strings.Split(strings.Split(
					row, txPrefix)[1], " sec")[0]
				tmpLongTransactionTime, err = strconv.Atoi(secString)
				if err != nil {
					continue
				}
				if tmpLongTransactionTime > *longTranTime {
					*longTranTime = tmpLongTransactionTime
				}
				break
			}
		}
		if err != nil {
			Log.Warn(err.Error(), "Longest_transaction metric parse Error: ", row)
		}
		if strings.Contains(row, "History list length") {
			hisListLengthStr := strings.Split(row, "length ")[1]
			hisListLength, _ := strconv.Atoi(hisListLengthStr)
			HistoryListLength := NewMetric(conf, "History_list_length")
			HistoryListLength.SetValue(hisListLength)
			*pdata = append(*pdata, HistoryListLength)
		}
	case "SEMAPHORES":
		matches := regexp.MustCompile(`^Mutex spin waits\s+(\d+),\s+rounds\s+(\d+),\s+OS waits\s+(\d+)`).FindStringSubmatch(row)
		if len(matches) == 4 {
			spinWaits, _ := strconv.Atoi(matches[1])
			innodbMutexSpinWaits := NewMetric(conf, "Innodb_mutex_spin_waits")
			innodbMutexSpinWaits.SetValue(spinWaits)
			*pdata = append(*pdata, innodbMutexSpinWaits)

			spinRounds, _ := strconv.Atoi(matches[2])
			InnodbMutexSpinRounds := NewMetric(conf, "Innodb_mutex_spin_rounds")
			InnodbMutexSpinRounds.SetValue(spinRounds)
			*pdata = append(*pdata, InnodbMutexSpinRounds)

			osWaits, _ := strconv.Atoi(matches[3])
			InnodbMutexOsWaits := NewMetric(conf, "Innodb_mutex_os_waits")
			InnodbMutexOsWaits.SetValue(osWaits)
			*pdata = append(*pdata, InnodbMutexOsWaits)
		}
	}

	return err
}

func parseInnodbStatus(conf *common.Config, rows []string) ([]*MetaData, error) {
	var section string
	longTranTime := 0
	var err error
	var data []*MetaData
	for _, row := range rows {
		switch row {
		case "BACKGROUND THREAD":
			section = row
			continue
		case "DEAD LOCK ERRORS":
			section = row
			continue
		case "LATEST DETECTED DEADLOCK":
			section = row
			continue
		case "FOREIGN KEY CONSTRAINT ERRORS", "LATEST FOREIGN KEY ERROR":
			section = row
			continue
		case "SEMAPHORES":
			section = row
			continue
		case "TRANSACTIONS":
			section = row
			continue
		case "FILE I/O":
			section = row
			continue
		case "INSERT BUFFER AND ADAPTIVE HASH INDEX":
			section = row
			continue
		case "LOG":
			section = row
			continue
		case "BUFFER POOL AND MEMORY":
			section = row
			continue
		case "ROW OPERATIONS":
			section = row
			continue
		}
		err = parseInnodbSection(conf, row, section, &data, &longTranTime)
		if err != nil {
			Log.Debug("parse innodb section error: %+v", err)
			return nil, err
		}
	}
	longTranMetric := NewMetric(conf, "longest_transaction")
	longTranMetric.SetValue(longTranTime)
	data = append(data, longTranMetric)
	return data, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"runtime/pprof"
	"sort"
	"text/tabwriter"
)

const VERSION = `0.1`

var languages = []Language{
	Language{"C", mExt(".c", ".h"), cComments},
	Language{"C++", mExt(".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx"), cComments},
	Language{"Go", mExt(".go"), cComments},
	Language{"Haskell", mExt(".hs", ".lhs"), noComments},
	Language{"Perl", mExt(".pl", ".pm"), shComments},
	Language{"Python", mExt(".py"), noComments},
	Language{"Lisp", mExt(".lsp"), semiComments},
	Language{"Make", mName("makefile", "Makefile", "MAKEFILE"), shComments},
	Language{"HTML", mExt(".htm", ".html", ".xhtml"), noComments},
}

type Commenter struct {
	LineComment  string
	StartComment string
	EndComment   string
	Nesting      bool
}

var (
	noComments = Commenter{"\000", "\000", "\000", false}
	cComments  = Commenter{`//`, `/*`, `*/`, false}
	shComments = Commenter{`#`, "\000", "\000", false}
	semiComments = Commenter{`;`, "\000", "\000", false}
)

type Language struct {
	Namer
	Matcher
	Commenter
}

func (l Language) Update(c []byte, s *Stats) {
	s.FileCount++

	inComment := 0 // this is an int for nesting
	inLComment := false
	blank := true
	lc := []byte(l.LineComment)
	sc := []byte(l.StartComment)
	ec := []byte(l.EndComment)
	lp, sp, ep := 0, 0, 0

	for _, b := range c {
		if b == lc[lp] && !(inComment > 0) {
			lp++
			if lp == len(lc) {
				inLComment = true
				lp = 0
			}
		} else { lp = 0 }
		if b == sc[sp] && !inLComment {
			sp++
			if sp == len(sc) {
				inComment++
				if inComment > 1 && !l.Nesting {
					inComment = 1
				}
				sp = 0
			}
		} else { sp = 0 }
		if b == ec[ep] && !inLComment && inComment > 0 {
			ep++
			if ep == len(ec) {
				inComment--
				ep = 0
			}
		} else { ep = 0 }

		if b != byte(' ') && b != byte('\t') && b != byte('\n') {
			blank = false
		}

		// Note that lines with both code and comment count towards
		// each, but are not counted twice in the total.
		if b == byte('\n') {
			s.TotalLines++
			if blank {
				s.BlankLines++
			}
			if inComment > 0 || inLComment {
				if blank {
					s.CodeLines++
				}
				inLComment = false
				s.CommentLines++
			} else { s.CodeLines++ }
			blank = true
			continue
		}
	}
}

type Namer string

func (l Namer) Name() string { return string(l) }

type Matcher func(string) bool

func (m Matcher) Match(fname string) bool { return m(fname) }

func mExt(exts ...string) Matcher {
	return func(fname string) bool {
		for _, ext := range exts {
			if ext == path.Ext(fname) {
				return true
			}
		}
		return false
	}
}

func mName(names ...string) Matcher {
	return func(fname string) bool {
		for _, name := range names {
			if name == path.Base(fname) {
				return true
			}
		}
		return false
	}
}

type Stats struct {
	FileCount    int
	TotalLines   int
	CodeLines    int
	BlankLines   int
	CommentLines int
}

var info = map[string]*Stats{}

func handleFile(fname string) {
	var l Language
	ok := false
	for _, lang := range languages {
		if lang.Match(fname) {
			ok = true
			l = lang
			break
		}
	}
	if !ok {
		return // ignore this file
	}
	i, ok := info[l.Name()]
	if !ok {
		i = &Stats{}
		info[l.Name()] = i
	}
	c, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  ! %s\n", fname)
		return
	}
	l.Update(c, i)
}

var files []string

func add(n string) {
	fi, err := os.Stat(n)
	if err != nil {
		goto invalid
	}
	if fi.IsDir() {
		fs, err := ioutil.ReadDir(n)
		if err != nil {
			goto invalid
		}
		for _, f := range fs {
			if f.Name()[0] != '.' {
				add(path.Join(n, f.Name()))
			}
		}
		return
	}
	if fi.Mode()&os.ModeType == 0 {
		files = append(files, n)
		return
	}

	println(fi.Mode())

invalid:
	fmt.Fprintf(os.Stderr, "  ! %s\n", n)
}

type LData []LResult

func (d LData) Len() int { return len(d) }

func (d LData) Less(i, j int) bool { return d[i].CodeLines > d[j].CodeLines }

func (d LData) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

type LResult struct {
	Name string
	FileCount int
	CodeLines int
	CommentLines int
	BlankLines int
	TotalLines int
}

func printInfo() {
	w := tabwriter.NewWriter(os.Stdout, 2, 8, 2, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "Language\tFiles\tCode\tComment\tBlank\tTotal\t")
	d := LData([]LResult{})
	for n, i := range info {
		d = append(d, LResult{
			n,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines,
		})
	}
	sort.Sort(d)
	for _, i := range d {
		fmt.Fprintf(
			w,
			"%s\t%d\t%d\t%d\t%d\t%d\t\n",
			i.Name,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines)
	}
	w.Flush()
}

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	version = flag.Bool("V", false, "display version info and exit")
)

func main() {
	flag.Parse()
	if *version {
		fmt.Printf("sloc %s\n", VERSION)
		return
	}
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
			return
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	args := flag.Args()
	if len(args) == 0 {
		args = append(args, `.`)
	}

	for _, n := range args {
		add(n)
	}

	for _, f := range files {
		handleFile(f)
	}
	printInfo()
}
<commit_msg>Adding 'Shell' and 'Bash'<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"runtime/pprof"
	"sort"
	"text/tabwriter"
)

const VERSION = `0.1`

var languages = []Language{
	Language{"C", mExt(".c", ".h"), cComments},
	Language{"C++", mExt(".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx"), cComments},
	Language{"Go", mExt(".go"), cComments},
	Language{"Haskell", mExt(".hs", ".lhs"), noComments},
	Language{"Perl", mExt(".pl", ".pm"), shComments},

	Language{"Shell", mExt(".sh"), shComments},
	Language{"Bash", mExt(".bash"), shComments},

	Language{"Python", mExt(".py"), noComments},
	Language{"Lisp", mExt(".lsp"), semiComments},
	Language{"Make", mName("makefile", "Makefile", "MAKEFILE"), shComments},
	Language{"HTML", mExt(".htm", ".html", ".xhtml"), noComments},
}

type Commenter struct {
	LineComment  string
	StartComment string
	EndComment   string
	Nesting      bool
}

var (
	noComments = Commenter{"\000", "\000", "\000", false}
	cComments  = Commenter{`//`, `/*`, `*/`, false}
	shComments = Commenter{`#`, "\000", "\000", false}
	semiComments = Commenter{`;`, "\000", "\000", false}
)

type Language struct {
	Namer
	Matcher
	Commenter
}

func (l Language) Update(c []byte, s *Stats) {
	s.FileCount++

	inComment := 0 // this is an int for nesting
	inLComment := false
	blank := true
	lc := []byte(l.LineComment)
	sc := []byte(l.StartComment)
	ec := []byte(l.EndComment)
	lp, sp, ep := 0, 0, 0

	for _, b := range c {
		if b == lc[lp] && !(inComment > 0) {
			lp++
			if lp == len(lc) {
				inLComment = true
				lp = 0
			}
		} else { lp = 0 }
		if b == sc[sp] && !inLComment {
			sp++
			if sp == len(sc) {
				inComment++
				if inComment > 1 && !l.Nesting {
					inComment = 1
				}
				sp = 0
			}
		} else { sp = 0 }
		if b == ec[ep] && !inLComment && inComment > 0 {
			ep++
			if ep == len(ec) {
				inComment--
				ep = 0
			}
		} else { ep = 0 }

		if b != byte(' ') && b != byte('\t') && b != byte('\n') {
			blank = false
		}

		// Note that lines with both code and comment count towards
		// each, but are not counted twice in the total.
		if b == byte('\n') {
			s.TotalLines++
			if blank {
				s.BlankLines++
			}
			if inComment > 0 || inLComment {
				if blank {
					s.CodeLines++
				}
				inLComment = false
				s.CommentLines++
			} else { s.CodeLines++ }
			blank = true
			continue
		}
	}
}

type Namer string

func (l Namer) Name() string { return string(l) }

type Matcher func(string) bool

func (m Matcher) Match(fname string) bool { return m(fname) }

func mExt(exts ...string) Matcher {
	return func(fname string) bool {
		for _, ext := range exts {
			if ext == path.Ext(fname) {
				return true
			}
		}
		return false
	}
}

func mName(names ...string) Matcher {
	return func(fname string) bool {
		for _, name := range names {
			if name == path.Base(fname) {
				return true
			}
		}
		return false
	}
}

type Stats struct {
	FileCount    int
	TotalLines   int
	CodeLines    int
	BlankLines   int
	CommentLines int
}

var info = map[string]*Stats{}

func handleFile(fname string) {
	var l Language
	ok := false
	for _, lang := range languages {
		if lang.Match(fname) {
			ok = true
			l = lang
			break
		}
	}
	if !ok {
		return // ignore this file
	}
	i, ok := info[l.Name()]
	if !ok {
		i = &Stats{}
		info[l.Name()] = i
	}
	c, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  ! %s\n", fname)
		return
	}
	l.Update(c, i)
}

var files []string

func add(n string) {
	fi, err := os.Stat(n)
	if err != nil {
		goto invalid
	}
	if fi.IsDir() {
		fs, err := ioutil.ReadDir(n)
		if err != nil {
			goto invalid
		}
		for _, f := range fs {
			if f.Name()[0] != '.' {
				add(path.Join(n, f.Name()))
			}
		}
		return
	}
	if fi.Mode()&os.ModeType == 0 {
		files = append(files, n)
		return
	}

	println(fi.Mode())

invalid:
	fmt.Fprintf(os.Stderr, "  ! %s\n", n)
}

type LData []LResult

func (d LData) Len() int { return len(d) }

func (d LData) Less(i, j int) bool { return d[i].CodeLines > d[j].CodeLines }

func (d LData) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

type LResult struct {
	Name string
	FileCount int
	CodeLines int
	CommentLines int
	BlankLines int
	TotalLines int
}

func printInfo() {
	w := tabwriter.NewWriter(os.Stdout, 2, 8, 2, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "Language\tFiles\tCode\tComment\tBlank\tTotal\t")
	d := LData([]LResult{})
	for n, i := range info {
		d = append(d, LResult{
			n,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines,
		})
	}
	sort.Sort(d)
	for _, i := range d {
		fmt.Fprintf(
			w,
			"%s\t%d\t%d\t%d\t%d\t%d\t\n",
			i.Name,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines)
	}
	w.Flush()
}

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	version = flag.Bool("V", false, "display version info and exit")
)

func main() {
	flag.Parse()
	if *version {
		fmt.Printf("sloc %s\n", VERSION)
		return
	}
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
			return
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	args := flag.Args()
	if len(args) == 0 {
		args = append(args, `.`)
	}

	for _, n := range args {
		add(n)
	}

	for _, f := range files {
		handleFile(f)
	}
	printInfo()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"runtime/pprof"
	"sort"
	"text/tabwriter"
)

const VERSION = `0.1`

var languages = []Language{
	Language{"C", mExt(".c", ".h"), cComments},
	Language{"C++", mExt(".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx"), cComments},
	Language{"Go", mExt(".go"), cComments},
	Language{"Haskell", mExt(".hs", ".lhs"), noComments},
	Language{"Perl", mExt(".pl", ".pm"), shComments},

	Language{"Shell", mExt(".sh"), shComments},
	Language{"Bash", mExt(".bash"), shComments},

	Language{"Python", mExt(".py"), noComments},
	Language{"Assembly", mExt(".asm", ".s"), semiComments},
	Language{"Lisp", mExt(".lsp", ".lisp"), semiComments},

	Language{"Make", mName("makefile", "Makefile", "MAKEFILE"), shComments},
	Language{"Jam", mName("Jamfile", "Jamrules"), shComments},

	Language{"Markdown", mExt(".md"), noComments},

	Language{"HTML", mExt(".htm", ".html", ".xhtml"), xmlComments},
	Language{"XML", mExt(".xml"), xmlComments},
	Language{"CSS", mExt(".css"), cssComments},
	Language{"JavaScript", mExt(".js"), cComments},
}

type Commenter struct {
	LineComment  string
	StartComment string
	EndComment   string
	Nesting      bool
}

var (
	noComments = Commenter{"\000", "\000", "\000", false}
	xmlComments = Commenter{"\000", `<!--`, `-->`, false}
	cComments  = Commenter{`//`, `/*`, `*/`, false}
	cssComments  = Commenter{"\000", `/*`, `*/`, false}
	shComments = Commenter{`#`, "\000", "\000", false}
	semiComments = Commenter{`;`, "\000", "\000", false}
)

type Language struct {
	Namer
	Matcher
	Commenter
}

// TODO work properly with unicode
func (l Language) Update(c []byte, s *Stats) {
	s.FileCount++

	inComment := 0 // this is an int for nesting
	inLComment := false
	blank := true
	lc := []byte(l.LineComment)
	sc := []byte(l.StartComment)
	ec := []byte(l.EndComment)
	lp, sp, ep := 0, 0, 0

	for _, b := range c {
		if inComment == 0 && b == lc[lp] {
			lp++
			if lp == len(lc) {
				inLComment = true
				lp = 0
			}
		} else { lp = 0 }
		if !inLComment && b == sc[sp] {
			sp++
			if sp == len(sc) {
				inComment++
				if inComment > 1 && !l.Nesting {
					inComment = 1
				}
				sp = 0
			}
		} else { sp = 0 }
		if !inLComment && inComment > 0 && b == ec[ep] {
			ep++
			if ep == len(ec) {
				if inComment > 0 { inComment-- }
				ep = 0
			}
		} else { ep = 0 }

		if b != byte(' ') && b != byte('\t') && b != byte('\n') {
			blank = false
		}

		// BUG(srl): lines with comment don't count towards code
		// Note that lines with both code and comment count towards
		// each, but are not counted twice in the total.
		if b == byte('\n') {
			s.TotalLines++
			if inComment > 0 || inLComment {
				inLComment = false
				s.CommentLines++
			} else if blank {
				s.BlankLines++
			} else { s.CodeLines++ }
			blank = true
			continue
		}
	}
}

type Namer string

func (l Namer) Name() string { return string(l) }

type Matcher func(string) bool

func (m Matcher) Match(fname string) bool { return m(fname) }

func mExt(exts ...string) Matcher {
	return func(fname string) bool {
		for _, ext := range exts {
			if ext == path.Ext(fname) {
				return true
			}
		}
		return false
	}
}

func mName(names ...string) Matcher {
	return func(fname string) bool {
		for _, name := range names {
			if name == path.Base(fname) {
				return true
			}
		}
		return false
	}
}

type Stats struct {
	FileCount    int
	TotalLines   int
	CodeLines    int
	BlankLines   int
	CommentLines int
}

var info = map[string]*Stats{}

func handleFile(fname string) {
	var l Language
	ok := false
	for _, lang := range languages {
		if lang.Match(fname) {
			ok = true
			l = lang
			break
		}
	}
	if !ok {
		return // ignore this file
	}
	i, ok := info[l.Name()]
	if !ok {
		i = &Stats{}
		info[l.Name()] = i
	}
	c, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  ! %s\n", fname)
		return
	}
	l.Update(c, i)
}

var files []string

func add(n string) {
	fi, err := os.Stat(n)
	if err != nil {
		goto invalid
	}
	if fi.IsDir() {
		fs, err := ioutil.ReadDir(n)
		if err != nil {
			goto invalid
		}
		for _, f := range fs {
			if f.Name()[0] != '.' {
				add(path.Join(n, f.Name()))
			}
		}
		return
	}
	if fi.Mode()&os.ModeType == 0 {
		files = append(files, n)
		return
	}

	println(fi.Mode())

invalid:
	fmt.Fprintf(os.Stderr, "  ! %s\n", n)
}

type LData []LResult

func (d LData) Len() int { return len(d) }

func (d LData) Less(i, j int) bool {
	if d[i].CodeLines == d[j].CodeLines {
		return d[i].Name > d[j].Name
	}
	return d[i].CodeLines > d[j].CodeLines
}

func (d LData) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

type LResult struct {
	Name string
	FileCount int
	CodeLines int
	CommentLines int
	BlankLines int
	TotalLines int
}

func (r *LResult) Add(a LResult) {
	r.FileCount += a.FileCount
	r.CodeLines += a.CodeLines
	r.CommentLines += a.CommentLines
	r.BlankLines += a.BlankLines
	r.TotalLines += a.TotalLines
}

func printInfo() {
	w := tabwriter.NewWriter(os.Stdout, 2, 8, 2, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "Language\tFiles\tCode\tComment\tBlank\tTotal\t")
	d := LData([]LResult{})
	total := &LResult{}
	total.Name = "Total"
	for n, i := range info {
		r := LResult{
			n,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines,
		}
		d = append(d, r)
		total.Add(r)
	}
	d = append(d, *total)
	sort.Sort(d)
	//d[0].Name = "Total"
	for _, i := range d {
		fmt.Fprintf(
			w,
			"%s\t%d\t%d\t%d\t%d\t%d\t\n",
			i.Name,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines)
	}

	w.Flush()
}

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	version = flag.Bool("V", false, "display version info and exit")
)

func main() {
	flag.Parse()
	if *version {
		fmt.Printf("sloc %s\n", VERSION)
		return
	}
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
			return
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	args := flag.Args()
	if len(args) == 0 {
		args = append(args, `.`)
	}

	for _, n := range args {
		add(n)
	}

	for _, f := range files {
		handleFile(f)
	}
	printInfo()
}
<commit_msg>JSON output<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"runtime/pprof"
	"sort"
	"text/tabwriter"
)

const VERSION = `0.1`

var languages = []Language{
	Language{"C", mExt(".c", ".h"), cComments},
	Language{"C++", mExt(".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx"), cComments},
	Language{"Go", mExt(".go"), cComments},
	Language{"Haskell", mExt(".hs", ".lhs"), noComments},
	Language{"Perl", mExt(".pl", ".pm"), shComments},

	Language{"Shell", mExt(".sh"), shComments},
	Language{"Bash", mExt(".bash"), shComments},

	Language{"Python", mExt(".py"), noComments},
	Language{"Assembly", mExt(".asm", ".s"), semiComments},
	Language{"Lisp", mExt(".lsp", ".lisp"), semiComments},

	Language{"Make", mName("makefile", "Makefile", "MAKEFILE"), shComments},
	Language{"Jam", mName("Jamfile", "Jamrules"), shComments},

	Language{"Markdown", mExt(".md"), noComments},

	Language{"HTML", mExt(".htm", ".html", ".xhtml"), xmlComments},
	Language{"XML", mExt(".xml"), xmlComments},
	Language{"CSS", mExt(".css"), cssComments},
	Language{"JavaScript", mExt(".js"), cComments},
}

type Commenter struct {
	LineComment  string
	StartComment string
	EndComment   string
	Nesting      bool
}

var (
	noComments = Commenter{"\000", "\000", "\000", false}
	xmlComments = Commenter{"\000", `<!--`, `-->`, false}
	cComments  = Commenter{`//`, `/*`, `*/`, false}
	cssComments  = Commenter{"\000", `/*`, `*/`, false}
	shComments = Commenter{`#`, "\000", "\000", false}
	semiComments = Commenter{`;`, "\000", "\000", false}
)

type Language struct {
	Namer
	Matcher
	Commenter
}

// TODO work properly with unicode
func (l Language) Update(c []byte, s *Stats) {
	s.FileCount++

	inComment := 0 // this is an int for nesting
	inLComment := false
	blank := true
	lc := []byte(l.LineComment)
	sc := []byte(l.StartComment)
	ec := []byte(l.EndComment)
	lp, sp, ep := 0, 0, 0

	for _, b := range c {
		if inComment == 0 && b == lc[lp] {
			lp++
			if lp == len(lc) {
				inLComment = true
				lp = 0
			}
		} else { lp = 0 }
		if !inLComment && b == sc[sp] {
			sp++
			if sp == len(sc) {
				inComment++
				if inComment > 1 && !l.Nesting {
					inComment = 1
				}
				sp = 0
			}
		} else { sp = 0 }
		if !inLComment && inComment > 0 && b == ec[ep] {
			ep++
			if ep == len(ec) {
				if inComment > 0 { inComment-- }
				ep = 0
			}
		} else { ep = 0 }

		if b != byte(' ') && b != byte('\t') && b != byte('\n') {
			blank = false
		}

		// BUG(srl): lines with comment don't count towards code
		// Note that lines with both code and comment count towards
		// each, but are not counted twice in the total.
		if b == byte('\n') {
			s.TotalLines++
			if inComment > 0 || inLComment {
				inLComment = false
				s.CommentLines++
			} else if blank {
				s.BlankLines++
			} else { s.CodeLines++ }
			blank = true
			continue
		}
	}
}

type Namer string

func (l Namer) Name() string { return string(l) }

type Matcher func(string) bool

func (m Matcher) Match(fname string) bool { return m(fname) }

func mExt(exts ...string) Matcher {
	return func(fname string) bool {
		for _, ext := range exts {
			if ext == path.Ext(fname) {
				return true
			}
		}
		return false
	}
}

func mName(names ...string) Matcher {
	return func(fname string) bool {
		for _, name := range names {
			if name == path.Base(fname) {
				return true
			}
		}
		return false
	}
}

type Stats struct {
	FileCount    int
	TotalLines   int
	CodeLines    int
	BlankLines   int
	CommentLines int
}

var info = map[string]*Stats{}

func handleFile(fname string) {
	var l Language
	ok := false
	for _, lang := range languages {
		if lang.Match(fname) {
			ok = true
			l = lang
			break
		}
	}
	if !ok {
		return // ignore this file
	}
	i, ok := info[l.Name()]
	if !ok {
		i = &Stats{}
		info[l.Name()] = i
	}
	c, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  ! %s\n", fname)
		return
	}
	l.Update(c, i)
}

var files []string

func add(n string) {
	fi, err := os.Stat(n)
	if err != nil {
		goto invalid
	}
	if fi.IsDir() {
		fs, err := ioutil.ReadDir(n)
		if err != nil {
			goto invalid
		}
		for _, f := range fs {
			if f.Name()[0] != '.' {
				add(path.Join(n, f.Name()))
			}
		}
		return
	}
	if fi.Mode()&os.ModeType == 0 {
		files = append(files, n)
		return
	}

	println(fi.Mode())

invalid:
	fmt.Fprintf(os.Stderr, "  ! %s\n", n)
}

type LData []LResult

func (d LData) Len() int { return len(d) }

func (d LData) Less(i, j int) bool {
	if d[i].CodeLines == d[j].CodeLines {
		return d[i].Name > d[j].Name
	}
	return d[i].CodeLines > d[j].CodeLines
}

func (d LData) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

type LResult struct {
	Name string
	FileCount int
	CodeLines int
	CommentLines int
	BlankLines int
	TotalLines int
}

func (r *LResult) Add(a LResult) {
	r.FileCount += a.FileCount
	r.CodeLines += a.CodeLines
	r.CommentLines += a.CommentLines
	r.BlankLines += a.BlankLines
	r.TotalLines += a.TotalLines
}

func printJSON() {
	bs, err := json.MarshalIndent(info, "", "  ")
	if err != nil { panic(err) }
	fmt.Println(string(bs))
}

func printInfo() {
	w := tabwriter.NewWriter(os.Stdout, 2, 8, 2, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "Language\tFiles\tCode\tComment\tBlank\tTotal\t")
	d := LData([]LResult{})
	total := &LResult{}
	total.Name = "Total"
	for n, i := range info {
		r := LResult{
			n,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines,
		}
		d = append(d, r)
		total.Add(r)
	}
	d = append(d, *total)
	sort.Sort(d)
	//d[0].Name = "Total"
	for _, i := range d {
		fmt.Fprintf(
			w,
			"%s\t%d\t%d\t%d\t%d\t%d\t\n",
			i.Name,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines)
	}

	w.Flush()
}

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	useJson = flag.Bool("json", false, "JSON-format output")
	version = flag.Bool("V", false, "display version info and exit")
)

func main() {
	flag.Parse()
	if *version {
		fmt.Printf("sloc %s\n", VERSION)
		return
	}
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
			return
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	args := flag.Args()
	if len(args) == 0 {
		args = append(args, `.`)
	}

	for _, n := range args {
		add(n)
	}

	for _, f := range files {
		handleFile(f)
	}

	if *useJson {
		printJSON()
	} else {
		printInfo()
	}
}
<|endoftext|>