<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package csv reads and writes comma-separated values (CSV) files.
// There are many kinds of CSV files; this package supports the format
// described in RFC 4180.
//
// A csv file contains zero or more records of one or more fields per record.
// Each record is separated by the newline character. The final record may
// optionally be followed by a newline character.
//
//	field1,field2,field3
//
// White space is considered part of a field.
//
// Carriage returns before newline characters are silently removed.
//
// Blank lines are ignored. A line with only whitespace characters (excluding
// the ending newline character) is not considered a blank line.
//
// Fields which start and stop with the quote character " are called
// quoted-fields. The beginning and ending quote are not part of the
// field.
//
// The source:
//
//	normal string,"quoted-field"
//
// results in the fields
//
//	{`normal string`, `quoted-field`}
//
// Within a quoted-field a quote character followed by a second quote
// character is considered a single quote.
//
//	"the ""word"" is true","a ""quoted-field"""
//
// results in
//
//	{`the "word" is true`, `a "quoted-field"`}
//
// Newlines and commas may be included in a quoted-field
//
//	"Multi-line
//	field","comma is ,"
//
// results in
//
//	{`Multi-line
//	field`, `comma is ,`}
package csv

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"unicode"
	"unicode/utf8"
)

// A ParseError is returned for parsing errors.
// Line numbers are 1-indexed and columns are 0-indexed.
type ParseError struct {
	RecordLine int   // Line where the record starts
	Line       int   // Line where the error occurred
	Column     int   // Column (rune index) where the error occurred
	Err        error // The actual error
}

func (e *ParseError) Error() string {
	if e.Err == ErrFieldCount {
		return fmt.Sprintf("record on line %d: %v", e.Line, e.Err)
	}
	if e.RecordLine != e.Line {
		return fmt.Sprintf("record on line %d; parse error on line %d, column %d: %v", e.RecordLine, e.Line, e.Column, e.Err)
	}
	return fmt.Sprintf("parse error on line %d, column %d: %v", e.Line, e.Column, e.Err)
}

// These are the errors that can be returned in ParseError.Err.
var (
	ErrTrailingComma = errors.New("extra delimiter at end of line") // Deprecated: No longer used.
	ErrBareQuote     = errors.New("bare \" in non-quoted-field")
	ErrQuote         = errors.New("extraneous or missing \" in field")
	ErrFieldCount    = errors.New("wrong number of fields")
)

// A Reader reads records from a CSV-encoded file.
//
// As returned by NewReader, a Reader expects input conforming to RFC 4180.
// The exported fields can be changed to customize the details before the
// first call to Read or ReadAll.
type Reader struct {
	// Comma is the field delimiter.
	// It is set to comma (',') by NewReader.
	Comma rune

	// Comment, if not 0, is the comment character. Lines beginning with the
	// Comment character without preceding whitespace are ignored.
	// With leading whitespace the Comment character becomes part of the
	// field, even if TrimLeadingSpace is true.
	Comment rune

	// FieldsPerRecord is the number of expected fields per record.
	// If FieldsPerRecord is positive, Read requires each record to
	// have the given number of fields. If FieldsPerRecord is 0, Read sets it to
	// the number of fields in the first record, so that future records must
	// have the same field count. If FieldsPerRecord is negative, no check is
	// made and records may have a variable number of fields.
	FieldsPerRecord int

	// If LazyQuotes is true, a quote may appear in an unquoted field and a
	// non-doubled quote may appear in a quoted field.
	LazyQuotes bool

	// If TrimLeadingSpace is true, leading white space in a field is ignored.
	// This is done even if the field delimiter, Comma, is white space.
	TrimLeadingSpace bool

	// ReuseRecord controls whether calls to Read may return a slice sharing
	// the backing array of the previous call's returned slice for performance.
	// By default, each call to Read returns newly allocated memory owned by the caller.
	ReuseRecord bool

	TrailingComma bool // Deprecated: No longer used.

	r *bufio.Reader

	// numLine is the current line being read in the CSV file.
	numLine int

	// rawBuffer is a line buffer only used by the readLine method.
	rawBuffer []byte

	// recordBuffer holds the unescaped fields, one after another.
	// The fields can be accessed by using the indexes in fieldIndexes.
	// E.g., For the row `a,"b","c""d",e`, recordBuffer will contain `abc"de`
	// and fieldIndexes will contain the indexes [1, 2, 5, 6].
	recordBuffer []byte

	// fieldIndexes is an index of fields inside recordBuffer.
	// The i'th field ends at offset fieldIndexes[i] in recordBuffer.
	fieldIndexes []int

	// lastRecord is a record cache and only used when ReuseRecord == true.
	lastRecord []string
}

// NewReader returns a new Reader that reads from r.
func NewReader(r io.Reader) *Reader {
	return &Reader{
		Comma: ',',
		r:     bufio.NewReader(r),
	}
}

// Read reads one record (a slice of fields) from r.
// If the record has an unexpected number of fields,
// Read returns the record along with the error ErrFieldCount.
// Except for that case, Read always returns either a non-nil
// record or a non-nil error, but not both.
// If there is no data left to be read, Read returns nil, io.EOF.
// If ReuseRecord is true, the returned slice may be shared
// between multiple calls to Read.
func (r *Reader) Read() (record []string, err error) {
	if r.ReuseRecord {
		record, err = r.readRecord(r.lastRecord)
		r.lastRecord = record
	} else {
		record, err = r.readRecord(nil)
	}
	return record, err
}

// ReadAll reads all the remaining records from r.
// Each record is a slice of fields.
// A successful call returns err == nil, not err == io.EOF. Because ReadAll is
// defined to read until EOF, it does not treat end of file as an error to be
// reported.
func (r *Reader) ReadAll() (records [][]string, err error) {
	for {
		record, err := r.readRecord(nil)
		if err == io.EOF {
			return records, nil
		}
		if err != nil {
			return nil, err
		}
		records = append(records, record)
	}
}

// readLine reads the next line (with the trailing endline).
// If EOF is hit without a trailing endline, it will be omitted.
// If some bytes were read, then the error is never io.EOF.
// The result is only valid until the next call to readLine.
func (r *Reader) readLine() ([]byte, error) {
	line, err := r.r.ReadSlice('\n')
	if err == bufio.ErrBufferFull {
		r.rawBuffer = append(r.rawBuffer[:0], line...)
		for err == bufio.ErrBufferFull {
			line, err = r.r.ReadSlice('\n')
			r.rawBuffer = append(r.rawBuffer, line...)
		}
		line = r.rawBuffer
	}
	if len(line) > 0 && err == io.EOF {
		err = nil
	}
	r.numLine++
	return line, err
}

// lengthCRLF reports the number of bytes for a trailing "\r\n".
func lengthCRLF(b []byte) int {
	if j := len(b) - 1; j >= 0 && b[j] == '\n' {
		if j := len(b) - 2; j >= 0 && b[j] == '\r' {
			return 2
		}
		return 1
	}
	return 0
}

// nextRune returns the next rune in b or utf8.RuneError.
func nextRune(b []byte) rune {
	r, _ := utf8.DecodeRune(b)
	return r
}

func (r *Reader) readRecord(dst []string) ([]string, error) {
	// Read line (automatically skipping past empty lines and any comments).
	var line, fullLine []byte
	var errRead error
	for errRead == nil {
		line, errRead = r.readLine()
		if r.Comment != 0 && nextRune(line) == r.Comment {
			line = nil
			continue // Skip comment lines
		}
		if errRead == nil && len(line) == lengthCRLF(line) {
			line = nil
			continue // Skip empty lines
		}
		fullLine = line
		break
	}
	if errRead == io.EOF {
		return nil, errRead
	}

	// Parse each field in the record.
	var err error
	const quoteLen = len(`"`)
	commaLen := utf8.RuneLen(r.Comma)
	recLine := r.numLine // Starting line for record
	r.recordBuffer = r.recordBuffer[:0]
	r.fieldIndexes = r.fieldIndexes[:0]
parseField:
	for {
		if r.TrimLeadingSpace {
			line = bytes.TrimLeftFunc(line, unicode.IsSpace)
		}
		if len(line) == 0 || line[0] != '"' {
			// Non-quoted string field
			i := bytes.IndexRune(line, r.Comma)
			field := line
			if i >= 0 {
				field = field[:i]
			} else {
				field = field[:len(field)-lengthCRLF(field)]
			}
			// Check to make sure a quote does not appear in field.
			if !r.LazyQuotes {
				if j := bytes.IndexByte(field, '"'); j >= 0 {
					col := utf8.RuneCount(fullLine[:len(fullLine)-len(line[j:])])
					err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrBareQuote}
					break parseField
				}
			}
			r.recordBuffer = append(r.recordBuffer, field...)
			r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
			if i >= 0 {
				line = line[i+commaLen:]
				continue parseField
			}
			break parseField
		} else {
			// Quoted string field
			line = line[quoteLen:]
			for {
				i := bytes.IndexByte(line, '"')
				if i >= 0 {
					// Hit next quote.
					r.recordBuffer = append(r.recordBuffer, line[:i]...)
					line = line[i+quoteLen:]
					switch rn := nextRune(line); {
					case rn == '"':
						// `""` sequence (append quote).
						r.recordBuffer = append(r.recordBuffer, '"')
						line = line[quoteLen:]
					case rn == r.Comma:
						// `",` sequence (end of field).
						line = line[commaLen:]
						r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
						continue parseField
					case lengthCRLF(line) == len(line):
						// `"\n` sequence (end of line).
						r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
						break parseField
					case r.LazyQuotes:
						// `"` sequence (bare quote).
						r.recordBuffer = append(r.recordBuffer, '"')
					default:
						// `"*` sequence (invalid non-escaped quote).
						col := utf8.RuneCount(fullLine[:len(fullLine)-len(line)-quoteLen])
						err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote}
						break parseField
					}
				} else if len(line) > 0 {
					// Hit end of line (copy all data so far).
					r.recordBuffer = append(r.recordBuffer, line...)
					if errRead != nil {
						break parseField
					}
					line, errRead = r.readLine()
					if errRead == io.EOF {
						errRead = nil
					}
					fullLine = line
				} else {
					// Abrupt end of file (EOF or error).
					if !r.LazyQuotes && errRead == nil {
						col := utf8.RuneCount(fullLine)
						err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote}
						break parseField
					}
					r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
					break parseField
				}
			}
		}
	}
	if err == nil {
		err = errRead
	}

	// Create a single string and create slices out of it.
	// This pins the memory of the fields together, but allocates once.
	str := string(r.recordBuffer) // Convert to string once to batch allocations
	dst = dst[:0]
	if cap(dst) < len(r.fieldIndexes) {
		dst = make([]string, len(r.fieldIndexes))
	}
	dst = dst[:len(r.fieldIndexes)]
	var preIdx int
	for i, idx := range r.fieldIndexes {
		dst[i] = str[preIdx:idx]
		preIdx = idx
	}

	// Check or update the expected fields per record.
	if r.FieldsPerRecord > 0 {
		if len(dst) != r.FieldsPerRecord && err == nil {
			err = &ParseError{RecordLine: recLine, Line: recLine, Err: ErrFieldCount}
		}
	} else if r.FieldsPerRecord == 0 {
		r.FieldsPerRecord = len(dst)
	}
	return dst, err
}
<commit_msg>encoding/csv: update ErrQuote message<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package csv reads and writes comma-separated values (CSV) files.
// There are many kinds of CSV files; this package supports the format
// described in RFC 4180.
//
// A csv file contains zero or more records of one or more fields per record.
// Each record is separated by the newline character. The final record may
// optionally be followed by a newline character.
//
//	field1,field2,field3
//
// White space is considered part of a field.
//
// Carriage returns before newline characters are silently removed.
//
// Blank lines are ignored. A line with only whitespace characters (excluding
// the ending newline character) is not considered a blank line.
//
// Fields which start and stop with the quote character " are called
// quoted-fields. The beginning and ending quote are not part of the
// field.
//
// The source:
//
//	normal string,"quoted-field"
//
// results in the fields
//
//	{`normal string`, `quoted-field`}
//
// Within a quoted-field a quote character followed by a second quote
// character is considered a single quote.
//
//	"the ""word"" is true","a ""quoted-field"""
//
// results in
//
//	{`the "word" is true`, `a "quoted-field"`}
//
// Newlines and commas may be included in a quoted-field
//
//	"Multi-line
//	field","comma is ,"
//
// results in
//
//	{`Multi-line
//	field`, `comma is ,`}
package csv

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"unicode"
	"unicode/utf8"
)

// A ParseError is returned for parsing errors.
// Line numbers are 1-indexed and columns are 0-indexed.
type ParseError struct {
	RecordLine int   // Line where the record starts
	Line       int   // Line where the error occurred
	Column     int   // Column (rune index) where the error occurred
	Err        error // The actual error
}

func (e *ParseError) Error() string {
	if e.Err == ErrFieldCount {
		return fmt.Sprintf("record on line %d: %v", e.Line, e.Err)
	}
	if e.RecordLine != e.Line {
		return fmt.Sprintf("record on line %d; parse error on line %d, column %d: %v", e.RecordLine, e.Line, e.Column, e.Err)
	}
	return fmt.Sprintf("parse error on line %d, column %d: %v", e.Line, e.Column, e.Err)
}

// These are the errors that can be returned in ParseError.Err.
var (
	ErrTrailingComma = errors.New("extra delimiter at end of line") // Deprecated: No longer used.
	ErrBareQuote     = errors.New("bare \" in non-quoted-field")
	ErrQuote         = errors.New("extraneous or missing \" in quoted-field")
	ErrFieldCount    = errors.New("wrong number of fields")
)

// A Reader reads records from a CSV-encoded file.
//
// As returned by NewReader, a Reader expects input conforming to RFC 4180.
// The exported fields can be changed to customize the details before the
// first call to Read or ReadAll.
type Reader struct {
	// Comma is the field delimiter.
	// It is set to comma (',') by NewReader.
	Comma rune

	// Comment, if not 0, is the comment character. Lines beginning with the
	// Comment character without preceding whitespace are ignored.
	// With leading whitespace the Comment character becomes part of the
	// field, even if TrimLeadingSpace is true.
	Comment rune

	// FieldsPerRecord is the number of expected fields per record.
	// If FieldsPerRecord is positive, Read requires each record to
	// have the given number of fields. If FieldsPerRecord is 0, Read sets it to
	// the number of fields in the first record, so that future records must
	// have the same field count. If FieldsPerRecord is negative, no check is
	// made and records may have a variable number of fields.
	FieldsPerRecord int

	// If LazyQuotes is true, a quote may appear in an unquoted field and a
	// non-doubled quote may appear in a quoted field.
	LazyQuotes bool

	// If TrimLeadingSpace is true, leading white space in a field is ignored.
	// This is done even if the field delimiter, Comma, is white space.
	TrimLeadingSpace bool

	// ReuseRecord controls whether calls to Read may return a slice sharing
	// the backing array of the previous call's returned slice for performance.
	// By default, each call to Read returns newly allocated memory owned by the caller.
	ReuseRecord bool

	TrailingComma bool // Deprecated: No longer used.

	r *bufio.Reader

	// numLine is the current line being read in the CSV file.
	numLine int

	// rawBuffer is a line buffer only used by the readLine method.
	rawBuffer []byte

	// recordBuffer holds the unescaped fields, one after another.
	// The fields can be accessed by using the indexes in fieldIndexes.
	// E.g., For the row `a,"b","c""d",e`, recordBuffer will contain `abc"de`
	// and fieldIndexes will contain the indexes [1, 2, 5, 6].
	recordBuffer []byte

	// fieldIndexes is an index of fields inside recordBuffer.
	// The i'th field ends at offset fieldIndexes[i] in recordBuffer.
	fieldIndexes []int

	// lastRecord is a record cache and only used when ReuseRecord == true.
	lastRecord []string
}

// NewReader returns a new Reader that reads from r.
func NewReader(r io.Reader) *Reader {
	return &Reader{
		Comma: ',',
		r:     bufio.NewReader(r),
	}
}

// Read reads one record (a slice of fields) from r.
// If the record has an unexpected number of fields,
// Read returns the record along with the error ErrFieldCount.
// Except for that case, Read always returns either a non-nil
// record or a non-nil error, but not both.
// If there is no data left to be read, Read returns nil, io.EOF.
// If ReuseRecord is true, the returned slice may be shared
// between multiple calls to Read.
func (r *Reader) Read() (record []string, err error) {
	if r.ReuseRecord {
		record, err = r.readRecord(r.lastRecord)
		r.lastRecord = record
	} else {
		record, err = r.readRecord(nil)
	}
	return record, err
}

// ReadAll reads all the remaining records from r.
// Each record is a slice of fields.
// A successful call returns err == nil, not err == io.EOF. Because ReadAll is
// defined to read until EOF, it does not treat end of file as an error to be
// reported.
func (r *Reader) ReadAll() (records [][]string, err error) {
	for {
		record, err := r.readRecord(nil)
		if err == io.EOF {
			return records, nil
		}
		if err != nil {
			return nil, err
		}
		records = append(records, record)
	}
}

// readLine reads the next line (with the trailing endline).
// If EOF is hit without a trailing endline, it will be omitted.
// If some bytes were read, then the error is never io.EOF.
// The result is only valid until the next call to readLine.
func (r *Reader) readLine() ([]byte, error) {
	line, err := r.r.ReadSlice('\n')
	if err == bufio.ErrBufferFull {
		r.rawBuffer = append(r.rawBuffer[:0], line...)
		for err == bufio.ErrBufferFull {
			line, err = r.r.ReadSlice('\n')
			r.rawBuffer = append(r.rawBuffer, line...)
		}
		line = r.rawBuffer
	}
	if len(line) > 0 && err == io.EOF {
		err = nil
	}
	r.numLine++
	return line, err
}

// lengthCRLF reports the number of bytes for a trailing "\r\n".
func lengthCRLF(b []byte) int {
	if j := len(b) - 1; j >= 0 && b[j] == '\n' {
		if j := len(b) - 2; j >= 0 && b[j] == '\r' {
			return 2
		}
		return 1
	}
	return 0
}

// nextRune returns the next rune in b or utf8.RuneError.
func nextRune(b []byte) rune {
	r, _ := utf8.DecodeRune(b)
	return r
}

func (r *Reader) readRecord(dst []string) ([]string, error) {
	// Read line (automatically skipping past empty lines and any comments).
	var line, fullLine []byte
	var errRead error
	for errRead == nil {
		line, errRead = r.readLine()
		if r.Comment != 0 && nextRune(line) == r.Comment {
			line = nil
			continue // Skip comment lines
		}
		if errRead == nil && len(line) == lengthCRLF(line) {
			line = nil
			continue // Skip empty lines
		}
		fullLine = line
		break
	}
	if errRead == io.EOF {
		return nil, errRead
	}

	// Parse each field in the record.
	var err error
	const quoteLen = len(`"`)
	commaLen := utf8.RuneLen(r.Comma)
	recLine := r.numLine // Starting line for record
	r.recordBuffer = r.recordBuffer[:0]
	r.fieldIndexes = r.fieldIndexes[:0]
parseField:
	for {
		if r.TrimLeadingSpace {
			line = bytes.TrimLeftFunc(line, unicode.IsSpace)
		}
		if len(line) == 0 || line[0] != '"' {
			// Non-quoted string field
			i := bytes.IndexRune(line, r.Comma)
			field := line
			if i >= 0 {
				field = field[:i]
			} else {
				field = field[:len(field)-lengthCRLF(field)]
			}
			// Check to make sure a quote does not appear in field.
			if !r.LazyQuotes {
				if j := bytes.IndexByte(field, '"'); j >= 0 {
					col := utf8.RuneCount(fullLine[:len(fullLine)-len(line[j:])])
					err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrBareQuote}
					break parseField
				}
			}
			r.recordBuffer = append(r.recordBuffer, field...)
			r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
			if i >= 0 {
				line = line[i+commaLen:]
				continue parseField
			}
			break parseField
		} else {
			// Quoted string field
			line = line[quoteLen:]
			for {
				i := bytes.IndexByte(line, '"')
				if i >= 0 {
					// Hit next quote.
					r.recordBuffer = append(r.recordBuffer, line[:i]...)
					line = line[i+quoteLen:]
					switch rn := nextRune(line); {
					case rn == '"':
						// `""` sequence (append quote).
						r.recordBuffer = append(r.recordBuffer, '"')
						line = line[quoteLen:]
					case rn == r.Comma:
						// `",` sequence (end of field).
						line = line[commaLen:]
						r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
						continue parseField
					case lengthCRLF(line) == len(line):
						// `"\n` sequence (end of line).
						r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
						break parseField
					case r.LazyQuotes:
						// `"` sequence (bare quote).
						r.recordBuffer = append(r.recordBuffer, '"')
					default:
						// `"*` sequence (invalid non-escaped quote).
						col := utf8.RuneCount(fullLine[:len(fullLine)-len(line)-quoteLen])
						err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote}
						break parseField
					}
				} else if len(line) > 0 {
					// Hit end of line (copy all data so far).
					r.recordBuffer = append(r.recordBuffer, line...)
					if errRead != nil {
						break parseField
					}
					line, errRead = r.readLine()
					if errRead == io.EOF {
						errRead = nil
					}
					fullLine = line
				} else {
					// Abrupt end of file (EOF or error).
					if !r.LazyQuotes && errRead == nil {
						col := utf8.RuneCount(fullLine)
						err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote}
						break parseField
					}
					r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
					break parseField
				}
			}
		}
	}
	if err == nil {
		err = errRead
	}

	// Create a single string and create slices out of it.
	// This pins the memory of the fields together, but allocates once.
	str := string(r.recordBuffer) // Convert to string once to batch allocations
	dst = dst[:0]
	if cap(dst) < len(r.fieldIndexes) {
		dst = make([]string, len(r.fieldIndexes))
	}
	dst = dst[:len(r.fieldIndexes)]
	var preIdx int
	for i, idx := range r.fieldIndexes {
		dst[i] = str[preIdx:idx]
		preIdx = idx
	}

	// Check or update the expected fields per record.
	if r.FieldsPerRecord > 0 {
		if len(dst) != r.FieldsPerRecord && err == nil {
			err = &ParseError{RecordLine: recLine, Line: recLine, Err: ErrFieldCount}
		}
	} else if r.FieldsPerRecord == 0 {
		r.FieldsPerRecord = len(dst)
	}
	return dst, err
}
<|endoftext|>


<commit_before>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"os"
	"strings"

	gcc "h12.io/go-gccxml"
)

var (
	apixml = flag.String("apixml", "", "Path of api.xml")
	header = flag.String("header", "", "Path of plplot.h")
	incdir = flag.String("incdir", "", "colon seperated include directories")
)

func main() {
	flag.Parse()
	if *header == "" || *apixml == "" {
		flag.PrintDefaults()
		return
	}

	var incDirs []string
	if incdir != nil {
		incDirs = strings.Split(*incdir, ":")
	}
	for i := range incDirs {
		incDirs[i] = "-I" + incDirs[i]
	}

	hasMismatch := false

	docFuncs := ParseApiXml(*apixml, "pl")
	h, err := gcc.Xml{*header, incDirs}.Doc()
	c(err)

	for _, hf := range h.Functions {
		if !strings.HasPrefix(hf.CName(), "c_") {
			continue
		}
		if df := docFuncs.Find(trimPrefix(hf.CName(), "c_")); df != nil {
		} else {
			hasMismatch = true
			p(hf.CName(), "is not documented.")
		}
	}

	for _, df := range docFuncs {
		if hf := findHeaderFunc(h, df.Name); hf != nil {
			if len(hf.Arguments) != len(df.Args) {
				hasMismatch = true
				p("para count mismatch for func", df.Name)
				continue
			}
			for i := range df.Args {
				docArg, headerArg := df.Args[i], hf.Arguments[i]
				if docArg.Name != headerArg.CName() {
					hasMismatch = true
					p("para", i, "of func", df.Name, "NAME mismatch doc[",
						docArg.Name, "] vs. header[", headerArg.CName(), "]")
				}
				gccType := trimArgType(headerArg.CType().CDecl(""))
				docType := trimArgType(docArg.Type)
				if docType != gccType {
					hasMismatch = true
					p("para", i, "of func", df.Name, "TYPE mismatch",
						"doc[", docType, "] vs. header[", gccType, "]")
				}
			}
		} else {
			hasMismatch = true
			p(df.Name, "does not exist but remains in doc.")
		}
	}

	if hasMismatch {
		os.Exit(-1)
	}
}

func findHeaderFunc(h *gcc.XmlDoc, name string) *gcc.Function {
	for _, f := range h.Functions {
		if name == trimPrefix(f.CName(), "c_") {
			return f
		}
	}
	return nil
}

func trimPrefix(s, prefix string) string {
	return strings.TrimPrefix(s, prefix)
}

func trimArgType(s string) string {
	s = strings.TrimSpace(s)
	s = strings.TrimPrefix(s, "function ")
	s = strings.TrimPrefix(s, "struct ")
	s = strings.TrimSpace(s)
	return s
}
<commit_msg>support castxml<commit_after>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"os"
	"strings"

	gcc "h12.io/go-gccxml"
)

var (
	apixml  = flag.String("apixml", "", "Path of api.xml")
	header  = flag.String("header", "", "Path of plplot.h")
	incdir  = flag.String("incdir", "", "colon seperated include directories")
	castxml = flag.Bool("castxml", false, "whether to use castxml instead of gccxml")
)

func main() {
	flag.Parse()
	if *header == "" || *apixml == "" {
		flag.PrintDefaults()
		return
	}

	var incDirs []string
	if incdir != nil {
		incDirs = strings.Split(*incdir, ":")
	}
	for i := range incDirs {
		incDirs[i] = "-I" + incDirs[i]
	}

	hasMismatch := false

	docFuncs := ParseApiXml(*apixml, "pl")
	h, err := gcc.Xml{File: *header, Args: incDirs, CastXml: *castxml}.Doc()
	c(err)

	for _, hf := range h.Functions {
		if !strings.HasPrefix(hf.CName(), "c_") {
			continue
		}
		if df := docFuncs.Find(trimPrefix(hf.CName(), "c_")); df != nil {
		} else {
			hasMismatch = true
			p(hf.CName(), "is not documented.")
		}
	}

	for _, df := range docFuncs {
		if hf := findHeaderFunc(h, df.Name); hf != nil {
			if len(hf.Arguments) != len(df.Args) {
				hasMismatch = true
				p("para count mismatch for func", df.Name)
				continue
			}
			for i := range df.Args {
				docArg, headerArg := df.Args[i], hf.Arguments[i]
				if docArg.Name != headerArg.CName() {
					hasMismatch = true
					p("para", i, "of func", df.Name, "NAME mismatch doc[",
						docArg.Name, "] vs. header[", headerArg.CName(), "]")
				}
				gccType := trimArgType(headerArg.CType().CDecl(""))
				docType := trimArgType(docArg.Type)
				if docType != gccType {
					hasMismatch = true
					p("para", i, "of func", df.Name, "TYPE mismatch",
						"doc[", docType, "] vs. header[", gccType, "]")
				}
			}
		} else {
			hasMismatch = true
			p(df.Name, "does not exist but remains in doc.")
		}
	}

	if hasMismatch {
		os.Exit(-1)
	}
}

func findHeaderFunc(h *gcc.XmlDoc, name string) *gcc.Function {
	for _, f := range h.Functions {
		if name == trimPrefix(f.CName(), "c_") {
			return f
		}
	}
	return nil
}

func trimPrefix(s, prefix string) string {
	return strings.TrimPrefix(s, prefix)
}

func trimArgType(s string) string {
	s = strings.TrimSpace(s)
	s = strings.TrimPrefix(s, "function ")
	s = strings.TrimPrefix(s, "struct ")
	s = strings.TrimSpace(s)
	return s
}
<|endoftext|>


<commit_before>package sweetiebot

import (
  "github.com/bwmarrin/discordgo"
  "time"
  "strconv"
  "strings"
)

// The emote module detects banned emotes and deletes them
type SpamModule struct {
  tracker map[uint64]*SaturationLimit
  channels *map[uint64]bool
  lastraid int64
}

func (w *SpamModule) SetChannelMap(c *map[uint64]bool) {
  w.channels = c
}
func (w *SpamModule) GetChannelMap() *map[uint64]bool {
  return w.channels
}

func (w *SpamModule) Name() string {
  return "Anti-Spam"
}

func (w *SpamModule) Register(hooks *ModuleHooks) {
  w.tracker = make(map[uint64]*SaturationLimit)
  w.lastraid = 0
  hooks.OnMessageCreate = append(hooks.OnMessageCreate, w)
  hooks.OnCommand = append(hooks.OnCommand, w)
  hooks.OnGuildMemberAdd = append(hooks.OnGuildMemberAdd, w)
  hooks.OnGuildMemberUpdate = append(hooks.OnGuildMemberUpdate, w)
}
func (w *SpamModule) Channels() []string {
  return []string{}
}

func KillSpammer(u *discordgo.User) {
  // Manually set our internal state to say this user has the Silent role, to prevent race conditions
  m, err := sb.dg.State.Member(sb.GuildID, u.ID)
  if err == nil {
    for _, v := range m.Roles {
      if v == sb.SilentRole {
        return // Spammer was already killed, so don't try killing it again
      }
    }
    m.Roles = append(m.Roles, sb.SilentRole)
  }

  sb.log.Log("Killing spammer ", u.Username)

  sb.dg.GuildMemberEdit(sb.GuildID, u.ID, m.Roles) // Tell discord to make this spammer silent
  messages := sb.db.GetRecentMessages(SBatoi(u.ID), 60) // Retrieve all messages in the past 60 seconds and delete them.

  for _, v := range messages {
    sb.dg.ChannelMessageDelete(strconv.FormatUint(v.channel, 10), strconv.FormatUint(v.message, 10))
  }

  sb.SendMessage(sb.ModChannelID, "`Alert: " + u.Username + " was silenced for spamming. Please investigate.`") // Alert admins
}
func (w *SpamModule) CheckSpam(s *discordgo.Session, m *discordgo.Message) bool {
  if m.Author != nil {
    if UserHasRole(m.Author.ID, sb.SilentRole) {
      s.ChannelMessageDelete(m.ChannelID, m.ID);
      return true
    }
    id := SBatoi(m.Author.ID)
    _, ok := w.tracker[id]
    if !ok {
      w.tracker[id] = &SaturationLimit{make([]int64, sb.config.Maxspam, sb.config.Maxspam), 0, AtomicFlag{0}};
    }
    limit := w.tracker[id]
    limit.append(time.Now().UTC().Unix())
    if limit.checkafter(5, 1) || limit.checkafter(10, 5) || limit.checkafter(12, 10) {
      KillSpammer(m.Author)
      return true
    }
  }
  return false
}
func (w *SpamModule) OnMessageCreate(s *discordgo.Session, m *discordgo.Message) {
  w.CheckSpam(s, m)
}
func (w *SpamModule) OnCommand(s *discordgo.Session, m *discordgo.Message) bool {
  return w.CheckSpam(s, m)
}
func (w *SpamModule) IsEnabled() bool {
  return true // always enabled
}
func (w *SpamModule) Enable(b bool) {}

func (w *SpamModule) OnGuildMemberAdd(s *discordgo.Session, m *discordgo.Member) {
  raidsize := sb.db.CountNewUsers(sb.config.MaxRaidTime);
  if sb.config.RaidSize > 0 && raidsize >= sb.config.RaidSize && RateLimit(&w.lastraid, sb.config.MaxRaidTime*2) {
    r := sb.db.GetNewestUsers(raidsize)
    s := make([]string, 0, len(r))

    for _, v := range r {
      s = append(s, v.Username + "  (joined: " + v.FirstSeen.Format(time.ANSIC) + ")")
    }
    ch := sb.ModChannelID
    if sb.config.Debug { ch = sb.DebugChannelID }
    sb.SendMessage(ch, "<@&" + sb.ModsRole + "> Possible Raid Detected!\n```" + strings.Join(s, "\n") + "```")
  }
}
func (w *SpamModule) OnGuildMemberUpdate(s *discordgo.Session, m *discordgo.Member) {
  w.OnGuildMemberAdd(s, m)
}
  <commit_msg>Damn it, Blastdown<commit_after>package sweetiebot

import (
  "github.com/bwmarrin/discordgo"
  "time"
  "strconv"
  "strings"
)

// The emote module detects banned emotes and deletes them
type SpamModule struct {
  tracker map[uint64]*SaturationLimit
  channels *map[uint64]bool
  lastraid int64
}

func (w *SpamModule) SetChannelMap(c *map[uint64]bool) {
  w.channels = c
}
func (w *SpamModule) GetChannelMap() *map[uint64]bool {
  return w.channels
}

func (w *SpamModule) Name() string {
  return "Anti-Spam"
}

func (w *SpamModule) Register(hooks *ModuleHooks) {
  w.tracker = make(map[uint64]*SaturationLimit)
  w.lastraid = 0
  hooks.OnMessageCreate = append(hooks.OnMessageCreate, w)
  hooks.OnCommand = append(hooks.OnCommand, w)
  hooks.OnGuildMemberAdd = append(hooks.OnGuildMemberAdd, w)
  hooks.OnGuildMemberUpdate = append(hooks.OnGuildMemberUpdate, w)
}
func (w *SpamModule) Channels() []string {
  return []string{}
}

func KillSpammer(u *discordgo.User) {
  // Manually set our internal state to say this user has the Silent role, to prevent race conditions
  m, err := sb.dg.State.Member(sb.GuildID, u.ID)
  if err == nil {
    for _, v := range m.Roles {
      if v == sb.SilentRole {
        return // Spammer was already killed, so don't try killing it again
      }
    }
    m.Roles = append(m.Roles, sb.SilentRole)
  } else {
    sb.log.Log("Tried to kill spammer ", u.Username, " but they were already banned??? (Error: ", err.Error(), ")")
  }

  sb.log.Log("Killing spammer ", u.Username)

  sb.dg.GuildMemberEdit(sb.GuildID, u.ID, m.Roles) // Tell discord to make this spammer silent
  messages := sb.db.GetRecentMessages(SBatoi(u.ID), 60) // Retrieve all messages in the past 60 seconds and delete them.

  for _, v := range messages {
    sb.dg.ChannelMessageDelete(strconv.FormatUint(v.channel, 10), strconv.FormatUint(v.message, 10))
  }

  sb.SendMessage(sb.ModChannelID, "`Alert: " + u.Username + " was silenced for spamming. Please investigate.`") // Alert admins
}
func (w *SpamModule) CheckSpam(s *discordgo.Session, m *discordgo.Message) bool {
  if m.Author != nil {
    if UserHasRole(m.Author.ID, sb.SilentRole) {
      s.ChannelMessageDelete(m.ChannelID, m.ID);
      return true
    }
    id := SBatoi(m.Author.ID)
    _, ok := w.tracker[id]
    if !ok {
      w.tracker[id] = &SaturationLimit{make([]int64, sb.config.Maxspam, sb.config.Maxspam), 0, AtomicFlag{0}};
    }
    limit := w.tracker[id]
    limit.append(time.Now().UTC().Unix())
    if limit.checkafter(5, 1) || limit.checkafter(10, 5) || limit.checkafter(12, 10) {
      KillSpammer(m.Author)
      return true
    }
  }
  return false
}
func (w *SpamModule) OnMessageCreate(s *discordgo.Session, m *discordgo.Message) {
  w.CheckSpam(s, m)
}
func (w *SpamModule) OnCommand(s *discordgo.Session, m *discordgo.Message) bool {
  return w.CheckSpam(s, m)
}
func (w *SpamModule) IsEnabled() bool {
  return true // always enabled
}
func (w *SpamModule) Enable(b bool) {}

func (w *SpamModule) OnGuildMemberAdd(s *discordgo.Session, m *discordgo.Member) {
  raidsize := sb.db.CountNewUsers(sb.config.MaxRaidTime);
  if sb.config.RaidSize > 0 && raidsize >= sb.config.RaidSize && RateLimit(&w.lastraid, sb.config.MaxRaidTime*2) {
    r := sb.db.GetNewestUsers(raidsize)
    s := make([]string, 0, len(r))

    for _, v := range r {
      s = append(s, v.Username + "  (joined: " + v.FirstSeen.Format(time.ANSIC) + ")")
    }
    ch := sb.ModChannelID
    if sb.config.Debug { ch = sb.DebugChannelID }
    sb.SendMessage(ch, "<@&" + sb.ModsRole + "> Possible Raid Detected!\n```" + strings.Join(s, "\n") + "```")
  }
}
func (w *SpamModule) OnGuildMemberUpdate(s *discordgo.Session, m *discordgo.Member) {
  w.OnGuildMemberAdd(s, m)
}
  <|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"bytes"
	"fmt"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/glog"
	githubapi "github.com/google/go-github/github"
	"github.com/spf13/cobra"

	"k8s.io/contrib/mungegithub/features"
	"k8s.io/contrib/mungegithub/github"
	c "k8s.io/contrib/mungegithub/mungers/matchers/comment"
	"k8s.io/kubernetes/pkg/util/sets"
)

const (
	approvalNotificationName = "ApprovalNotifier"
	approveCommand           = "approve"
	cancel                   = "cancel"
	ownersFileName           = "OWNERS"
)

// ApprovalHandler will try to add "approved" label once
// all files of change has been approved by approvers.
type ApprovalHandler struct {
	features *features.Features
}

func init() {
	h := &ApprovalHandler{}
	RegisterMungerOrDie(h)
}

// Name is the name usable in --pr-mungers
func (*ApprovalHandler) Name() string { return "approval-handler" }

// RequiredFeatures is a slice of 'features' that must be provided
func (*ApprovalHandler) RequiredFeatures() []string {
	return []string{features.RepoFeatureName, features.AliasesFeature}
}

// Initialize will initialize the munger
func (h *ApprovalHandler) Initialize(config *github.Config, features *features.Features) error {
	h.features = features
	return nil
}

// EachLoop is called at the start of every munge loop
func (*ApprovalHandler) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (*ApprovalHandler) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is the workhorse the will actually make updates to the PR
// The algorithm goes as:
// - Initially, we build an approverSet
//   - Go through all comments after latest commit.
//	- If anyone said "/approve", add them to approverSet.
// - Then, for each file, we see if any approver of this file is in approverSet and keep track of files without approval
//   - An approver of a file is defined as:
//     - Someone listed as an "approver" in an OWNERS file in the files directory OR
//     - in one of the file's parent directorie
// - Iff all files have been approved, the bot will add the "approved" label.
// - Iff a cancel command is found, that reviewer will be removed from the approverSet
// 	and the munger will remove the approved label if it has been applied
func (h *ApprovalHandler) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}
	files, ok := obj.ListFiles()
	if !ok {
		return
	}

	comments, ok := getCommentsAfterLastModified(obj)

	if !ok {
		return
	}

	ownersMap := h.getApprovedOwners(files, createApproverSet(comments))

	if err := h.updateNotification(obj, ownersMap); err != nil {
		return
	}

	for _, approverSet := range ownersMap {
		if approverSet.Len() == 0 {
			if obj.HasLabel(approvedLabel) {
				obj.RemoveLabel(approvedLabel)
			}
			return
		}
	}

	if !obj.HasLabel(approvedLabel) {
		obj.AddLabel(approvedLabel)
	}
}

func (h *ApprovalHandler) updateNotification(obj *github.MungeObject, ownersMap map[string]sets.String) error {
	notificationMatcher := c.MungerNotificationName(approvalNotificationName)
	comments, ok := obj.ListComments()
	if !ok {
		return fmt.Errorf("Unable to ListComments for %d", obj.Number())
	}

	notifications := c.FilterComments(comments, notificationMatcher)
	latestNotification := notifications.GetLast()
	if latestNotification == nil {
		body := h.getMessage(obj, ownersMap)
		obj.WriteComment(body)
	}

	latestApprove := c.FilterComments(comments, c.CommandName(approveCommand)).GetLast()
	if latestApprove == nil || latestApprove.CreatedAt == nil {
		// there was already a bot notification and nothing has changed since
		// or we wouldn't tell when the latestApproval occurred
		return nil
	}
	if latestApprove.CreatedAt.After(*latestNotification.CreatedAt) {
		// if someone approved since the last comment, we should update the comment
		glog.Infof("Latest approve was after last time notified")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return fmt.Errorf("Unable to get LastModifiedTime for %d", obj.Number())
	}
	if latestNotification.CreatedAt.Before(*lastModified) {
		// the PR was modified After our last notification, so we should update the approvers notification
		// i.e. People that have formerly approved haven't necessarily approved of new changes
		glog.Infof("PR Modified After Last Notification")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	return nil
}

// findPeopleToApprove Takes the Owners Files that Are Needed for the PR and chooses a good
// subset of Approvers that are guaranteed to cover all of them (exact cover)
// This is a greedy approximation and not guaranteed to find the minimum number of OWNERS
func (h ApprovalHandler) findPeopleToApprove(ownersPaths sets.String, prAuthor string) sets.String {

	// approverCount contains a map: person -> set of relevant OWNERS file they are in
	approverCount := make(map[string]sets.String)
	for ownersFile := range ownersPaths {
		// LeafApprovers removes the last part of a path for dirs and files, so we append owners to the path
		for approver := range h.features.Repos.LeafApprovers(filepath.Join(ownersFile, ownersFileName)) {
			if approver == prAuthor {
				// don't add the author of the PR to the list of candidates that can approve
				continue
			}
			if _, ok := approverCount[approver]; ok {
				approverCount[approver].Insert(ownersFile)
			} else {
				approverCount[approver] = sets.NewString(ownersFile)
			}
		}
	}

	copyOfFiles := sets.NewString()
	for fn := range ownersPaths {
		copyOfFiles.Insert(fn)
	}

	approverGroup := sets.NewString()
	var bestPerson string
	for copyOfFiles.Len() > 0 {
		maxCovered := 0
		for k, v := range approverCount {
			if v.Intersection(copyOfFiles).Len() > maxCovered {
				maxCovered = len(v)
				bestPerson = k
			}
		}

		approverGroup.Insert(bestPerson)
		toDelete := sets.NewString()
		// remove all files in the directories that our approver approved AND
		// in the subdirectories that s/he approved.  HasPrefix finds subdirs
		for fn := range copyOfFiles {
			for approvedFile := range approverCount[bestPerson] {
				if strings.HasPrefix(fn, approvedFile) {
					toDelete.Insert(fn)
				}

			}
		}
		copyOfFiles.Delete(toDelete.List()...)
	}
	return approverGroup
}

// removeSubdirs takes a list of directories as an input and returns a set of directories with all
// subdirectories removed.  E.g. [/a,/a/b/c,/d/e,/d/e/f] -> [/a, /d/e]
func removeSubdirs(dirList []string) sets.String {
	toDel := sets.String{}
	for i := 0; i < len(dirList)-1; i++ {
		for j := i + 1; j < len(dirList); j++ {
			// ex /a/b has prefix /a so if remove /a/b since its already covered
			if strings.HasPrefix(dirList[i], dirList[j]) {
				toDel.Insert(dirList[i])
			} else if strings.HasPrefix(dirList[j], dirList[i]) {
				toDel.Insert(dirList[j])
			}
		}
	}
	finalSet := sets.NewString(dirList...)
	finalSet.Delete(toDel.List()...)
	return finalSet
}

// getMessage returns the comment body that we want the approval-handler to display on PRs
// The comment shows:
// 	- a list of approvers files (and links) needed to get the PR approved
// 	- a list of approvers files with strikethroughs that already have an approver's approval
// 	- a suggested list of people from each OWNERS files that can fully approve the PR
// 	- how an approver can indicate their approval
// 	- how an approver can cancel their approval
func (h *ApprovalHandler) getMessage(obj *github.MungeObject, ownersMap map[string]sets.String) string {
	// sort the keys so we always display OWNERS files in same order
	sliceOfKeys := make([]string, len(ownersMap))
	i := 0
	for path := range ownersMap {
		sliceOfKeys[i] = path
		i++
	}
	sort.Strings(sliceOfKeys)

	unapprovedOwners := sets.NewString()
	context := bytes.NewBufferString("")
	for _, path := range sliceOfKeys {
		approverSet := ownersMap[path]
		if approverSet.Len() == 0 {
			fullOwnersPath := filepath.Join(path, ownersFileName)
			link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", obj.Org(), obj.Project(), fullOwnersPath)
			context.WriteString(fmt.Sprintf("- **[%s](%s)** \n", fullOwnersPath, link))
			unapprovedOwners.Insert(path)
		} else {
			context.WriteString(fmt.Sprintf("- ~~%s~~ [%v]\n", path, strings.Join(approverSet.List(), ",")))
		}
	}
	context.WriteString("\n")
	if unapprovedOwners.Len() > 0 {
		context.WriteString("We suggest the following people:\n")
		context.WriteString("cc ")
		toBeAssigned := h.findPeopleToApprove(unapprovedOwners, *obj.Issue.User.Login)
		for person := range toBeAssigned {
			context.WriteString("@" + person + " ")
		}
	}
	context.WriteString("\n You can indicate your approval by writing `/approve` in a comment")
	context.WriteString("\n You can cancel your approval by writing `/approve cancel` in a comment")
	notif := c.Notification{approvalNotificationName, "Needs approval from an approver in each of these OWNERS Files:\n", context.String()}
	return notif.String()
}

// createApproverSet iterates through the list of comments on a PR
// and identifies all of the people that have said /approve and adds
// them to the approverSet.  The function uses the latest approve or cancel comment
// to determine the Users intention
func createApproverSet(comments []*githubapi.IssueComment) sets.String {
	approverSet := sets.NewString()

	approverMatcher := c.CommandName(approveCommand)
	for _, comment := range c.FilterComments(comments, approverMatcher) {
		cmd := c.ParseCommand(comment)
		if cmd.Arguments == cancel {
			approverSet.Delete(*comment.User.Login)
		} else {
			approverSet.Insert(*comment.User.Login)
		}
	}
	return approverSet
}

// getApprovedOwners finds all the relevant OWNERS files for the PRs and identifies all the people from them
// that have approved the PR.  For all files that have not been approved, it finds the minimum number of owners files
// that cover all of them.  E.g. If /a/b/c.txt and /a/d.txt need approval, it will only indicate that an approval from
// someone in /a/OWNERS is needed
func (h ApprovalHandler) getApprovedOwners(files []*githubapi.CommitFile, approverSet sets.String) map[string]sets.String {
	ownersApprovers := make(map[string]sets.String)
	// TODO: go through the files starting at the top of the tree
	needsApproval := sets.NewString()
	for _, file := range files {
		fileOwners := h.features.Repos.Approvers(*file.Filename)
		ownersFile := h.features.Repos.FindOwnersForPath(*file.Filename)
		hasApproved := fileOwners.Intersection(approverSet)
		if len(hasApproved) != 0 {
			ownersApprovers[ownersFile] = hasApproved
		} else {
			needsApproval.Insert(ownersFile)
		}

	}
	needsApproval = removeSubdirs(needsApproval.List())
	for fn := range needsApproval {
		ownersApprovers[fn] = sets.NewString()
	}
	return ownersApprovers
}

// gets the comments since the obj was last changed.  If we can't figure out when the object was last changed
// return all the comments on the issue
func getCommentsAfterLastModified(obj *github.MungeObject) ([]*githubapi.IssueComment, bool) {
	comments, ok := obj.ListComments()
	if !ok {
		return comments, ok
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return comments, ok
	}
	return c.FilterComments(comments, c.CreatedAfter(*lastModified)), true
}
<commit_msg>mungegithub: Write consistent approval OWNERS path<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"bytes"
	"fmt"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/glog"
	githubapi "github.com/google/go-github/github"
	"github.com/spf13/cobra"

	"k8s.io/contrib/mungegithub/features"
	"k8s.io/contrib/mungegithub/github"
	c "k8s.io/contrib/mungegithub/mungers/matchers/comment"
	"k8s.io/kubernetes/pkg/util/sets"
)

const (
	approvalNotificationName = "ApprovalNotifier"
	approveCommand           = "approve"
	cancel                   = "cancel"
	ownersFileName           = "OWNERS"
)

// ApprovalHandler will try to add "approved" label once
// all files of change has been approved by approvers.
type ApprovalHandler struct {
	features *features.Features
}

func init() {
	h := &ApprovalHandler{}
	RegisterMungerOrDie(h)
}

// Name is the name usable in --pr-mungers
func (*ApprovalHandler) Name() string { return "approval-handler" }

// RequiredFeatures is a slice of 'features' that must be provided
func (*ApprovalHandler) RequiredFeatures() []string {
	return []string{features.RepoFeatureName, features.AliasesFeature}
}

// Initialize will initialize the munger
func (h *ApprovalHandler) Initialize(config *github.Config, features *features.Features) error {
	h.features = features
	return nil
}

// EachLoop is called at the start of every munge loop
func (*ApprovalHandler) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (*ApprovalHandler) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is the workhorse the will actually make updates to the PR
// The algorithm goes as:
// - Initially, we build an approverSet
//   - Go through all comments after latest commit.
//	- If anyone said "/approve", add them to approverSet.
// - Then, for each file, we see if any approver of this file is in approverSet and keep track of files without approval
//   - An approver of a file is defined as:
//     - Someone listed as an "approver" in an OWNERS file in the files directory OR
//     - in one of the file's parent directorie
// - Iff all files have been approved, the bot will add the "approved" label.
// - Iff a cancel command is found, that reviewer will be removed from the approverSet
// 	and the munger will remove the approved label if it has been applied
func (h *ApprovalHandler) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}
	files, ok := obj.ListFiles()
	if !ok {
		return
	}

	comments, ok := getCommentsAfterLastModified(obj)

	if !ok {
		return
	}

	ownersMap := h.getApprovedOwners(files, createApproverSet(comments))

	if err := h.updateNotification(obj, ownersMap); err != nil {
		return
	}

	for _, approverSet := range ownersMap {
		if approverSet.Len() == 0 {
			if obj.HasLabel(approvedLabel) {
				obj.RemoveLabel(approvedLabel)
			}
			return
		}
	}

	if !obj.HasLabel(approvedLabel) {
		obj.AddLabel(approvedLabel)
	}
}

func (h *ApprovalHandler) updateNotification(obj *github.MungeObject, ownersMap map[string]sets.String) error {
	notificationMatcher := c.MungerNotificationName(approvalNotificationName)
	comments, ok := obj.ListComments()
	if !ok {
		return fmt.Errorf("Unable to ListComments for %d", obj.Number())
	}

	notifications := c.FilterComments(comments, notificationMatcher)
	latestNotification := notifications.GetLast()
	if latestNotification == nil {
		body := h.getMessage(obj, ownersMap)
		obj.WriteComment(body)
	}

	latestApprove := c.FilterComments(comments, c.CommandName(approveCommand)).GetLast()
	if latestApprove == nil || latestApprove.CreatedAt == nil {
		// there was already a bot notification and nothing has changed since
		// or we wouldn't tell when the latestApproval occurred
		return nil
	}
	if latestApprove.CreatedAt.After(*latestNotification.CreatedAt) {
		// if someone approved since the last comment, we should update the comment
		glog.Infof("Latest approve was after last time notified")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return fmt.Errorf("Unable to get LastModifiedTime for %d", obj.Number())
	}
	if latestNotification.CreatedAt.Before(*lastModified) {
		// the PR was modified After our last notification, so we should update the approvers notification
		// i.e. People that have formerly approved haven't necessarily approved of new changes
		glog.Infof("PR Modified After Last Notification")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	return nil
}

// findPeopleToApprove Takes the Owners Files that Are Needed for the PR and chooses a good
// subset of Approvers that are guaranteed to cover all of them (exact cover)
// This is a greedy approximation and not guaranteed to find the minimum number of OWNERS
func (h ApprovalHandler) findPeopleToApprove(ownersPaths sets.String, prAuthor string) sets.String {

	// approverCount contains a map: person -> set of relevant OWNERS file they are in
	approverCount := make(map[string]sets.String)
	for ownersFile := range ownersPaths {
		// LeafApprovers removes the last part of a path for dirs and files, so we append owners to the path
		for approver := range h.features.Repos.LeafApprovers(filepath.Join(ownersFile, ownersFileName)) {
			if approver == prAuthor {
				// don't add the author of the PR to the list of candidates that can approve
				continue
			}
			if _, ok := approverCount[approver]; ok {
				approverCount[approver].Insert(ownersFile)
			} else {
				approverCount[approver] = sets.NewString(ownersFile)
			}
		}
	}

	copyOfFiles := sets.NewString()
	for fn := range ownersPaths {
		copyOfFiles.Insert(fn)
	}

	approverGroup := sets.NewString()
	var bestPerson string
	for copyOfFiles.Len() > 0 {
		maxCovered := 0
		for k, v := range approverCount {
			if v.Intersection(copyOfFiles).Len() > maxCovered {
				maxCovered = len(v)
				bestPerson = k
			}
		}

		approverGroup.Insert(bestPerson)
		toDelete := sets.NewString()
		// remove all files in the directories that our approver approved AND
		// in the subdirectories that s/he approved.  HasPrefix finds subdirs
		for fn := range copyOfFiles {
			for approvedFile := range approverCount[bestPerson] {
				if strings.HasPrefix(fn, approvedFile) {
					toDelete.Insert(fn)
				}

			}
		}
		copyOfFiles.Delete(toDelete.List()...)
	}
	return approverGroup
}

// removeSubdirs takes a list of directories as an input and returns a set of directories with all
// subdirectories removed.  E.g. [/a,/a/b/c,/d/e,/d/e/f] -> [/a, /d/e]
func removeSubdirs(dirList []string) sets.String {
	toDel := sets.String{}
	for i := 0; i < len(dirList)-1; i++ {
		for j := i + 1; j < len(dirList); j++ {
			// ex /a/b has prefix /a so if remove /a/b since its already covered
			if strings.HasPrefix(dirList[i], dirList[j]) {
				toDel.Insert(dirList[i])
			} else if strings.HasPrefix(dirList[j], dirList[i]) {
				toDel.Insert(dirList[j])
			}
		}
	}
	finalSet := sets.NewString(dirList...)
	finalSet.Delete(toDel.List()...)
	return finalSet
}

// getMessage returns the comment body that we want the approval-handler to display on PRs
// The comment shows:
// 	- a list of approvers files (and links) needed to get the PR approved
// 	- a list of approvers files with strikethroughs that already have an approver's approval
// 	- a suggested list of people from each OWNERS files that can fully approve the PR
// 	- how an approver can indicate their approval
// 	- how an approver can cancel their approval
func (h *ApprovalHandler) getMessage(obj *github.MungeObject, ownersMap map[string]sets.String) string {
	// sort the keys so we always display OWNERS files in same order
	sliceOfKeys := make([]string, len(ownersMap))
	i := 0
	for path := range ownersMap {
		sliceOfKeys[i] = path
		i++
	}
	sort.Strings(sliceOfKeys)

	unapprovedOwners := sets.NewString()
	context := bytes.NewBufferString("")
	for _, path := range sliceOfKeys {
		approverSet := ownersMap[path]
		fullOwnersPath := filepath.Join(path, ownersFileName)
		link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", obj.Org(), obj.Project(), fullOwnersPath)

		if approverSet.Len() == 0 {
			context.WriteString(fmt.Sprintf("- **[%s](%s)** \n", fullOwnersPath, link))
			unapprovedOwners.Insert(path)
		} else {
			context.WriteString(fmt.Sprintf("- ~~[%s](%s)~~ [%v]\n", fullOwnersPath, link, strings.Join(approverSet.List(), ",")))
		}
	}
	context.WriteString("\n")
	if unapprovedOwners.Len() > 0 {
		context.WriteString("We suggest the following people:\n")
		context.WriteString("cc ")
		toBeAssigned := h.findPeopleToApprove(unapprovedOwners, *obj.Issue.User.Login)
		for person := range toBeAssigned {
			context.WriteString("@" + person + " ")
		}
	}
	context.WriteString("\n You can indicate your approval by writing `/approve` in a comment")
	context.WriteString("\n You can cancel your approval by writing `/approve cancel` in a comment")
	notif := c.Notification{approvalNotificationName, "Needs approval from an approver in each of these OWNERS Files:\n", context.String()}
	return notif.String()
}

// createApproverSet iterates through the list of comments on a PR
// and identifies all of the people that have said /approve and adds
// them to the approverSet.  The function uses the latest approve or cancel comment
// to determine the Users intention
func createApproverSet(comments []*githubapi.IssueComment) sets.String {
	approverSet := sets.NewString()

	approverMatcher := c.CommandName(approveCommand)
	for _, comment := range c.FilterComments(comments, approverMatcher) {
		cmd := c.ParseCommand(comment)
		if cmd.Arguments == cancel {
			approverSet.Delete(*comment.User.Login)
		} else {
			approverSet.Insert(*comment.User.Login)
		}
	}
	return approverSet
}

// getApprovedOwners finds all the relevant OWNERS files for the PRs and identifies all the people from them
// that have approved the PR.  For all files that have not been approved, it finds the minimum number of owners files
// that cover all of them.  E.g. If /a/b/c.txt and /a/d.txt need approval, it will only indicate that an approval from
// someone in /a/OWNERS is needed
func (h ApprovalHandler) getApprovedOwners(files []*githubapi.CommitFile, approverSet sets.String) map[string]sets.String {
	ownersApprovers := make(map[string]sets.String)
	// TODO: go through the files starting at the top of the tree
	needsApproval := sets.NewString()
	for _, file := range files {
		fileOwners := h.features.Repos.Approvers(*file.Filename)
		ownersFile := h.features.Repos.FindOwnersForPath(*file.Filename)
		hasApproved := fileOwners.Intersection(approverSet)
		if len(hasApproved) != 0 {
			ownersApprovers[ownersFile] = hasApproved
		} else {
			needsApproval.Insert(ownersFile)
		}

	}
	needsApproval = removeSubdirs(needsApproval.List())
	for fn := range needsApproval {
		ownersApprovers[fn] = sets.NewString()
	}
	return ownersApprovers
}

// gets the comments since the obj was last changed.  If we can't figure out when the object was last changed
// return all the comments on the issue
func getCommentsAfterLastModified(obj *github.MungeObject) ([]*githubapi.IssueComment, bool) {
	comments, ok := obj.ListComments()
	if !ok {
		return comments, ok
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return comments, ok
	}
	return c.FilterComments(comments, c.CreatedAfter(*lastModified)), true
}
<|endoftext|>


<commit_before>// +build linux

package libcontainer

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

func newStateTransitionError(from, to containerState) error {
	return &stateTransitionError{
		From: from.status().String(),
		To:   to.status().String(),
	}
}

// stateTransitionError is returned when an invalid state transition happens from one
// state to another.
type stateTransitionError struct {
	From string
	To   string
}

func (s *stateTransitionError) Error() string {
	return fmt.Sprintf("invalid state transition from %s to %s", s.From, s.To)
}

type containerState interface {
	transition(containerState) error
	destroy() error
	status() Status
}

func destroy(c *linuxContainer) error {
	if !c.config.Namespaces.Contains(configs.NEWPID) {
		if err := signalAllProcesses(c.cgroupManager, unix.SIGKILL); err != nil {
			logrus.Warn(err)
		}
	}
	err := c.cgroupManager.Destroy()
	if c.intelRdtManager != nil {
		if ierr := c.intelRdtManager.Destroy(); err == nil {
			err = ierr
		}
	}
	if rerr := os.RemoveAll(c.root); err == nil {
		err = rerr
	}
	c.initProcess = nil
	if herr := runPoststopHooks(c); err == nil {
		err = herr
	}
	c.state = &stoppedState{c: c}
	return err
}

func runPoststopHooks(c *linuxContainer) error {
	hooks := c.config.Hooks
	if hooks == nil {
		return nil
	}

	s, err := c.currentOCIState()
	if err != nil {
		return err
	}
	s.Status = specs.StateStopped

	if err := hooks[configs.Poststop].RunHooks(s); err != nil {
		return err
	}

	return nil
}

// stoppedState represents a container is a stopped/destroyed state.
type stoppedState struct {
	c *linuxContainer
}

func (b *stoppedState) status() Status {
	return Stopped
}

func (b *stoppedState) transition(s containerState) error {
	switch s.(type) {
	case *runningState, *restoredState:
		b.c.state = s
		return nil
	case *stoppedState:
		return nil
	}
	return newStateTransitionError(b, s)
}

func (b *stoppedState) destroy() error {
	return destroy(b.c)
}

// runningState represents a container that is currently running.
type runningState struct {
	c *linuxContainer
}

func (r *runningState) status() Status {
	return Running
}

func (r *runningState) transition(s containerState) error {
	switch s.(type) {
	case *stoppedState:
		if r.c.runType() == Running {
			return newGenericError(fmt.Errorf("container still running"), ContainerNotStopped)
		}
		r.c.state = s
		return nil
	case *pausedState:
		r.c.state = s
		return nil
	case *runningState:
		return nil
	}
	return newStateTransitionError(r, s)
}

func (r *runningState) destroy() error {
	if r.c.runType() == Running {
		return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped)
	}
	return destroy(r.c)
}

type createdState struct {
	c *linuxContainer
}

func (i *createdState) status() Status {
	return Created
}

func (i *createdState) transition(s containerState) error {
	switch s.(type) {
	case *runningState, *pausedState, *stoppedState:
		i.c.state = s
		return nil
	case *createdState:
		return nil
	}
	return newStateTransitionError(i, s)
}

func (i *createdState) destroy() error {
	i.c.initProcess.signal(unix.SIGKILL)
	return destroy(i.c)
}

// pausedState represents a container that is currently pause.  It cannot be destroyed in a
// paused state and must transition back to running first.
type pausedState struct {
	c *linuxContainer
}

func (p *pausedState) status() Status {
	return Paused
}

func (p *pausedState) transition(s containerState) error {
	switch s.(type) {
	case *runningState, *stoppedState:
		p.c.state = s
		return nil
	case *pausedState:
		return nil
	}
	return newStateTransitionError(p, s)
}

func (p *pausedState) destroy() error {
	t := p.c.runType()
	if t != Running && t != Created {
		if err := p.c.cgroupManager.Freeze(configs.Thawed); err != nil {
			return err
		}
		return destroy(p.c)
	}
	return newGenericError(fmt.Errorf("container is paused"), ContainerPaused)
}

// restoredState is the same as the running state but also has associated checkpoint
// information that maybe need destroyed when the container is stopped and destroy is called.
type restoredState struct {
	imageDir string
	c        *linuxContainer
}

func (r *restoredState) status() Status {
	return Running
}

func (r *restoredState) transition(s containerState) error {
	switch s.(type) {
	case *stoppedState, *runningState:
		return nil
	}
	return newStateTransitionError(r, s)
}

func (r *restoredState) destroy() error {
	if _, err := os.Stat(filepath.Join(r.c.root, "checkpoint")); err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	}
	return destroy(r.c)
}

// loadedState is used whenever a container is restored, loaded, or setting additional
// processes inside and it should not be destroyed when it is exiting.
type loadedState struct {
	c *linuxContainer
	s Status
}

func (n *loadedState) status() Status {
	return n.s
}

func (n *loadedState) transition(s containerState) error {
	n.c.state = s
	return nil
}

func (n *loadedState) destroy() error {
	if err := n.c.refreshState(); err != nil {
		return err
	}
	return n.c.state.destroy()
}
<commit_msg>Make sure signalAllProcesses is invoked in the function of destroy<commit_after>// +build linux

package libcontainer

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

func newStateTransitionError(from, to containerState) error {
	return &stateTransitionError{
		From: from.status().String(),
		To:   to.status().String(),
	}
}

// stateTransitionError is returned when an invalid state transition happens from one
// state to another.
type stateTransitionError struct {
	From string
	To   string
}

func (s *stateTransitionError) Error() string {
	return fmt.Sprintf("invalid state transition from %s to %s", s.From, s.To)
}

type containerState interface {
	transition(containerState) error
	destroy() error
	status() Status
}

func destroy(c *linuxContainer) error {
	if !c.config.Namespaces.Contains(configs.NEWPID) ||
		c.config.Namespaces.PathOf(configs.NEWPID) != "" {
		if err := signalAllProcesses(c.cgroupManager, unix.SIGKILL); err != nil {
			logrus.Warn(err)
		}
	}
	err := c.cgroupManager.Destroy()
	if c.intelRdtManager != nil {
		if ierr := c.intelRdtManager.Destroy(); err == nil {
			err = ierr
		}
	}
	if rerr := os.RemoveAll(c.root); err == nil {
		err = rerr
	}
	c.initProcess = nil
	if herr := runPoststopHooks(c); err == nil {
		err = herr
	}
	c.state = &stoppedState{c: c}
	return err
}

func runPoststopHooks(c *linuxContainer) error {
	hooks := c.config.Hooks
	if hooks == nil {
		return nil
	}

	s, err := c.currentOCIState()
	if err != nil {
		return err
	}
	s.Status = specs.StateStopped

	if err := hooks[configs.Poststop].RunHooks(s); err != nil {
		return err
	}

	return nil
}

// stoppedState represents a container is a stopped/destroyed state.
type stoppedState struct {
	c *linuxContainer
}

func (b *stoppedState) status() Status {
	return Stopped
}

func (b *stoppedState) transition(s containerState) error {
	switch s.(type) {
	case *runningState, *restoredState:
		b.c.state = s
		return nil
	case *stoppedState:
		return nil
	}
	return newStateTransitionError(b, s)
}

func (b *stoppedState) destroy() error {
	return destroy(b.c)
}

// runningState represents a container that is currently running.
type runningState struct {
	c *linuxContainer
}

func (r *runningState) status() Status {
	return Running
}

func (r *runningState) transition(s containerState) error {
	switch s.(type) {
	case *stoppedState:
		if r.c.runType() == Running {
			return newGenericError(fmt.Errorf("container still running"), ContainerNotStopped)
		}
		r.c.state = s
		return nil
	case *pausedState:
		r.c.state = s
		return nil
	case *runningState:
		return nil
	}
	return newStateTransitionError(r, s)
}

func (r *runningState) destroy() error {
	if r.c.runType() == Running {
		return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped)
	}
	return destroy(r.c)
}

type createdState struct {
	c *linuxContainer
}

func (i *createdState) status() Status {
	return Created
}

func (i *createdState) transition(s containerState) error {
	switch s.(type) {
	case *runningState, *pausedState, *stoppedState:
		i.c.state = s
		return nil
	case *createdState:
		return nil
	}
	return newStateTransitionError(i, s)
}

func (i *createdState) destroy() error {
	i.c.initProcess.signal(unix.SIGKILL)
	return destroy(i.c)
}

// pausedState represents a container that is currently pause.  It cannot be destroyed in a
// paused state and must transition back to running first.
type pausedState struct {
	c *linuxContainer
}

func (p *pausedState) status() Status {
	return Paused
}

func (p *pausedState) transition(s containerState) error {
	switch s.(type) {
	case *runningState, *stoppedState:
		p.c.state = s
		return nil
	case *pausedState:
		return nil
	}
	return newStateTransitionError(p, s)
}

func (p *pausedState) destroy() error {
	t := p.c.runType()
	if t != Running && t != Created {
		if err := p.c.cgroupManager.Freeze(configs.Thawed); err != nil {
			return err
		}
		return destroy(p.c)
	}
	return newGenericError(fmt.Errorf("container is paused"), ContainerPaused)
}

// restoredState is the same as the running state but also has associated checkpoint
// information that maybe need destroyed when the container is stopped and destroy is called.
type restoredState struct {
	imageDir string
	c        *linuxContainer
}

func (r *restoredState) status() Status {
	return Running
}

func (r *restoredState) transition(s containerState) error {
	switch s.(type) {
	case *stoppedState, *runningState:
		return nil
	}
	return newStateTransitionError(r, s)
}

func (r *restoredState) destroy() error {
	if _, err := os.Stat(filepath.Join(r.c.root, "checkpoint")); err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	}
	return destroy(r.c)
}

// loadedState is used whenever a container is restored, loaded, or setting additional
// processes inside and it should not be destroyed when it is exiting.
type loadedState struct {
	c *linuxContainer
	s Status
}

func (n *loadedState) status() Status {
	return n.s
}

func (n *loadedState) transition(s containerState) error {
	n.c.state = s
	return nil
}

func (n *loadedState) destroy() error {
	if err := n.c.refreshState(); err != nil {
		return err
	}
	return n.c.state.destroy()
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"path/filepath"
	"sync"

	"github.com/keybase/kbfs/kbfsblock"
	"github.com/keybase/kbfs/kbfscrypto"
	"github.com/keybase/kbfs/tlf"
	"github.com/syndtr/goleveldb/leveldb"
	"golang.org/x/net/context"
)

const (
	evictionConsiderationFactor uint   = 3
	blockDbFilename             string = "diskCacheBlocks.leveldb"
	lruDbFilename               string = "diskCacheLRU.leveldb"
)

// DiskBlockCacheStandard is the standard implementation for DiskBlockCache.
type DiskBlockCacheStandard struct {
	maxBytes uint64
	// protects everything below
	lock     sync.RWMutex
	isClosed bool
	blockDb  *leveldb.DB
	lruDb    *leveldb.DB
}

var _ DiskBlockCache = (*DiskBlockCacheStandard)(nil)

func newDiskBlockCacheStandard(dirPath string, maxBytes uint64) (
	*DiskBlockCacheStandard, error) {
	blockDbPath := filepath.Join(dirPath, blockDbFilename)
	blockDb, err := leveldb.OpenFile(blockDbPath, leveldbOptions)
	if err != nil {
		return nil, err
	}

	lruDbPath := filepath.Join(dirPath, lruDbFilename)
	lruDb, err := leveldb.OpenFile(lruDbPath, leveldbOptions)
	if err != nil {
		blockDb.Close()
		return nil, err
	}
	return &DiskBlockCacheStandard{
		blockDb:  blockDb,
		lruDb:    lruDb,
		maxBytes: maxBytes,
	}, nil
}

// Get implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Get(ctx context.Context, tlfID tlf.ID,
	blockID kbfsblock.ID) ([]byte, kbfscrypto.BlockCryptKeyServerHalf, error) {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return nil, kbfscrypto.BlockCryptKeyServerHalf{},
			DiskCacheClosedError{"Get"}
	}
	return nil, kbfscrypto.BlockCryptKeyServerHalf{}, NoSuchBlockError{blockID}
}

// Put implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Put(ctx context.Context, tlfID tlf.ID,
	blockID kbfsblock.ID, buf []byte,
	serverHalf kbfscrypto.BlockCryptKeyServerHalf) error {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return DiskCacheClosedError{"Put"}
	}
	return nil
}

// Delete implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Delete(ctx context.Context, tlfID tlf.ID,
	blockID kbfsblock.ID) error {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return DiskCacheClosedError{"Delete"}
	}
	return nil
}

// Evict implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Evict(ctx context.Context, tlfID tlf.ID,
	numBlocks int) error {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return DiskCacheClosedError{"Evict"}
	}
	return nil
}

// Shutdown implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Shutdown() {
	cache.lock.Lock()
	defer cache.lock.Unlock()
	if cache.isClosed {
		return
	}
	cache.isClosed = true
	cache.blockDb.Close()
	cache.lruDb.Close()
}
<commit_msg>disk_block_cache: Add comment detailing eviction algorithm<commit_after>package libkbfs

import (
	"path/filepath"
	"sync"

	"github.com/keybase/kbfs/kbfsblock"
	"github.com/keybase/kbfs/kbfscrypto"
	"github.com/keybase/kbfs/tlf"
	"github.com/syndtr/goleveldb/leveldb"
	"golang.org/x/net/context"
)

const (
	evictionConsiderationFactor uint   = 3
	blockDbFilename             string = "diskCacheBlocks.leveldb"
	lruDbFilename               string = "diskCacheLRU.leveldb"
)

// DiskBlockCacheStandard is the standard implementation for DiskBlockCache.
type DiskBlockCacheStandard struct {
	maxBytes uint64
	// protects everything below
	lock     sync.RWMutex
	isClosed bool
	blockDb  *leveldb.DB
	lruDb    *leveldb.DB
}

var _ DiskBlockCache = (*DiskBlockCacheStandard)(nil)

func newDiskBlockCacheStandard(dirPath string, maxBytes uint64) (
	*DiskBlockCacheStandard, error) {
	blockDbPath := filepath.Join(dirPath, blockDbFilename)
	blockDb, err := leveldb.OpenFile(blockDbPath, leveldbOptions)
	if err != nil {
		return nil, err
	}

	lruDbPath := filepath.Join(dirPath, lruDbFilename)
	lruDb, err := leveldb.OpenFile(lruDbPath, leveldbOptions)
	if err != nil {
		blockDb.Close()
		return nil, err
	}
	return &DiskBlockCacheStandard{
		blockDb:  blockDb,
		lruDb:    lruDb,
		maxBytes: maxBytes,
	}, nil
}

// Get implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Get(ctx context.Context, tlfID tlf.ID,
	blockID kbfsblock.ID) ([]byte, kbfscrypto.BlockCryptKeyServerHalf, error) {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return nil, kbfscrypto.BlockCryptKeyServerHalf{},
			DiskCacheClosedError{"Get"}
	}
	return nil, kbfscrypto.BlockCryptKeyServerHalf{}, NoSuchBlockError{blockID}
}

// Put implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Put(ctx context.Context, tlfID tlf.ID,
	blockID kbfsblock.ID, buf []byte,
	serverHalf kbfscrypto.BlockCryptKeyServerHalf) error {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return DiskCacheClosedError{"Put"}
	}
	return nil
}

// Delete implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Delete(ctx context.Context, tlfID tlf.ID,
	blockID kbfsblock.ID) error {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return DiskCacheClosedError{"Delete"}
	}
	return nil
}

// Evict implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Evict(ctx context.Context, tlfID tlf.ID,
	numBlocks int) error {
	cache.lock.RLock()
	defer cache.lock.RUnlock()
	if cache.isClosed {
		return DiskCacheClosedError{"Evict"}
	}
	// Use kbfscrypto.MakeTemporaryID() to create a random hash ID. Then begin
	// an interator into cache.lruDb.Range(b, nil) and iterate from there to
	// get numBlocks * evictionConsiderationFactor block IDs.  We sort the
	// resulting blocks by value (LRU time) and pick the minimum numBlocks. We
	// put those block IDs into a leveldb.Batch for cache.blockDb via
	// Batch.Delete(), then Write() that batch.
	// NOTE: It is important that we store LRU times using a monotonic clock
	// for this device. Use runtime.nanotime() for now.
	return nil
}

// Shutdown implements the DiskBlockCache interface for DiskBlockCacheStandard.
func (cache *DiskBlockCacheStandard) Shutdown() {
	cache.lock.Lock()
	defer cache.lock.Unlock()
	if cache.isClosed {
		return
	}
	cache.isClosed = true
	cache.blockDb.Close()
	cache.lruDb.Close()
}
<|endoftext|>


<commit_before>package version

import (
	"encoding/json"
	"fmt"
	goupdate "github.com/inconshreveable/go-update"
	"how/request"
	"io/ioutil"
	"os"
	"runtime"
	"strconv"
)

const (
	Version     = 1.2
	Url         = "https://api.github.com/repos/brettbukowski/how/releases"
	DownloadUrl = "https://github.com/BrettBukowski/how/releases/download/%.1f/how-%s"
)

var releases = make([]map[string]interface{}, 10)
var newestVersion = 0.0

func getReleases() []map[string]interface{} {
	response, _ := request.Get(Url, map[string]string{})

	defer response.Body.Close()

	body, _ := ioutil.ReadAll(response.Body)

	if err := json.Unmarshal(body, &releases); err != nil {
		panic(err)
	}

	return releases
}

func getNewestVersion() float64 {
	releases := getReleases()
	latestInfo := releases[0]
	latestVersion, _ := strconv.ParseFloat(latestInfo["name"].(string), 64)

	return latestVersion
}

func download(url string) (err error, errRecover error) {
	fmt.Println("Downloading from <%s>", url)

	dl := goupdate.NewDownload(url)

	if err = dl.Get(); err != nil || !dl.Available {
		return
	}

	fmt.Println("Updating...")

	if err, errRecover = goupdate.FromFile(dl.Path); err != nil || errRecover != nil {
		return
	}

	os.Remove(dl.Path)

	fmt.Println("Updated!")

	return
}

func NewerVersionAvailable() bool {
	newest := NewestVersion()

	return newest > Version
}

func NewestVersion() float64 {
	if newestVersion == 0.0 {
		// Cache the operation.
		newestVersion = getNewestVersion()
	}

	return newestVersion
}

func Update() bool {
	if !NewerVersionAvailable() {
		return false
	}

	var err error
	var errRecover error

	if err = goupdate.SanityCheck(); err != nil {
		fmt.Println(err)
		return false
	}

	version := NewestVersion()
	platform := runtime.GOOS

	var target string

	if platform == "darwin" {
		target = "osx"
	} else if platform == "linux" {
		target = "linux-amd64"
	} else if platform == "windows" {
		target = "windows-amd64"
	}

	if err, errRecover = download(fmt.Sprintf(DownloadUrl, version, target)); err != nil || errRecover != nil {
		fmt.Println(err)
		return false
	}

	return true
}
<commit_msg>Fix download url<commit_after>package version

import (
	"encoding/json"
	"fmt"
	goupdate "github.com/inconshreveable/go-update"
	"how/request"
	"io/ioutil"
	"os"
	"runtime"
	"strconv"
)

const (
	Version     = 1.3
	Url         = "https://api.github.com/repos/brettbukowski/how/releases"
	DownloadUrl = "https://github.com/BrettBukowski/how/releases/download/v%.1f/how-%s"
)

var releases = make([]map[string]interface{}, 10)
var newestVersion = 0.0

func getReleases() []map[string]interface{} {
	response, _ := request.Get(Url, map[string]string{})

	defer response.Body.Close()

	body, _ := ioutil.ReadAll(response.Body)

	if err := json.Unmarshal(body, &releases); err != nil {
		panic(err)
	}

	return releases
}

func getNewestVersion() float64 {
	releases := getReleases()
	latestInfo := releases[0]
	latestVersion, _ := strconv.ParseFloat(latestInfo["name"].(string), 64)

	return latestVersion
}

func download(url string) (err error, errRecover error) {
	fmt.Printf("Downloading from <%s>\n", url)

	dl := goupdate.NewDownload(url)

	if err = dl.Get(); err != nil || !dl.Available {
		return
	}

	fmt.Println("Updating...")

	if err, errRecover = goupdate.FromFile(dl.Path); err != nil || errRecover != nil {
		return
	}

	os.Remove(dl.Path)

	fmt.Println("Updated!")

	return
}

func NewerVersionAvailable() bool {
	newest := NewestVersion()

	return newest > Version
}

func NewestVersion() float64 {
	if newestVersion == 0.0 {
		// Cache the operation.
		newestVersion = getNewestVersion()
	}

	return newestVersion
}

func Update() bool {
	if !NewerVersionAvailable() {
		return false
	}

	var err error
	var errRecover error

	if err = goupdate.SanityCheck(); err != nil {
		fmt.Println(err)
		return false
	}

	version := NewestVersion()
	platform := runtime.GOOS

	var target string

	if platform == "darwin" {
		target = "osx"
	} else if platform == "linux" {
		target = "linux-amd64"
	} else if platform == "windows" {
		target = "windows-amd64"
	}

	if err, errRecover = download(fmt.Sprintf(DownloadUrl, version, target)); err != nil || errRecover != nil {
		fmt.Println(err)
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package http_util

import (
	// "crypto/md5"
	// "errors"
	"curl_cmd"
	// "fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"
)

const ChromeUserAgent string = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.56 Safari/537.17"

type ResourceInfo struct {
	length   int64
	filename string
}
type RedirectError struct {
	s string
}

func (re RedirectError) Error() string {
	return re.s
}

func CheckRedirect(*http.Request, []*http.Request) error {
	return RedirectError{"don't follow redirect"}
}

type myUrlEror url.Error

func (e myUrlEror) Error() string {
	return (&e).Error()
}

var open_file_func openFileFunc = NewFile

const (
	BlockSize                 int64 = 1024 * 1024
	NBlocksPerRequest               = 300
	DownloaderChanBufferSize        = 100
	TimeoutOfGetResourceInfo        = 60 * time.Second
	TimeoutOfPerBlockDownload       = 1024 / 42 * time.Second
)

type DownloadRange struct {
	Start  int64
	Length int64
}

type File interface {
	Size() int64
	Name() string
	io.ReadWriteSeeker
	Truncate(size int64) error
	WriteAt([]byte, int64) (int, error)
	Sync() error
	Close() error
}

type openFileFunc func(string) (File, error)

type FileS struct {
	os.File
}

func (f *FileS) Size() int64 {
	stat, err := f.Stat()
	if err != nil {
		return 0
	}
	return stat.Size()
}

func NewFile(name string) (File, error) {
	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0666)
	fs := FileS{*f}
	return &fs, err
}

func Truncate(name string, size int64) error {
	f, err := os.Create(name)
	if err != nil {
		return err
	}
	f.Close()
	err = os.Truncate(name, size)
	return err
}

type Request struct {
	url    string
	header http.Header
}

type DownloadTaskInfo struct {
	DownloadRange
	Request
	Name        string
	FailedTimes int
	LastWorkerN int
}

type DownloadChunk struct {
	Data  []byte
	Start int64
	Name  string
}

func Downloader(task_info_c <-chan DownloadTaskInfo, finished_c chan<- DownloadChunk, failed_task_info_c chan<- DownloadTaskInfo, wg *sync.WaitGroup, worker_n int) {
	for task_info := range task_info_c {
		length := task_info.Length
		start := task_info.Start
		name := task_info.Name
		log.Printf("Worker[%v] %v %v", name, worker_n, task_info.DownloadRange)
		var downloaded int64 = 0
		task_start_time := GetNowEpochInMilli()
		for try_times := 0; try_times < 3; try_times++ {
			chunk_datas := make(chan []byte)
			go RangeGet(task_info.Request, start, length-(start-task_info.Start), chunk_datas)
			for chunk_data := range chunk_datas {
				downloaded += int64(len(chunk_data))
				log.Printf("Wroker[%v] %v %v k/s %v%%", name, worker_n, downloaded*1000/1024/(GetNowEpochInMilli()-task_start_time), 100*downloaded/length)
				finished_c <- DownloadChunk{Data: chunk_data, Name: name, Start: start}
				start += int64(len(chunk_data))
			}
			if downloaded == length {
				wg.Done()
				break
			}
		}
		if downloaded < length {
			task_info.Start = start
			task_info.Length = length - downloaded
			task_info.LastWorkerN = worker_n
			task_info.FailedTimes++
			failed_task_info_c <- task_info
		}
	}
}

func DownloadChunkWaitGroupAutoCloser(wg *sync.WaitGroup, c chan<- DownloadChunk) {
	wg.Wait()
	close(c)
}

func Receiver(fileDownloadInfoC <-chan FileDownloadInfo, chunks <-chan DownloadChunk, finished chan<- int) {
	defer func() {
		finished <- 0
	}()
	fileDownloadInfoMap := make(map[string]FileDownloadInfo)
	fileFdMap := make(map[string]File)
	for {
		select {
		case info, ok := <-fileDownloadInfoC:
			if !ok {
				if len(fileDownloadInfoMap) == 0 {
					return
				}
				fileDownloadInfoC = nil
				break
			}
			if info.Finished() {
				break
			}
			fileDownloadInfoMap[info.Name] = info
			fd, err := open_file_func(info.Name)
			if err != nil {
				log.Fatalf("can't open %v", info.Name)
			}
			fileFdMap[info.Name] = fd
		case chunk, ok := <-chunks:
			if !ok {
				return
			}
			info, ok := fileDownloadInfoMap[chunk.Name]
			info.Update(chunk.Start, int64(len(chunk.Data)))
			info.Sync()
			if !ok {
				log.Fatalf("can't find chunk.Name in info_map, %v", chunk.Name)
			}
			fd, ok := fileFdMap[chunk.Name]
			if !ok {
				log.Fatalf("can't find chunk.Name in file_fd_map, %v", chunk.Name)
			}
			_, err := fd.WriteAt(chunk.Data, chunk.Start)
			if err != nil {
				log.Fatalf("can't write content to file : %v", chunk.Name)
			}
			err = fd.Sync()
			if err != nil {
				log.Fatalf("can't write content to file : %v", chunk.Name)
			}
			if info.Finished() {
				delete(fileDownloadInfoMap, chunk.Name)
				fd.Close()
				delete(fileFdMap, chunk.Name)
				log.Print(info, info.Finished(), len(fileDownloadInfoMap))
				if len(fileDownloadInfoMap) == 0 {
					return
				}
			}
		}
	}
}

func Run(curl_cmd_strs []string, num_of_workers int) {
	file_download_info_c := make(chan FileDownloadInfo, 1)
	downloader_task_info_cs := make([]chan DownloadTaskInfo, num_of_workers)
	chunk_c := make(chan DownloadChunk, 1)
	url_chan_map := make(map[string]int)
	file_name_reqs_map := make(map[string]*[]Request)
	receiver_finish_channel := make(chan int, 1)
	task_info_wait_group := sync.WaitGroup{}
	failed_task_info_c := make(chan DownloadTaskInfo, 1)

	go Receiver(file_download_info_c, chunk_c, receiver_finish_channel)
	for i := 0; i < num_of_workers; i++ {
		tmp := make(chan DownloadTaskInfo, DownloaderChanBufferSize)
		downloader_task_info_cs[i] = tmp
		go Downloader(tmp, chunk_c, failed_task_info_c, &task_info_wait_group, i)
	}

	file_download_infos := []FileDownloadInfo{}
	// resource_infos := []ResourceInfo{}
	worker_n := -1
	task_infos := []DownloadTaskInfo{}
	for _, curl_cmd_str := range curl_cmd_strs {
		worker_n++
		worker_n = worker_n % num_of_workers
		url := curl_cmd.ParseCmdStr(curl_cmd_str)[1]
		// log.Printf("%v %v", url, worker_n)
		url_chan_map[url] = worker_n
		header := curl_cmd.GetHeadersFromCurlCmd(curl_cmd_str)
		req := Request{url, header}
		// reqs = append(reqs, req)
		// reqs := []Request{req}
		resource_info, err := GetResourceInfo(url, header)
		if err != nil {
			log.Printf("Can't get resource_info of url(%v), err(%v)", url, err)
			continue
		}
		file_name_reqs, ok := file_name_reqs_map[resource_info.filename]
		if !ok {
			tmp := make([]Request, 0, 1)
			file_name_reqs = &tmp
			file_name_reqs_map[resource_info.filename] = file_name_reqs
			*file_name_reqs = append(*file_name_reqs, req)
		} else {
			*file_name_reqs = append(*file_name_reqs, req)
			continue
		}

		log.Printf("Get Resource %v %v", resource_info.filename, resource_info.length)
		file_download_info, err := NewFileDownloadInfo(resource_info.filename, resource_info.length)
		if err != nil {
			log.Printf("Can't create file downloaded info of %v, %v", resource_info.filename, resource_info.length)
			continue
		}
		err = file_download_info.Sync()
		if err != nil {
			log.Printf("Can't sync file downloaded info of %v %v", resource_info.filename, resource_info.length)
			continue
		}
		file_download_info_c <- *file_download_info
		file_download_infos = append(file_download_infos, *file_download_info)
		for _, range_ := range file_download_info.UndownloadedRanges() {
			task := DownloadTaskInfo{range_, Request{}, resource_info.filename, 0, worker_n}
			task_info_wait_group.Add(1)
			task_infos = append(task_infos, task)
		}
	}
	all_task_finish_c := make(chan bool)
	go ConvertWaitGroupToBoolChan(&task_info_wait_group, all_task_finish_c)
	close(file_download_info_c)
	for file_name, reqs := range file_name_reqs_map {
		log.Printf("%v %v", file_name, len(*reqs))
	}
	worker_n = -1
	for _, task_info := range task_infos {
		worker_n++
		reqs := file_name_reqs_map[task_info.Name]
		req := (*reqs)[worker_n%len(*reqs)]
		task_info.Request = req
		// log.Printf("%v %v %v", task_info.Name, task_info.DownloadRange, url_chan_map[task_info.url])
		select {
		case downloader_task_info_cs[url_chan_map[task_info.url]] <- task_info:
		case failed_task_info := <-failed_task_info_c:
			if failed_task_info.FailedTimes < 3 {
				reqs := file_name_reqs_map[failed_task_info.Name]
				last_worker_n := failed_task_info.LastWorkerN
				new_req := (*reqs)[(last_worker_n+1)%len(*reqs)]
				failed_task_info.Request = new_req
				downloader_task_info_cs[url_chan_map[new_req.url]] <- failed_task_info
			} else {
				task_info_wait_group.Done()
			}
		}
	}
ForLoop:
	for {
		select {
		case failed_task_info := <-failed_task_info_c:
			if failed_task_info.FailedTimes < 3 {
				reqs := file_name_reqs_map[failed_task_info.Name]
				last_worker_n := failed_task_info.LastWorkerN
				new_req := (*reqs)[(last_worker_n+1)%len(*reqs)]
				failed_task_info.Request = new_req
				downloader_task_info_cs[url_chan_map[new_req.url]] <- failed_task_info
			} else {
				task_info_wait_group.Done()
			}
		case <-all_task_finish_c:
			for _, chan_ := range downloader_task_info_cs {
				close(chan_)
			}
			close(chunk_c)
			close(failed_task_info_c)
			break ForLoop
		}
	}
}
<commit_msg>add host in log<commit_after>package http_util

import (
	// "crypto/md5"
	// "errors"
	"curl_cmd"
	// "fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"
)

const ChromeUserAgent string = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.56 Safari/537.17"

type ResourceInfo struct {
	length   int64
	filename string
}
type RedirectError struct {
	s string
}

func (re RedirectError) Error() string {
	return re.s
}

func CheckRedirect(*http.Request, []*http.Request) error {
	return RedirectError{"don't follow redirect"}
}

type myUrlEror url.Error

func (e myUrlEror) Error() string {
	return (&e).Error()
}

var open_file_func openFileFunc = NewFile

const (
	BlockSize                 int64 = 1024 * 1024
	NBlocksPerRequest               = 300
	DownloaderChanBufferSize        = 100
	TimeoutOfGetResourceInfo        = 60 * time.Second
	TimeoutOfPerBlockDownload       = 1024 / 42 * time.Second
)

type DownloadRange struct {
	Start  int64
	Length int64
}

type File interface {
	Size() int64
	Name() string
	io.ReadWriteSeeker
	Truncate(size int64) error
	WriteAt([]byte, int64) (int, error)
	Sync() error
	Close() error
}

type openFileFunc func(string) (File, error)

type FileS struct {
	os.File
}

func (f *FileS) Size() int64 {
	stat, err := f.Stat()
	if err != nil {
		return 0
	}
	return stat.Size()
}

func NewFile(name string) (File, error) {
	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0666)
	fs := FileS{*f}
	return &fs, err
}

func Truncate(name string, size int64) error {
	f, err := os.Create(name)
	if err != nil {
		return err
	}
	f.Close()
	err = os.Truncate(name, size)
	return err
}

type Request struct {
	url    string
	header http.Header
}

type DownloadTaskInfo struct {
	DownloadRange
	Request
	Name        string
	FailedTimes int
	LastWorkerN int
}

type DownloadChunk struct {
	Data  []byte
	Start int64
	Name  string
}

func Downloader(task_info_c <-chan DownloadTaskInfo, finished_c chan<- DownloadChunk, failed_task_info_c chan<- DownloadTaskInfo, wg *sync.WaitGroup, worker_n int) {
	for task_info := range task_info_c {
		length := task_info.Length
		start := task_info.Start
		name := task_info.Name
		url_obj, err := url.Parse(task_info.Request.url)
		if err != nil {
			log.Fatal(err)
		}
		host := url_obj.Host
		log.Printf("Worker[%v] %v %v %v", name, host, worker_n, task_info.DownloadRange)
		var downloaded int64 = 0
		task_start_time := GetNowEpochInMilli()
		for try_times := 0; try_times < 3; try_times++ {
			chunk_datas := make(chan []byte)
			go RangeGet(task_info.Request, start, length-(start-task_info.Start), chunk_datas)
			for chunk_data := range chunk_datas {
				downloaded += int64(len(chunk_data))
				log.Printf("Wroker[%v] %v %v %v k/s %v%%", name, host, worker_n, downloaded*1000/1024/(GetNowEpochInMilli()-task_start_time), 100*downloaded/length)
				finished_c <- DownloadChunk{Data: chunk_data, Name: name, Start: start}
				start += int64(len(chunk_data))
			}
			if downloaded == length {
				wg.Done()
				break
			}
		}
		if downloaded < length {
			task_info.Start = start
			task_info.Length = length - downloaded
			task_info.LastWorkerN = worker_n
			task_info.FailedTimes++
			failed_task_info_c <- task_info
		}
	}
}

func DownloadChunkWaitGroupAutoCloser(wg *sync.WaitGroup, c chan<- DownloadChunk) {
	wg.Wait()
	close(c)
}

func Receiver(fileDownloadInfoC <-chan FileDownloadInfo, chunks <-chan DownloadChunk, finished chan<- int) {
	defer func() {
		finished <- 0
	}()
	fileDownloadInfoMap := make(map[string]FileDownloadInfo)
	fileFdMap := make(map[string]File)
	for {
		select {
		case info, ok := <-fileDownloadInfoC:
			if !ok {
				if len(fileDownloadInfoMap) == 0 {
					return
				}
				fileDownloadInfoC = nil
				break
			}
			if info.Finished() {
				break
			}
			fileDownloadInfoMap[info.Name] = info
			fd, err := open_file_func(info.Name)
			if err != nil {
				log.Fatalf("can't open %v", info.Name)
			}
			fileFdMap[info.Name] = fd
		case chunk, ok := <-chunks:
			if !ok {
				return
			}
			info, ok := fileDownloadInfoMap[chunk.Name]
			info.Update(chunk.Start, int64(len(chunk.Data)))
			info.Sync()
			if !ok {
				log.Fatalf("can't find chunk.Name in info_map, %v", chunk.Name)
			}
			fd, ok := fileFdMap[chunk.Name]
			if !ok {
				log.Fatalf("can't find chunk.Name in file_fd_map, %v", chunk.Name)
			}
			_, err := fd.WriteAt(chunk.Data, chunk.Start)
			if err != nil {
				log.Fatalf("can't write content to file : %v", chunk.Name)
			}
			err = fd.Sync()
			if err != nil {
				log.Fatalf("can't write content to file : %v", chunk.Name)
			}
			if info.Finished() {
				delete(fileDownloadInfoMap, chunk.Name)
				fd.Close()
				delete(fileFdMap, chunk.Name)
				log.Print(info, info.Finished(), len(fileDownloadInfoMap))
				if len(fileDownloadInfoMap) == 0 {
					return
				}
			}
		}
	}
}

func Run(curl_cmd_strs []string, num_of_workers int) {
	file_download_info_c := make(chan FileDownloadInfo, 1)
	downloader_task_info_cs := make([]chan DownloadTaskInfo, num_of_workers)
	chunk_c := make(chan DownloadChunk, 1)
	url_chan_map := make(map[string]int)
	file_name_reqs_map := make(map[string]*[]Request)
	receiver_finish_channel := make(chan int, 1)
	task_info_wait_group := sync.WaitGroup{}
	failed_task_info_c := make(chan DownloadTaskInfo, 1)

	go Receiver(file_download_info_c, chunk_c, receiver_finish_channel)
	for i := 0; i < num_of_workers; i++ {
		tmp := make(chan DownloadTaskInfo, DownloaderChanBufferSize)
		downloader_task_info_cs[i] = tmp
		go Downloader(tmp, chunk_c, failed_task_info_c, &task_info_wait_group, i)
	}

	file_download_infos := []FileDownloadInfo{}
	// resource_infos := []ResourceInfo{}
	worker_n := -1
	task_infos := []DownloadTaskInfo{}
	for _, curl_cmd_str := range curl_cmd_strs {
		worker_n++
		worker_n = worker_n % num_of_workers
		url := curl_cmd.ParseCmdStr(curl_cmd_str)[1]
		// log.Printf("%v %v", url, worker_n)
		url_chan_map[url] = worker_n
		header := curl_cmd.GetHeadersFromCurlCmd(curl_cmd_str)
		req := Request{url, header}
		// reqs = append(reqs, req)
		// reqs := []Request{req}
		resource_info, err := GetResourceInfo(url, header)
		if err != nil {
			log.Printf("Can't get resource_info of url(%v), err(%v)", url, err)
			continue
		}
		file_name_reqs, ok := file_name_reqs_map[resource_info.filename]
		if !ok {
			tmp := make([]Request, 0, 1)
			file_name_reqs = &tmp
			file_name_reqs_map[resource_info.filename] = file_name_reqs
			*file_name_reqs = append(*file_name_reqs, req)
		} else {
			*file_name_reqs = append(*file_name_reqs, req)
			continue
		}

		log.Printf("Get Resource %v %v", resource_info.filename, resource_info.length)
		file_download_info, err := NewFileDownloadInfo(resource_info.filename, resource_info.length)
		if err != nil {
			log.Printf("Can't create file downloaded info of %v, %v", resource_info.filename, resource_info.length)
			continue
		}
		err = file_download_info.Sync()
		if err != nil {
			log.Printf("Can't sync file downloaded info of %v %v", resource_info.filename, resource_info.length)
			continue
		}
		file_download_info_c <- *file_download_info
		file_download_infos = append(file_download_infos, *file_download_info)
		for _, range_ := range file_download_info.UndownloadedRanges() {
			task := DownloadTaskInfo{range_, Request{}, resource_info.filename, 0, worker_n}
			task_info_wait_group.Add(1)
			task_infos = append(task_infos, task)
		}
	}
	all_task_finish_c := make(chan bool)
	go ConvertWaitGroupToBoolChan(&task_info_wait_group, all_task_finish_c)
	close(file_download_info_c)
	for file_name, reqs := range file_name_reqs_map {
		log.Printf("%v %v", file_name, len(*reqs))
	}
	worker_n = -1
	for _, task_info := range task_infos {
		worker_n++
		reqs := file_name_reqs_map[task_info.Name]
		req := (*reqs)[worker_n%len(*reqs)]
		task_info.Request = req
		// log.Printf("%v %v %v", task_info.Name, task_info.DownloadRange, url_chan_map[task_info.url])
		select {
		case downloader_task_info_cs[url_chan_map[task_info.url]] <- task_info:
		case failed_task_info := <-failed_task_info_c:
			if failed_task_info.FailedTimes < 3 {
				reqs := file_name_reqs_map[failed_task_info.Name]
				last_worker_n := failed_task_info.LastWorkerN
				new_req := (*reqs)[(last_worker_n+1)%len(*reqs)]
				failed_task_info.Request = new_req
				downloader_task_info_cs[url_chan_map[new_req.url]] <- failed_task_info
			} else {
				task_info_wait_group.Done()
			}
		}
	}
ForLoop:
	for {
		select {
		case failed_task_info := <-failed_task_info_c:
			if failed_task_info.FailedTimes < 3 {
				reqs := file_name_reqs_map[failed_task_info.Name]
				last_worker_n := failed_task_info.LastWorkerN
				new_req := (*reqs)[(last_worker_n+1)%len(*reqs)]
				failed_task_info.Request = new_req
				downloader_task_info_cs[url_chan_map[new_req.url]] <- failed_task_info
			} else {
				task_info_wait_group.Done()
			}
		case <-all_task_finish_c:
			for _, chan_ := range downloader_task_info_cs {
				close(chan_)
			}
			close(chunk_c)
			close(failed_task_info_c)
			break ForLoop
		}
	}
}
<|endoftext|>