<commit_before>package main

import (
	"fmt"
	"unicode/utf8"
)

type Node struct {
	tag string
	attrs map[string]string
	text string

	// dumb appended string while attribute assignment is still being determined
	// this is used if it turns out tokens are NOT part of an attribute assignment
	// and the string (with whitespace if any) gets used as the node's text
	attrString string
}

type Parser struct {

	debug bool

	Scanner Scanner
	output string

	// node heirarchy
	stack []*Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool
	dedentCount int

	// flag indicates we are still checking for attribute assignments
	isAttr bool

	// attrName and attrValue are buffers to determine if TokWord for a tag are
	// potentially for an attribute, or if they are just normal text
	attrName string
	attrValue string
	attrAssigned bool

	node *Node
}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.stack = make([]*Node, 0)
	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.isAttr = false
	p.attrName = ""
	p.attrValue = ""
	p.attrAssigned = false
	p.dedentCount = 0
	p.node = p.newNode()

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			if p.debug {
				fmt.Printf("[debug] token: [%s]\n", TokenString(token))
				fmt.Printf("[debug] text:  [%s]\n", text)
			}
			p.processToken(token, text)
			if p.debug {
				fmt.Printf("[debug] node:  [%s]\n\n", p.node)
			}
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) closeLastNode() {
	if len(p.stack) > 0 {
		oldNode := p.stack[len(p.stack) - 1]
		p.stack = p.stack[0: len(p.stack) - 1]
		if r, l := utf8.DecodeLastRuneInString(p.output); r == ' ' {
			p.output = p.output[0: len(p.output) - l]
		}
		p.output += "</" + oldNode.tag + ">"
	}
}

func (p *Parser) lastNode() *Node {
	return p.stack[len(p.stack) - 1]
}

func (p *Parser) outputNode(node *Node) {
	if node.tag != "" {
		p.output += "<" + node.tag
		if len(node.attrs) > 0 {
			for k, v := range node.attrs {
				p.output += fmt.Sprintf(" %s='%s'", k, v)
			}
		}
		p.output += ">"
		if node.text != "" {
			p.output += node.text
		} else {
			p.output += node.attrString
		}
	}
}
func (p *Parser) pushNode(node *Node) {
	if p.node.tag != "" {
		p.stack = append(p.stack, p.node)
	}
}

func (p *Parser) processToken(tok rune, text string) {
	switch tok {
	case TokWord, TokComma:
		if p.node.tag == "" {
			p.node.tag = text
			// found the tag, now check for attributes
			p.isAttr = true
		} else if p.isAttr {
			if p.attrAssigned {
				if _, found := p.node.attrs[p.attrName]; found {
					p.node.attrs[p.attrName] += " "
					p.node.attrs[p.attrName] += text
				} else {
					p.node.attrs[p.attrName] = text
				}

				// reset to look for a new attribute assignment
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
			} else if p.attrName == "" {
				p.node.attrString += text
				p.attrName = text
			} else {
				p.node.text += p.node.attrString
				p.node.text += text
				p.node.attrString = ""
				p.isAttr = false
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
			}
		} else {
			p.node.text += text
		}
	case TokAssign:
		if p.isAttr {
			if p.attrName != "" && p.attrValue == "" {
				p.attrAssigned = true
				p.node.attrString += text
			}
			if p.attrName == "" || p.attrValue != "" {
				p.attrAssigned = false
				p.isAttr = false
				p.node.text += p.node.attrString
				p.node.attrString = ""
			}
		} else {
			p.node.text += text
		}
	case TokWhitespace:
		if p.isAttr && p.node.attrString != "" {
			p.node.attrString += text

		// skip initial whitespace for text
		} else if p.node.text != "" {
			p.node.text += text
		}
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentCount += 1
	case TokNewLine:
		// up to this point a node has been constructed but not appended to the
		// output
		if p.isDedent {
			// for dedents, pop off the stack and close the node
			for p.dedentCount > 0 {
				p.closeLastNode()
				p.dedentCount -= 1
			}
		}
		if p.isDedent || !p.isIndent {
			// closing one more time because this new node replaces the top node in the
			// stack for dedents or siblings (non indent)
			p.closeLastNode()
		}
		p.outputNode(p.node)
		p.pushNode(p.node)

		// reset
		p.node = p.newNode()
		p.isIndent = false
		p.isDedent = false
		p.isAttr = false
		p.attrName = ""
		p.attrValue = ""
	case TokEOF:
		// since p.node is never on the stack until TokNewLine we push to the stack
		// in this scenario
		p.pushNode(p.node)
		p.outputNode(p.node)
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			p.closeLastNode()
		}
	default:
		if !p.isAttr {
			p.node.text += text
		}
	}
}

func (p *Parser) newNode() *Node {
	n := new(Node)
	n.attrs = make(map[string]string)
	return n
}
<commit_msg>nodent support and attribute bug fixes<commit_after>package main

import (
	"fmt"
	"unicode/utf8"
)

type Node struct {
	tag string
	attrs map[string]string
	text string

	// dumb appended string while attribute assignment is still being determined
	// this is used if it turns out tokens are NOT part of an attribute assignment
	// and the string (with whitespace if any) gets used as the node's text
	attrString string
}

type Parser struct {

	debug bool

	Scanner Scanner
	output string

	// node heirarchy
	stack []*Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool
	dedentCount int

	// flag indicates we are still checking for attribute assignments
	isAttr bool

	// attrName and attrValue are buffers to determine if TokWord for a tag are
	// potentially for an attribute, or if they are just normal text
	attrName string
	attrValue string
	attrAssigned bool

	node *Node
}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.stack = make([]*Node, 0)
	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.isAttr = false
	p.attrName = ""
	p.attrValue = ""
	p.attrAssigned = false
	p.dedentCount = 0
	p.node = p.newNode()

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			if p.debug {
				fmt.Printf("[debug] token: [%s]\n", TokenString(token))
				fmt.Printf("[debug] text:  [%s]\n", text)
			}
			p.processToken(token, text)
			if p.debug {
				fmt.Printf("[debug] node:  [%s]\n\n", p.node)
			}
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) closeLastNode() {
	if len(p.stack) > 0 {
		oldNode := p.stack[len(p.stack) - 1]
		p.stack = p.stack[0: len(p.stack) - 1]
		if r, l := utf8.DecodeLastRuneInString(p.output); r == ' ' {
			p.output = p.output[0: len(p.output) - l]
		}
		p.output += "</" + oldNode.tag + ">"
	}
}

func (p *Parser) lastNode() *Node {
	return p.stack[len(p.stack) - 1]
}

func (p *Parser) outputNode(node *Node) {
	if node.tag != "" {
		p.output += "<" + node.tag
		if len(node.attrs) > 0 {
			for k, v := range node.attrs {
				p.output += fmt.Sprintf(" %s='%s'", k, v)
			}
		}
		p.output += ">"
		if node.text != "" {
			p.output += node.text
		} else {
			p.output += node.attrString
		}
	}
}
func (p *Parser) pushNode(node *Node) {
	if p.node.tag != "" {
		p.stack = append(p.stack, p.node)
	}
}

func (p *Parser) processToken(tok rune, text string) {
	fmt.Printf("isattr %s assigned %s name %s val %s\n", p.isAttr, p.attrAssigned, p.attrName, p.attrValue)
	switch tok {
	case TokWord, TokComma:
		if p.node.tag == "" {
			p.node.tag = text
			// found the tag, now check for attributes
			p.isAttr = true
			p.attrAssigned = false
		} else if p.isAttr {
			if p.attrAssigned {
				if _, found := p.node.attrs[p.attrName]; found {
					p.node.attrs[p.attrName] += " "
					p.node.attrs[p.attrName] += text
				} else {
					p.node.attrs[p.attrName] = text
				}

				// reset to look for a new attribute assignment
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
				p.node.attrString = ""
			} else if p.attrName == "" {
				p.node.attrString += text
				p.attrName = text
			} else {
				p.node.text += p.node.attrString
				p.node.text += text
				p.node.attrString = ""
				p.isAttr = false
				p.attrAssigned = false
				p.attrName = ""
				p.attrValue = ""
			}
		} else {
			p.node.text += text
		}
	case TokString:
		if p.isAttr {
			if p.attrAssigned {
				if _, found := p.node.attrs[p.attrName]; found {
					p.node.attrs[p.attrName] += " "
					p.node.attrs[p.attrName] += text[1:len(text)-1]
				} else {
					p.node.attrs[p.attrName] = text[1:len(text)-1]
				}

				// reset to look for a new attribute assignment
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
				p.node.attrString = ""
			} else if p.attrName != "" {
				// TODO using string in attrName shouldn't be allowed
				p.node.text += p.node.attrString
				p.node.text += text[1:len(text)-1]
				p.node.attrString = ""
				p.isAttr = false
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
			}
		} else {
			p.node.text += text
		}
	case TokAssign:
		if p.isAttr {
			if p.attrName != "" && p.attrValue == "" {
				p.attrAssigned = true
				p.node.attrString += text
			}
			if p.attrName == "" || p.attrValue != "" {
				p.attrAssigned = false
				p.isAttr = false
				p.node.text += p.node.attrString
				p.node.attrString = ""
			}
		} else {
			p.node.text += text
		}
	case TokWhitespace:
		if p.isAttr && p.node.attrString != "" {
			p.node.attrString += text

		// skip initial whitespace for text
		} else if p.node.text != "" {
			p.node.text += text
		}
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentCount += 1
	case TokNodent:
		p.isIndent = false
		p.isDedent = false
	case TokNewLine:
		// up to this point a node has been constructed but not appended to the
		// output
		if p.isDedent {
			// for dedents, pop off the stack and close the node
			for p.dedentCount > 0 {
				p.closeLastNode()
				p.dedentCount -= 1
			}
		}
		if p.isDedent || !p.isIndent {
			// closing one more time because this new node replaces the top node in the
			// stack for dedents or siblings (non indent)
			p.closeLastNode()
		}
		p.outputNode(p.node)
		p.pushNode(p.node)

		// reset
		p.node = p.newNode()
		p.isIndent = false
		p.isDedent = false
		p.isAttr = false
		p.attrName = ""
		p.attrValue = ""
	case TokEOF:
		// since p.node is never on the stack until TokNewLine we push to the stack
		// in this scenario
		p.pushNode(p.node)
		p.outputNode(p.node)
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			p.closeLastNode()
		}
	default:
		if !p.isAttr {
			p.node.text += text
		}
	}
}

func (p *Parser) newNode() *Node {
	n := new(Node)
	n.attrs = make(map[string]string)
	return n
}
<|endoftext|>


<commit_before>package ini

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

const newline = 10

var MissingEqualSignError = errors.New("missing equal sign")
var TooManyEqualSignsError = errors.New("too many equal signs")

var assignmentPattern = regexp.MustCompile("[^\\\\]=")

type lineReader struct {
	io.ByteReader
}

// create a new LineReader struct from any given io.ByteReader
func newLineReader(r io.ByteReader) *lineReader {
	return &lineReader{r}
}

// Read bytes from the given LineReader until a newline occurs. If the reader
// contains no newlines, its whole content is returned. If the reader is empty,
// i.e. contains no bytes at all, the empty string and no error is returned.
func (r *lineReader) ReadLine() (line string, err error) {
	var bytes []byte
	var b byte
	for b != newline {
		b, err = r.ReadByte()
		if err != nil {
			if err == io.EOF {
				break
			}
			return line, err
		}
		bytes = append(bytes, b)
	}
	return string(bytes), nil
}

// A section is a string that start with an open bracket [, ends with an open
// bracket ] and has at least one character between those brackets.
func isSection(line string) bool {
	return (strings.HasPrefix(line, "[") &&
		strings.HasSuffix(line, "]") &&
		len(line) > 2)
}

type Item struct {
	Property string
	Value    string
}

// An assignment is of the form `name=value`. Whitespace before and after the
// equal sign is ignored. Equals signs within the value must be quoted or
// escaped with the backslash.
func parseItem(line string) (item *Item, err error) {
	matches := assignmentPattern.FindAllStringIndex(line, -1)
	if matches == nil {
		return item, MissingEqualSignError
	}
	if len(matches) > 1 {
		return item, TooManyEqualSignsError
	}
	loc := matches[0]
	property := strings.TrimSpace(line[:loc[0]+1])
	value := strings.TrimSpace(line[loc[1]:])
	// TODO: use a seperate function for translating escaped control characters
	// to real control characters
	// replace all \= by =
	value = strings.Replace(value, "\\=", "=", -1)
	// replace all \\t by \t
	value = strings.Replace(value, "\\t", "\t", -1)
	// replace all \\n by \n
	value = strings.Replace(value, "\\n", "\n", -1)
	item = &Item{property, value}
	return
}

// a Config map maps from section names to maps of assignments
type Config map[string]map[string]string

// Get a new empty config. This is equivalent to:
// 	NewConfigFromString("")
func NewConfig() *Config {
	c := make(Config)
	return &c
}

// Create a new *Config from a string. This is a shortcut for:
//	NewConfigFromByteReader(strings.NewReader(s))
func NewConfigFromString(s string) (*Config, error) {
	return NewConfigFromByteReader(strings.NewReader(s))
}

// Create a new *Config from a file. This is a shortcut for:
//	NewConfigFromByteReader(bufio.NewReader(file))
func NewConfigFromFile(file *os.File) (*Config, error) {
	return NewConfigFromByteReader(bufio.NewReader(file))
}

// Create a new *Config from a ByteReader.
func NewConfigFromByteReader(reader io.ByteReader) (*Config, error) {
	return parseINI(newLineReader(reader))
}

// Parse the given *LineReader to a *Config. If the reader is empty, an empty
// *Config and no error will be returned. Errors may occur when one assignment
// does not belong to any section, i.e. if it was written before the first
// section was declared. Other errors are syntax errors: Examples for syntax
// errors are: no equals sign in an assignment, more than one unescaped equal
// sign in an assignment.
func parseINI(reader *lineReader) (*Config, error) {
	conf := make(Config)
	var line string
	var err error
	var section string
	for {
		line, err = reader.ReadLine()
		if err != nil {
			return &conf, err
		}
		if line == "" {
			// stop reading at EOF
			break
		}
		trimmedLine := strings.TrimSpace(line)
		// ignore lines beginning with # or ;
		firstCharacter = trimmedLine[0]
		if firstCharacter == '#' || firstCharacter == ';' {
			continue
		}
		if isSection(trimmedLine) {
			section = strings.Trim(trimmedLine, "[]")
			conf.AddSection(section)
		} else {
			// If the line is not a section, it must be an
			// assignment. Otherwise it's a syntax error
			item, err := parseItem(line)
			if err != nil {
				return &conf, err
			}
			if section != "" {
				conf.Set(section, item.Property, item.Value)
			} else {
				// assignment outside a section.
				// this is a syntax error
				return &conf, AssignmentOutsideSectionError
			}
		}
	}
	return &conf, nil
}

// Return a normalized representation of the config. The order of sections and
// the order of assignments within each section is non-deterministic. Each
// section declaration begins with an open bracket [ and end with a closing
// bracket ] plus a newline \n. An Assignment starts with a property, followed
// by an equal sign which is enclosed in spaces and ends with a value and a
// newline.
func (c *Config) String() string {
	buf := new(bytes.Buffer)
	for _, section := range c.GetSections() {
		buf.WriteString(fmt.Sprintf("[%s]\n", section))
		// error can be ignored because the section surely exists
		items, _ := c.GetItems(section)
		for _, item := range items {
			buf.WriteString(fmt.Sprintf("%s = %s\n", item.Property, item.Value))
		}
	}
	// TODO: this looks inefficient and ugly. find some better way to cut of
	// the last byte of buf
	// remove trailing linebreak to be consistent with empty *Config values
	return strings.TrimSpace(string(buf.Bytes()))
}
<commit_msg>fixed bug with detecting comments<commit_after>package ini

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

const newline = 10

var MissingEqualSignError = errors.New("missing equal sign")
var TooManyEqualSignsError = errors.New("too many equal signs")

var assignmentPattern = regexp.MustCompile("[^\\\\]=")

type lineReader struct {
	io.ByteReader
}

// create a new LineReader struct from any given io.ByteReader
func newLineReader(r io.ByteReader) *lineReader {
	return &lineReader{r}
}

// Read bytes from the given LineReader until a newline occurs. If the reader
// contains no newlines, its whole content is returned. If the reader is empty,
// i.e. contains no bytes at all, the empty string and no error is returned.
func (r *lineReader) ReadLine() (line string, err error) {
	var bytes []byte
	var b byte
	for b != newline {
		b, err = r.ReadByte()
		if err != nil {
			if err == io.EOF {
				break
			}
			return line, err
		}
		bytes = append(bytes, b)
	}
	return string(bytes), nil
}

// A section is a string that start with an open bracket [, ends with an open
// bracket ] and has at least one character between those brackets.
func isSection(line string) bool {
	return (strings.HasPrefix(line, "[") &&
		strings.HasSuffix(line, "]") &&
		len(line) > 2)
}

type Item struct {
	Property string
	Value    string
}

// An assignment is of the form `name=value`. Whitespace before and after the
// equal sign is ignored. Equals signs within the value must be quoted or
// escaped with the backslash.
func parseItem(line string) (item *Item, err error) {
	matches := assignmentPattern.FindAllStringIndex(line, -1)
	if matches == nil {
		return item, MissingEqualSignError
	}
	if len(matches) > 1 {
		return item, TooManyEqualSignsError
	}
	loc := matches[0]
	property := strings.TrimSpace(line[:loc[0]+1])
	value := strings.TrimSpace(line[loc[1]:])
	// TODO: use a seperate function for translating escaped control characters
	// to real control characters
	// replace all \= by =
	value = strings.Replace(value, "\\=", "=", -1)
	// replace all \\t by \t
	value = strings.Replace(value, "\\t", "\t", -1)
	// replace all \\n by \n
	value = strings.Replace(value, "\\n", "\n", -1)
	item = &Item{property, value}
	return
}

// a Config map maps from section names to maps of assignments
type Config map[string]map[string]string

// Get a new empty config. This is equivalent to:
// 	NewConfigFromString("")
func NewConfig() *Config {
	c := make(Config)
	return &c
}

// Create a new *Config from a string. This is a shortcut for:
//	NewConfigFromByteReader(strings.NewReader(s))
func NewConfigFromString(s string) (*Config, error) {
	return NewConfigFromByteReader(strings.NewReader(s))
}

// Create a new *Config from a file. This is a shortcut for:
//	NewConfigFromByteReader(bufio.NewReader(file))
func NewConfigFromFile(file *os.File) (*Config, error) {
	return NewConfigFromByteReader(bufio.NewReader(file))
}

// Create a new *Config from a ByteReader.
func NewConfigFromByteReader(reader io.ByteReader) (*Config, error) {
	return parseINI(newLineReader(reader))
}

// Parse the given *LineReader to a *Config. If the reader is empty, an empty
// *Config and no error will be returned. Errors may occur when one assignment
// does not belong to any section, i.e. if it was written before the first
// section was declared. Other errors are syntax errors: Examples for syntax
// errors are: no equals sign in an assignment, more than one unescaped equal
// sign in an assignment.
func parseINI(reader *lineReader) (*Config, error) {
	conf := make(Config)
	var line string
	var err error
	var section string
	for {
		line, err = reader.ReadLine()
		if err != nil {
			return &conf, err
		}
		if line == "" {
			// stop reading at EOF
			break
		}
		trimmedLine := strings.TrimSpace(line)
		// ignore lines beginning with # or ;
		firstCharacter := trimmedLine[0]
		if firstCharacter == '#' || firstCharacter == ';' {
			continue
		}
		if isSection(trimmedLine) {
			section = strings.Trim(trimmedLine, "[]")
			conf.AddSection(section)
		} else {
			// If the line is not a section, it must be an
			// assignment. Otherwise it's a syntax error
			item, err := parseItem(line)
			if err != nil {
				return &conf, err
			}
			if section != "" {
				conf.Set(section, item.Property, item.Value)
			} else {
				// assignment outside a section.
				// this is a syntax error
				return &conf, AssignmentOutsideSectionError
			}
		}
	}
	return &conf, nil
}

// Return a normalized representation of the config. The order of sections and
// the order of assignments within each section is non-deterministic. Each
// section declaration begins with an open bracket [ and end with a closing
// bracket ] plus a newline \n. An Assignment starts with a property, followed
// by an equal sign which is enclosed in spaces and ends with a value and a
// newline.
func (c *Config) String() string {
	buf := new(bytes.Buffer)
	for _, section := range c.GetSections() {
		buf.WriteString(fmt.Sprintf("[%s]\n", section))
		// error can be ignored because the section surely exists
		items, _ := c.GetItems(section)
		for _, item := range items {
			buf.WriteString(fmt.Sprintf("%s = %s\n", item.Property, item.Value))
		}
	}
	// TODO: this looks inefficient and ugly. find some better way to cut of
	// the last byte of buf
	// remove trailing linebreak to be consistent with empty *Config values
	return strings.TrimSpace(string(buf.Bytes()))
}
<|endoftext|>


<commit_before>package wikimediaparser

import (
  "fmt"
  "log"
  "os"
)

type parser struct {
  items           []item
  start           int
  pos             int
  logger          *log.Logger
  ignoreNextBlock bool
}

func create_parser(items []item) *parser {
  p := &parser{
    items:           items,
    start:           0,
    pos:             0,
    logger:          log.New(os.Stdout, "[Parse]\t", log.LstdFlags),
    ignoreNextBlock: false,
  }
  return p
}

func Parse(items []item) Nodes {
  p := create_parser(items)
  defer p.handleParseError()

  return p.Parse(envAlteration{})
}

func (p *parser) CurrentItem() item {
  if p.pos > len(p.items) {
    outOfBoundsPanic(p, p.start)
  }

  return p.items[p.pos]
}

func (p *parser) eatCurrentItem() item {
  ret := p.CurrentItem()
  p.pos += 1
  return ret
}

func (p *parser) nextBlock() {
  for p.pos < len(p.items) {
    if p.eatCurrentItem().typ == tokenLF {
      if p.eatCurrentItem().typ == tokenLF {
        return
      }
    }
  }
}

func (p *parser) next() {
  p.pos += 1
}

func (p *parser) eat(types ...token) {
  it := p.eatCurrentItem()
  exp := make([]string, 0)

  for _, typ := range types {
    if it.typ == token(typ) {
      return
    }
    exp = append(exp, token(typ).String())
  }
  panic(fmt.Sprintf("Syntax Error at %q\nExpected any of %q, got %q", it.val, exp, it.typ.String()))

}

func (p *parser) backup() {
  p.backupI(1)
}

func (p *parser) backupI(count int) {
  p.pos -= count
}

func (p *parser) ahead(count int) {
  p.pos += count
}

type envAlteration struct {
  exitTypes    []token
  exitSequence []token
}

func (ev *envAlteration) String() string {
  st := ""
  if len(ev.exitTypes) > 0 {
    st += "Closing types: "
  }
  for _, et := range ev.exitTypes {
    st += et.String() + " "
  }

  if len(ev.exitTypes) > 0 {
    st += "\n"
  }

  if len(ev.exitSequence) > 0 {
    st += "Closing Sequence: "
  }
  for _, es := range ev.exitSequence {
    st += es.String()
  }
  return st
}

func (p *parser) Parse(st envAlteration) Nodes {
  ret := make([]Node, 0)

  for p.pos < len(p.items)-1 {
    it := p.CurrentItem()

    // If the exit Sequence match, abort immediately
    if len(st.exitSequence) > 0 {
      matching := 0
      for _, ty := range st.exitSequence {
        if p.CurrentItem().typ == ty {
          matching += 1
          p.next()
        } else {
          break
        }
      }
      if matching == len(st.exitSequence) {
        p.backupI(len(st.exitSequence))
        return ret
      } else {
        p.backupI(matching)
      }
    }

    var n Node = Node{typ: nodeInvalid}
    switch it.typ {
    case itemText:
      n = Node{typ: nodeText, val: it.val}
    case linkStart:
      n = p.ParseLink()
    case templateStart:
      n = p.ParseTemplate()
    case tokenEOF:
      break
    case tokenEq:
      p.eatCurrentItem()
      if p.CurrentItem().typ == tokenEq {
        n = p.parseTitle()
      } else {
        p.backup()
        n = Node{typ: nodeText, val: "="}
      }
    }
    if n.typ != nodeInvalid {
      ret = append(ret, n)
    }

    it = p.CurrentItem()
    for _, typ := range st.exitTypes {
      if it.typ == token(typ) {
        return ret
      }
    }
    p.pos += 1
  }
  return ret
}

func (p *parser) ParseLink() Node {
  ret := Node{typ: nodeLink, namedParams: make(map[string]Nodes), params: make([]Nodes, 0)}

  p.eat(linkStart)
  ret.namedParams["link"] = p.subparse(envAlteration{exitTypes: []token{itemPipe, linkEnd}})
  p.scanSubArgumentsUntil(&ret, linkEnd)

  return ret
}

func (p *parser) ParseTemplate() Node {
  ret := Node{typ: nodeTemplate, namedParams: make(map[string]Nodes), params: make([]Nodes, 0)}

  p.eat(templateStart)
  ret.namedParams["name"] = p.subparse(envAlteration{exitTypes: []token{itemPipe, templateEnd}})
  p.scanSubArgumentsUntil(&ret, templateEnd)

  return ret
}

func (p *parser) parseTitle() Node {
  var ret Node
  exitSequence := make([]token, 0)

  item := p.CurrentItem()
  level := 0

  for item.typ == tokenEq {
    exitSequence = append(exitSequence, tokenEq)
    item = p.eatCurrentItem()
    level += 1
  }
  defer p.handleTitleError(p.pos, level)

  if level < 2 {
    ret = Node{typ: nodeEq}
    return ret
  }
  // put last seen token in front of here
  p.backup()

  ret = Node{typ: nodeTitle, namedParams: make(map[string]Nodes), params: make([]Nodes, 0)}
  ret.namedParams["level"] = Nodes{Node{typ: nodeText, val: fmt.Sprintf("%d", level)}}
  ret.namedParams["title"] = p.subparse(envAlteration{exitSequence: exitSequence})
  p.ahead(level)

  return ret
}

func (p *parser) scanSubArgumentsUntil(node *Node, stop token) {
  cont := true
  for cont {
    elt := p.CurrentItem()
    switch elt.typ {
    case stop:
      return
    case itemPipe:
      p.next()
    case itemText:
      p.next()
      if p.CurrentItem().typ == tokenEq {
        k := elt.val
        p.next()
        node.namedParams[k] = p.subparse(envAlteration{exitTypes: []token{itemPipe, stop}})
      } else {
        p.backup()
        node.params = append(node.params, p.subparse(envAlteration{exitTypes: []token{itemPipe, stop}}))
      }
    default:
      node.params = append(node.params, p.subparse(envAlteration{exitTypes: []token{itemPipe, stop}}))
    }
  }

}

func (p *parser) subparse(st envAlteration) Nodes {
  var res Nodes
  res = p.Parse(st)
  return res
}
<commit_msg>change method visibility<commit_after>package wikimediaparser

import (
  "fmt"
  "log"
  "os"
)

type parser struct {
  items           []item
  start           int
  pos             int
  logger          *log.Logger
  ignoreNextBlock bool
}

func create_parser(items []item) *parser {
  p := &parser{
    items:           items,
    start:           0,
    pos:             0,
    logger:          log.New(os.Stdout, "[Parse]\t", log.LstdFlags),
    ignoreNextBlock: false,
  }
  return p
}

func Parse(items []item) Nodes {
  p := create_parser(items)
  defer p.handleParseError()

  return p.Parse(envAlteration{})
}

func (p *parser) currentItem() item {
  if p.pos > len(p.items) {
    outOfBoundsPanic(p, p.start)
  }

  return p.items[p.pos]
}

func (p *parser) eatCurrentItem() item {
  ret := p.currentItem()
  p.pos += 1
  return ret
}

func (p *parser) nextBlock() {
  for p.pos < len(p.items) {
    if p.eatCurrentItem().typ == tokenLF {
      if p.eatCurrentItem().typ == tokenLF {
        return
      }
    }
  }
}

func (p *parser) next() {
  p.pos += 1
}

func (p *parser) eat(types ...token) {
  it := p.eatCurrentItem()
  exp := make([]string, 0)

  for _, typ := range types {
    if it.typ == token(typ) {
      return
    }
    exp = append(exp, token(typ).String())
  }
  panic(fmt.Sprintf("Syntax Error at %q\nExpected any of %q, got %q", it.val, exp, it.typ.String()))

}

func (p *parser) backup() {
  p.backupI(1)
}

func (p *parser) backupI(count int) {
  p.pos -= count
}

func (p *parser) ahead(count int) {
  p.pos += count
}

type envAlteration struct {
  exitTypes    []token
  exitSequence []token
}

func (ev *envAlteration) String() string {
  st := ""
  if len(ev.exitTypes) > 0 {
    st += "Closing types: "
  }
  for _, et := range ev.exitTypes {
    st += et.String() + " "
  }

  if len(ev.exitTypes) > 0 {
    st += "\n"
  }

  if len(ev.exitSequence) > 0 {
    st += "Closing Sequence: "
  }
  for _, es := range ev.exitSequence {
    st += es.String()
  }
  return st
}

func (p *parser) Parse(st envAlteration) Nodes {
  ret := make([]Node, 0)

  for p.pos < len(p.items)-1 {
    it := p.currentItem()

    // If the exit Sequence match, abort immediately
    if len(st.exitSequence) > 0 {
      matching := 0
      for _, ty := range st.exitSequence {
        if p.currentItem().typ == ty {
          matching += 1
          p.next()
        } else {
          break
        }
      }
      if matching == len(st.exitSequence) {
        p.backupI(len(st.exitSequence))
        return ret
      } else {
        p.backupI(matching)
      }
    }

    var n Node = Node{typ: nodeInvalid}
    switch it.typ {
    case itemText:
      n = Node{typ: nodeText, val: it.val}
    case linkStart:
      n = p.ParseLink()
    case templateStart:
      n = p.ParseTemplate()
    case tokenEOF:
      break
    case tokenEq:
      p.eatCurrentItem()
      if p.currentItem().typ == tokenEq {
        n = p.parseTitle()
      } else {
        p.backup()
        n = Node{typ: nodeText, val: "="}
      }
    }
    if n.typ != nodeInvalid {
      ret = append(ret, n)
    }

    it = p.currentItem()
    for _, typ := range st.exitTypes {
      if it.typ == token(typ) {
        return ret
      }
    }
    p.pos += 1
  }
  return ret
}

func (p *parser) ParseLink() Node {
  ret := Node{typ: nodeLink, namedParams: make(map[string]Nodes), params: make([]Nodes, 0)}

  p.eat(linkStart)
  ret.namedParams["link"] = p.subparse(envAlteration{exitTypes: []token{itemPipe, linkEnd}})
  p.scanSubArgumentsUntil(&ret, linkEnd)

  return ret
}

func (p *parser) ParseTemplate() Node {
  ret := Node{typ: nodeTemplate, namedParams: make(map[string]Nodes), params: make([]Nodes, 0)}

  p.eat(templateStart)
  ret.namedParams["name"] = p.subparse(envAlteration{exitTypes: []token{itemPipe, templateEnd}})
  p.scanSubArgumentsUntil(&ret, templateEnd)

  return ret
}

func (p *parser) parseTitle() Node {
  var ret Node
  exitSequence := make([]token, 0)

  item := p.currentItem()
  level := 0

  for item.typ == tokenEq {
    exitSequence = append(exitSequence, tokenEq)
    item = p.eatCurrentItem()
    level += 1
  }
  defer p.handleTitleError(p.pos, level)

  if level < 2 {
    ret = Node{typ: nodeEq}
    return ret
  }
  // put last seen token in front of here
  p.backup()

  ret = Node{typ: nodeTitle, namedParams: make(map[string]Nodes), params: make([]Nodes, 0)}
  ret.namedParams["level"] = Nodes{Node{typ: nodeText, val: fmt.Sprintf("%d", level)}}
  ret.namedParams["title"] = p.subparse(envAlteration{exitSequence: exitSequence})
  p.ahead(level)

  return ret
}

func (p *parser) scanSubArgumentsUntil(node *Node, stop token) {
  cont := true
  for cont {
    elt := p.currentItem()
    switch elt.typ {
    case stop:
      return
    case itemPipe:
      p.next()
    case itemText:
      p.next()
      if p.currentItem().typ == tokenEq {
        k := elt.val
        p.next()
        node.namedParams[k] = p.subparse(envAlteration{exitTypes: []token{itemPipe, stop}})
      } else {
        p.backup()
        node.params = append(node.params, p.subparse(envAlteration{exitTypes: []token{itemPipe, stop}}))
      }
    default:
      node.params = append(node.params, p.subparse(envAlteration{exitTypes: []token{itemPipe, stop}}))
    }
  }

}

func (p *parser) subparse(st envAlteration) Nodes {
  var res Nodes
  res = p.Parse(st)
  return res
}
<|endoftext|>


<commit_before>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText, itemStrong, itemItalic, itemStrike, itemCode,
			itemLink, itemAutoLink, itemGfmLink, itemImage:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText()
			n = tmp
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			// 0 for the depth
			n = t.parseList(0)
		case itemTable, itemLpTable:
			n = t.parseTable()
		default:
			fmt.Println("[Skip]:", t.next())
		}
		t.append(n)
	}
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseParagraph scan until itemBr occur.
func (t *Tree) parseText() []Node {
	var nodes []Node
Loop:
	for {
		var node Node
		switch token := t.next(); token.typ {
		case eof, itemError, itemHeading, itemList, itemIndent:
			t.backup()
			break Loop
		case itemNewLine:
			// Two or more lines continuosly, or block below
			if typ := t.peek().typ; typ == itemNewLine || isBlock(typ) {
				t.backup2(token)
				break Loop
			}
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemText:
			node = t.newText(token.pos, token.val)
		case itemStrong, itemItalic, itemStrike, itemCode:
			match := span[token.typ].FindStringSubmatch(token.val)
			text := match[len(match)-1]
			if text == "" {
				text = match[1]
			}
			node = t.newEmphasis(token.pos, token.typ, text)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		default:
			fmt.Println("Matching not found for this shit:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// itemLHeading will always be in level 1.
		node = t.newHeading(token.pos, 1, match[1])
	}
	return
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		lang, text = match[1], match[2]
		if text == "" {
			text = match[4]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

// parse list
func (t *Tree) parseList(depth int) *ListNode {
	token := t.next()
	list := t.newList(token.pos, depth, isDigit(token.val))
	item := new(ListItemNode)
Loop:
	for {
		switch token = t.next(); token.typ {
		case eof, itemError:
			break Loop
		// It's actually a listItem
		case itemList:
			// List, but not the same type
			if list.Ordered != isDigit(token.val) || depth > 0 {
				t.backup()
				break Loop
			}
			item = t.parseListItem(token.pos, list)
		case itemNewLine:
			if t.peek().typ == itemNewLine {
				break Loop
			}
			fallthrough
		case itemIndent:
			if depth == len(token.val) {
				item = t.parseListItem(token.pos, list)
			}
			t.backup()
			break Loop
		default:
			t.backup()
			item = t.parseListItem(token.pos, list)
		}
		list.append(item)
	}
	return list
}

// parse listItem
func (t *Tree) parseListItem(pos Pos, list *ListNode) *ListItemNode {
	item := t.newListItem(pos, list)
	var n Node
Loop:
	for {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemList:
			t.backup()
			break Loop
		case itemNewLine:
			switch typ := t.peek().typ; typ {
			case itemNewLine, eof, itemError, itemList, itemIndent:
				continue
			default:
				n = t.newLine(token.pos)
			}
		case itemIndent:
			if t.peek().typ == itemList {
				depth := len(token.val)
				// If it's in the same depth - sibling
				// or if it's less-than - exit
				if depth <= item.List.Depth {
					t.backup2(token)
					break Loop
				}
				n = t.parseList(depth)
			} else {
				n = t.newText(token.pos, token.val)
			}
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		default:
			// DRY
			for _, n := range t.parseText() {
				if n.Type() != NodeNewLine {
					item.append(n)
				}
			}
			continue
		}
		item.append(n)
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			if len(cell) > 0 {
				row = append(row, cell)
				rows.Data = append(rows.Data, row)
			}
			break Loop
		case itemPipe, itemNewLine:
			fmt.Println("In pipe, cell situation:", cell, i)
			if len(cell) == 0 {
				continue
			}
			if i == 0 {
				rows.Header = append(rows.Header, cell)
			} else if i == 1 {
				align := cell[0].val
				if len(cell) > 1 {
					for i := 1; i < len(cell); i++ {
						align += cell[i].val
					}
				}
				// Trim spaces
				rows.Align = append(rows.Align, parseAlign(align))
			} else {
				row = append(row, cell)
			}
			if token.typ == itemNewLine {
				i++
				if i >= 2 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Tranform to nodes
	//rowLen := len(rows.Align)
	fmt.Println(rows)
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) (nodes []Node) {
	for i, item := range items {
		// Cell contain nodes
		cell := t.newCell(item[0].pos, kind, align[i])
		for _, token := range item {
			var node Node
			switch token.typ {
			case itemText:
				node = t.newText(token.pos, token.val)
			}
			cell.append(node)
		}
		nodes = append(nodes, cell)
	}
	return
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.Trim(s, " ")
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}
<commit_msg>wip table parsing<commit_after>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText, itemStrong, itemItalic, itemStrike, itemCode,
			itemLink, itemAutoLink, itemGfmLink, itemImage:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText()
			n = tmp
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			// 0 for the depth
			n = t.parseList(0)
		case itemTable, itemLpTable:
			n = t.parseTable()
		default:
			fmt.Println("[Skip]:", t.next())
		}
		t.append(n)
	}
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseParagraph scan until itemBr occur.
func (t *Tree) parseText() []Node {
	var nodes []Node
Loop:
	for {
		var node Node
		switch token := t.next(); token.typ {
		case eof, itemError, itemHeading, itemList, itemIndent:
			t.backup()
			break Loop
		case itemNewLine:
			// Two or more lines continuosly, or block below
			if typ := t.peek().typ; typ == itemNewLine || isBlock(typ) {
				t.backup2(token)
				break Loop
			}
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemText:
			node = t.newText(token.pos, token.val)
		case itemStrong, itemItalic, itemStrike, itemCode:
			match := span[token.typ].FindStringSubmatch(token.val)
			text := match[len(match)-1]
			if text == "" {
				text = match[1]
			}
			node = t.newEmphasis(token.pos, token.typ, text)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		default:
			fmt.Println("Matching not found for this shit:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// itemLHeading will always be in level 1.
		node = t.newHeading(token.pos, 1, match[1])
	}
	return
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		lang, text = match[1], match[2]
		if text == "" {
			text = match[4]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

// parse list
func (t *Tree) parseList(depth int) *ListNode {
	token := t.next()
	list := t.newList(token.pos, depth, isDigit(token.val))
	item := new(ListItemNode)
Loop:
	for {
		switch token = t.next(); token.typ {
		case eof, itemError:
			break Loop
		// It's actually a listItem
		case itemList:
			// List, but not the same type
			if list.Ordered != isDigit(token.val) || depth > 0 {
				t.backup()
				break Loop
			}
			item = t.parseListItem(token.pos, list)
		case itemNewLine:
			if t.peek().typ == itemNewLine {
				break Loop
			}
			fallthrough
		case itemIndent:
			if depth == len(token.val) {
				item = t.parseListItem(token.pos, list)
			}
			t.backup()
			break Loop
		default:
			t.backup()
			item = t.parseListItem(token.pos, list)
		}
		list.append(item)
	}
	return list
}

// parse listItem
func (t *Tree) parseListItem(pos Pos, list *ListNode) *ListItemNode {
	item := t.newListItem(pos, list)
	var n Node
Loop:
	for {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemList:
			t.backup()
			break Loop
		case itemNewLine:
			switch typ := t.peek().typ; typ {
			case itemNewLine, eof, itemError, itemList, itemIndent:
				continue
			default:
				n = t.newLine(token.pos)
			}
		case itemIndent:
			if t.peek().typ == itemList {
				depth := len(token.val)
				// If it's in the same depth - sibling
				// or if it's less-than - exit
				if depth <= item.List.Depth {
					t.backup2(token)
					break Loop
				}
				n = t.parseList(depth)
			} else {
				n = t.newText(token.pos, token.val)
			}
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		default:
			// DRY
			for _, n := range t.parseText() {
				if n.Type() != NodeNewLine {
					item.append(n)
				}
			}
			continue
		}
		item.append(n)
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			if len(cell) > 0 {
				row = append(row, cell)
			}
			if len(row) > 0 {
				rows.Data = append(rows.Data, row)
			}
			break Loop
		case itemPipe, itemNewLine:
			if len(cell) == 0 {
				continue
			}
			if i == 0 {
				rows.Header = append(rows.Header, cell)
			} else if i == 1 {
				align := cell[0].val
				if len(cell) > 1 {
					for i := 1; i < len(cell); i++ {
						align += cell[i].val
					}
				}
				// Trim spaces
				rows.Align = append(rows.Align, parseAlign(align))
			} else {
				row = append(row, cell)
			}
			if token.typ == itemNewLine {
				i++
				if i >= 2 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Tranform to nodes
	// Add an average mechanisem, that ignore empty(or " ") in end of cell
	//	rowLen := len(rows.Align)
	// Table head
	table.append(t.parseCells(Header, rows.Header, rows.Align))
	// Table body
	for _, row := range rows.Data {
		table.append(t.parseCells(Data, row, rows.Align))
	}
	fmt.Println(table)
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) *RowNode {
	// Add real position
	row := t.newRow(1)
	for i, item := range items {
		// Cell contain nodes
		cell := t.newCell(item[0].pos, kind, align[i])
		for _, token := range item {
			var node Node
			switch token.typ {
			case itemText:
				node = t.newText(token.pos, token.val)
			}
			cell.append(node)
		}
		row.Cells = append(row.Cells, cell)
	}
	return row
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.Trim(s, " ")
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// go test *.go -bench=".*"

package gaes_test

import (
	"testing"

	"github.com/gogf/gf/g/crypto/gaes"
	"github.com/gogf/gf/g/test/gtest"
)

var (
	content = []byte("pibigstar")
	// iv 长度必须等于blockSize，只能为16
	iv         = []byte("Hello My GoFrame")
	key_16     = []byte("1234567891234567")
	key_17     = []byte("12345678912345670")
	key_24     = []byte("123456789123456789123456")
	key_32     = []byte("12345678912345678912345678912345")
	keys       = []byte("12345678912345678912345678912346")
	key_err    = []byte("1234")
	key_32_err = []byte("1234567891234567891234567891234 ")
)

func TestEncrypt(t *testing.T) {
	gtest.Case(t, func() {
		_, err := gaes.Encrypt(content, key_16)
		gtest.Assert(err, nil)
		_, err = gaes.Encrypt(content, key_24)
		gtest.Assert(err, nil)
		_, err = gaes.Encrypt(content, key_32)
		gtest.Assert(err, nil)
		_, err = gaes.Encrypt(content, key_16, iv)
		gtest.Assert(err, nil)
	})
}

func TestEncryptErr(t *testing.T) {
	gtest.Case(t, func() {
		// encrypt key error
		_, err := gaes.Encrypt(content, key_err)
		gtest.AssertNE(err, nil)
	})
}

func TestDecryptErr(t *testing.T) {
	gtest.Case(t, func() {
		// decrypt key error
		encrypt, err := gaes.Encrypt(content, key_16)
		_, err = gaes.Decrypt(encrypt, key_err)
		gtest.AssertNE(err, nil)

		// decrypt content too short error
		_, err = gaes.Decrypt([]byte("test"), key_16)
		gtest.AssertNE(err, nil)

		// decrypt content size error
		_, err = gaes.Decrypt(key_17, key_16)
		gtest.AssertNE(err, nil)
	})
}

func TestPKCS5UnPaddingErr(t *testing.T) {
	gtest.Case(t, func() {
		// PKCS5UnPadding blockSize zero
		_, err := gaes.PKCS5UnPadding(content, 0)
		gtest.AssertNE(err, nil)

		// PKCS5UnPadding src len zero
		_, err = gaes.PKCS5UnPadding([]byte(""), 16)
		gtest.AssertNE(err, nil)

		// PKCS5UnPadding src len > blockSize
		_, err = gaes.PKCS5UnPadding(key_17, 16)
		gtest.AssertNE(err, nil)

		// PKCS5UnPadding src len > blockSize
		_, err = gaes.PKCS5UnPadding(key_32_err, 32)
		gtest.AssertNE(err, nil)
	})
}

func TestDecrypt(t *testing.T) {
	gtest.Case(t, func() {
		encrypt, err := gaes.Encrypt(content, key_16)
		decrypt, err := gaes.Decrypt(encrypt, key_16)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_24)
		decrypt, err = gaes.Decrypt(encrypt, key_24)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_32)
		decrypt, err = gaes.Decrypt(encrypt, key_32)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_32, iv)
		decrypt, err = gaes.Decrypt(encrypt, key_32, iv)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_32, iv)
		decrypt, err = gaes.Decrypt(encrypt, keys, iv)
		gtest.Assert(err, "invalid padding")
	})
}
<commit_msg>add gaes test3: add encrypt assert content<commit_after>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// go test *.go -bench=".*"

package gaes_test

import (
	"github.com/gogf/gf/g/encoding/gbase64"
	"testing"

	"github.com/gogf/gf/g/crypto/gaes"
	"github.com/gogf/gf/g/test/gtest"
)

var (
	content          = []byte("pibigstar")
	content_16, _    = gbase64.Decode("v1jqsGHId/H8onlVHR8Vaw==")
	content_24, _    = gbase64.Decode("0TXOaj5KMoLhNWmJ3lxY1A==")
	content_32, _    = gbase64.Decode("qM/Waw1kkWhrwzek24rCSA==")
	content_16_iv, _ = gbase64.Decode("DqQUXiHgW/XFb6Qs98+hrA==")
	// iv 长度必须等于blockSize，只能为16
	iv         = []byte("Hello My GoFrame")
	key_16     = []byte("1234567891234567")
	key_17     = []byte("12345678912345670")
	key_24     = []byte("123456789123456789123456")
	key_32     = []byte("12345678912345678912345678912345")
	keys       = []byte("12345678912345678912345678912346")
	key_err    = []byte("1234")
	key_32_err = []byte("1234567891234567891234567891234 ")
)

func TestEncrypt(t *testing.T) {
	gtest.Case(t, func() {
		data, err := gaes.Encrypt(content, key_16)
		gtest.Assert(err, nil)
		gtest.Assert(string(data), content_16)
		data, err = gaes.Encrypt(content, key_24)
		gtest.Assert(err, nil)
		gtest.Assert(string(data), content_24)
		data, err = gaes.Encrypt(content, key_32)
		gtest.Assert(err, nil)
		gtest.Assert(string(data), content_32)
		data, err = gaes.Encrypt(content, key_16, iv)
		gtest.Assert(err, nil)
		gtest.Assert(string(data), content_16_iv)
	})
}

func TestEncryptErr(t *testing.T) {
	gtest.Case(t, func() {
		// encrypt key error
		_, err := gaes.Encrypt(content, key_err)
		gtest.AssertNE(err, nil)
	})
}

func TestDecryptErr(t *testing.T) {
	gtest.Case(t, func() {
		// decrypt key error
		encrypt, err := gaes.Encrypt(content, key_16)
		_, err = gaes.Decrypt(encrypt, key_err)
		gtest.AssertNE(err, nil)

		// decrypt content too short error
		_, err = gaes.Decrypt([]byte("test"), key_16)
		gtest.AssertNE(err, nil)

		// decrypt content size error
		_, err = gaes.Decrypt(key_17, key_16)
		gtest.AssertNE(err, nil)
	})
}

func TestPKCS5UnPaddingErr(t *testing.T) {
	gtest.Case(t, func() {
		// PKCS5UnPadding blockSize zero
		_, err := gaes.PKCS5UnPadding(content, 0)
		gtest.AssertNE(err, nil)

		// PKCS5UnPadding src len zero
		_, err = gaes.PKCS5UnPadding([]byte(""), 16)
		gtest.AssertNE(err, nil)

		// PKCS5UnPadding src len > blockSize
		_, err = gaes.PKCS5UnPadding(key_17, 16)
		gtest.AssertNE(err, nil)

		// PKCS5UnPadding src len > blockSize
		_, err = gaes.PKCS5UnPadding(key_32_err, 32)
		gtest.AssertNE(err, nil)
	})
}

func TestDecrypt(t *testing.T) {
	gtest.Case(t, func() {
		encrypt, err := gaes.Encrypt(content, key_16)
		decrypt, err := gaes.Decrypt(encrypt, key_16)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_24)
		decrypt, err = gaes.Decrypt(encrypt, key_24)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_32)
		decrypt, err = gaes.Decrypt(encrypt, key_32)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_32, iv)
		decrypt, err = gaes.Decrypt(encrypt, key_32, iv)
		gtest.Assert(err, nil)
		gtest.Assert(string(decrypt), string(content))

		encrypt, err = gaes.Encrypt(content, key_32, iv)
		decrypt, err = gaes.Decrypt(encrypt, keys, iv)
		gtest.Assert(err, "invalid padding")
	})
}
<|endoftext|>


<commit_before>package fluxd

import (
	"encoding/base64"
	"fmt"
	"github.com/A2B-Bikeshare/go-flux/msg"
	"io"
	"net/http"
	"strconv"
	"sync"
)

// ElasticsearchDB conforms to the
// DB interface.
type ElasticsearchDB struct {
	Schema *msg.Schema
	Addr   string
	Index  string
	Dtype  string
	fqaddr string
	once   *sync.Once
}

// Address returns the endpoint that this db POSTs to.
func (e *ElasticsearchDB) Address() string {
	e.once.Do(func() {
		e.fqaddr = fmt.Sprintf("%s/%s/%s", e.Addr, e.Index, e.Dtype)
	})
	return e.fqaddr
}

// Translate uses e.Schema to write json into 'w'
func (e ElasticsearchDB) Translate(p []byte, w msg.Writer) error {
	var nr int //totoal number of bytes read
	var n int  //each number of bytes read
	var err error
	empty := []byte{}
	w.WriteByte(lcurly)

	for i, o := range *(e.Schema) {
		//write comma
		if i != 0 {
			w.WriteByte(comma)
		}
		w.Write(strconv.AppendQuote(empty, o.Name)) // write value name
		w.WriteByte(colon)
		// read value; write value
		switch o.T {
		case msg.String:
			var s string
			s, n, err = msg.ReadStringZeroCopy(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendQuote(empty, s))
			nr += n
			continue

		case msg.Int:
			var i int64
			i, n, err = msg.ReadIntBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendInt(empty, i, 10))
			nr += n
			continue

		case msg.Uint:
			var u uint64
			u, n, err = msg.ReadUintBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendUint(empty, u, 10))
			nr += n
			continue

		case msg.Bool:
			var b bool
			b, n, err = msg.ReadBoolBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendBool(empty, b))
			nr += n
			continue

		case msg.Float:
			var f float64
			f, n, err = msg.ReadFloatBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendFloat(empty, f, 'f', -1, 64))
			nr += n
			continue

		case msg.Bin:
			// Elasticsearch specifies base-64 encoding for binary data
			var dat []byte
			dat, n, err = msg.ReadBinZeroCopy(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendQuote(empty, base64.StdEncoding.EncodeToString(dat)))
			nr += n
			continue
		case msg.Ext:
			var dat []byte
			var etype int8
			// Ext is the only nested object
			dat, etype, n, err = msg.ReadExtZeroCopy(p[nr:])
			if err != nil {
				return err
			}
			w.WriteByte(lcurly)
			w.Write(strconv.AppendQuote(empty, "extension_type:"))
			w.Write(strconv.AppendInt(empty, int64(etype), 10))
			w.WriteByte(comma)
			w.Write(strconv.AppendQuote(empty, "data:"))
			w.Write(strconv.AppendQuote(empty, base64.StdEncoding.EncodeToString(dat)))
			w.WriteByte(rcurly)
			nr += n
			continue

		default:
			return msg.ErrTypeNotSupported

		}

	}

	w.WriteByte(rcurly)
	return err
}

// Req returns the proper POST request to elasticsearch
func (e ElasticsearchDB) Req(r io.Reader) (hr *http.Request, err error) {
	hr, err = http.NewRequest("POST", e.Address(), r)
	return
}

// Validate returns an error if res.StatusCode is not 200 or 201
func (e ElasticsearchDB) Validate(res *http.Response) error {
	if res.StatusCode != 200 && res.StatusCode != 201 {
		return fmt.Errorf("[ERR] Elasticsearch (%s/%s/%s): status code %d", e.Addr, e.Index, e.Dtype, res.StatusCode)
	}
	return nil
}
<commit_msg>ESClient translate unit test.<commit_after>package fluxd

import (
	"encoding/base64"
	"fmt"
	"github.com/A2B-Bikeshare/go-flux/msg"
	"io"
	"net/http"
	"strconv"
	"sync"
)

// ElasticsearchDB conforms to the
// DB interface.
type ElasticsearchDB struct {
	Schema msg.Schema
	Addr   string
	Index  string
	Dtype  string
	fqaddr string
	once   *sync.Once
}

// Address returns the endpoint that this db POSTs to.
func (e ElasticsearchDB) Address() string {
	e.once.Do(func() {
		e.fqaddr = fmt.Sprintf("%s/%s/%s", e.Addr, e.Index, e.Dtype)
	})
	return e.fqaddr
}

// Translate uses e.Schema to write json into 'w'
func (e ElasticsearchDB) Translate(p []byte, w msg.Writer) error {
	var nr int //totoal number of bytes read
	var n int  //each number of bytes read
	var err error
	empty := []byte{}
	w.WriteByte(lcurly)

	for i, o := range e.Schema {
		//write comma
		if i != 0 {
			w.WriteByte(comma)
		}
		w.Write(strconv.AppendQuote(empty, o.Name)) // write value name
		w.WriteByte(colon)
		// read value; write value
		switch o.T {
		case msg.String:
			var s string
			s, n, err = msg.ReadStringZeroCopy(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendQuote(empty, s))
			nr += n
			continue

		case msg.Int:
			var i int64
			i, n, err = msg.ReadIntBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendInt(empty, i, 10))
			nr += n
			continue

		case msg.Uint:
			var u uint64
			u, n, err = msg.ReadUintBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendUint(empty, u, 10))
			nr += n
			continue

		case msg.Bool:
			var b bool
			b, n, err = msg.ReadBoolBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendBool(empty, b))
			nr += n
			continue

		case msg.Float:
			var f float64
			f, n, err = msg.ReadFloatBytes(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendFloat(empty, f, 'f', -1, 64))
			nr += n
			continue

		case msg.Bin:
			// Elasticsearch specifies base-64 encoding for binary data
			var dat []byte
			dat, n, err = msg.ReadBinZeroCopy(p[nr:])
			if err != nil {
				return err
			}
			w.Write(strconv.AppendQuote(empty, base64.StdEncoding.EncodeToString(dat)))
			nr += n
			continue
		case msg.Ext:
			var dat []byte
			var etype int8
			// Ext is the only nested object
			dat, etype, n, err = msg.ReadExtZeroCopy(p[nr:])
			if err != nil {
				return err
			}
			w.WriteByte(lcurly)
			w.Write(strconv.AppendQuote(empty, "extension_type:"))
			w.Write(strconv.AppendInt(empty, int64(etype), 10))
			w.WriteByte(comma)
			w.Write(strconv.AppendQuote(empty, "data:"))
			w.Write(strconv.AppendQuote(empty, base64.StdEncoding.EncodeToString(dat)))
			w.WriteByte(rcurly)
			nr += n
			continue

		default:
			return msg.ErrTypeNotSupported

		}

	}

	w.WriteByte(rcurly)
	return err
}

// Req returns the proper POST request to elasticsearch
func (e ElasticsearchDB) Req(r io.Reader) (hr *http.Request, err error) {
	hr, err = http.NewRequest("POST", e.Address(), r)
	return
}

// Validate returns an error if res.StatusCode is not 200 or 201
func (e ElasticsearchDB) Validate(res *http.Response) error {
	if res.StatusCode != 200 && res.StatusCode != 201 {
		return fmt.Errorf("[ERR] Elasticsearch (%s/%s/%s): status code %d", e.Addr, e.Index, e.Dtype, res.StatusCode)
	}
	return nil
}
<|endoftext|>


<commit_before>package gateway

import (
	"fmt"
	"net/http"
	"sync/atomic"
	"testing"

	"github.com/TykTechnologies/tyk/config"

	"github.com/stretchr/testify/assert"

	"github.com/TykTechnologies/tyk/test"

	"github.com/TykTechnologies/tyk/trace"
)

func TestOpenTracing(t *testing.T) {
	ts := StartTest()
	defer ts.Close()
	trace.SetupTracing("test", nil)
	defer trace.Close()

	t.Run("ensure the manager is enabled", func(ts *testing.T) {
		if !trace.IsEnabled() {
			ts.Error("expected tracing manager should be enabled")
		}
	})

	t.Run("ensure services are initialized", func(ts *testing.T) {
		var s atomic.Value
		trace.SetInit(func(name string, service string, opts map[string]interface{}, logger trace.Logger) (trace.Tracer, error) {
			s.Store(service)
			return trace.NoopTracer{}, nil
		})
		name := "trace"
		BuildAndLoadAPI(
			func(spec *APISpec) {
				spec.Name = name
				spec.UseOauth2 = true
			},
		)
		var n string
		if v := s.Load(); v != nil {
			n = v.(string)
		}
		if name != n {
			ts.Errorf("expected %s got %s", name, n)
		}
	})
}

func TestInternalAPIUsage(t *testing.T) {
	g := StartTest()
	defer g.Close()

	internal := BuildAPI(func(spec *APISpec) {
		spec.Name = "internal"
		spec.APIID = "test1"
		spec.Proxy.ListenPath = "/"
	})[0]

	normal := BuildAPI(func(spec *APISpec) {
		spec.Name = "normal"
		spec.APIID = "test2"
		spec.Proxy.TargetURL = fmt.Sprintf("tyk://%s", internal.Name)
		spec.Proxy.ListenPath = "/normal-api"
	})[0]

	LoadAPI(internal, normal)

	t.Run("with name", func(t *testing.T) {
		_, _ = g.Run(t, []test.TestCase{
			{Path: "/normal-api", Code: http.StatusOK},
		}...)
	})

	t.Run("with api id", func(t *testing.T) {
		normal.Proxy.TargetURL = fmt.Sprintf("tyk://%s", internal.APIID)

		LoadAPI(internal, normal)

		_, _ = g.Run(t, []test.TestCase{
			{Path: "/normal-api", Code: http.StatusOK},
		}...)
	})
}

func TestFuzzyFindAPI(t *testing.T) {
	BuildAndLoadAPI(func(spec *APISpec) {
		spec.Name = "IgnoreCase"
		spec.APIID = "123456"
		spec.Proxy.ListenPath = "/"
	})

	spec := fuzzyFindAPI("ignoreCase")
	assert.Equal(t, "123456", spec.APIID)
}

func TestGraphQLPlayground(t *testing.T) {
	g := StartTest()
	defer g.Close()

	defer ResetTestConfig()

	const apiName = "graphql-api"

	api := BuildAPI(func(spec *APISpec) {
		spec.APIID = "APIID"
		spec.Proxy.ListenPath = fmt.Sprintf("/%s/", apiName)
		spec.GraphQL.Enabled = true
		spec.GraphQL.GraphQLPlayground.Enabled = true
	})[0]

	for _, env := range []string{"on-premise", "cloud"} {
		if env == "cloud" {
			api.Proxy.ListenPath = fmt.Sprintf("/%s/", api.APIID)
			api.Slug = apiName
			globalConf := config.Global()
			globalConf.Cloud = true
			config.SetGlobal(globalConf)
		}

		t.Run(env, func(t *testing.T) {
			t.Run("path is empty", func(t *testing.T) {
				api.GraphQL.GraphQLPlayground.Path = ""
				LoadAPI(api)
				_, _ = g.Run(t, []test.TestCase{
					{Path: api.Proxy.ListenPath, BodyMatch: `<link rel="stylesheet" href="playground.css" />`},
					{Path: api.Proxy.ListenPath, BodyMatch: `endpoint: "\\/` + apiName + `\\/"`},
					{Path: api.Proxy.ListenPath + "playground.css", BodyMatch: "body{margin:0;padding:0;font-family:.*"},
				}...)
			})

			t.Run("path is /", func(t *testing.T) {
				api.GraphQL.GraphQLPlayground.Path = "/"
				LoadAPI(api)
				_, _ = g.Run(t, []test.TestCase{
					{Path: api.Proxy.ListenPath, BodyMatch: `<link rel="stylesheet" href="playground.css" />`},
					{Path: api.Proxy.ListenPath, BodyMatch: `endpoint: "\\/` + apiName + `\\/"`},
					{Path: api.Proxy.ListenPath + "playground.css", BodyMatch: "body{margin:0;padding:0;font-family:.*"},
				}...)
			})

			t.Run("path is /playground", func(t *testing.T) {
				api.GraphQL.GraphQLPlayground.Path = "/playground"
				LoadAPI(api)
				_, _ = g.Run(t, []test.TestCase{
					{Path: api.Proxy.ListenPath + "playground", BodyMatch: `<link rel="stylesheet" href="playground/playground.css" />`},
					{Path: api.Proxy.ListenPath + "playground", BodyMatch: `endpoint: "\\/` + apiName + `\\/"`},
					{Path: api.Proxy.ListenPath + "playground/playground.css", BodyMatch: "body{margin:0;padding:0;font-family:.*"},
				}...)
			})
		})
	}
}

func TestCORS(t *testing.T) {
	g := StartTest()
	defer g.Close()

	apis := BuildAndLoadAPI(func(spec *APISpec) {
		spec.Name = "CORS test API"
		spec.APIID = "cors-api"
		spec.Proxy.ListenPath = "/cors-api/"
		spec.CORS.Enable = false
		spec.CORS.ExposedHeaders = []string{"Custom-Header"}
		spec.CORS.AllowedOrigins = []string{"*"}
	}, func(spec *APISpec) {
		spec.Name = "Another API"
		spec.APIID = "another-api"
		spec.Proxy.ListenPath = "/another-api/"
		spec.CORS.ExposedHeaders = []string{"Custom-Header"}
		spec.CORS.AllowedOrigins = []string{"*"}
	})

	headers := map[string]string{
		"Origin": "my-custom-origin",
	}

	headersMatch := map[string]string{
		"Access-Control-Allow-Origin":   "*",
		"Access-Control-Expose-Headers": "Custom-Header",
	}

	t.Run("CORS disabled", func(t *testing.T) {
		_, _ = g.Run(t, []test.TestCase{
			{Path: "/cors-api/", Headers: headers, HeadersNotMatch: headersMatch, Code: http.StatusOK},
		}...)
	})

	t.Run("CORS enabled", func(t *testing.T) {
		apis[0].CORS.Enable = true
		LoadAPI(apis...)

		_, _ = g.Run(t, []test.TestCase{
			{Path: "/cors-api/", Headers: headers, HeadersMatch: headersMatch, Code: http.StatusOK},
		}...)

		_, _ = g.Run(t, []test.TestCase{
			{Path: "/another-api/", Headers: headers, HeadersNotMatch: headersMatch, Code: http.StatusOK},
		}...)
	})

	t.Run("oauth endpoints", func(t *testing.T) {
		apis[0].UseOauth2 = true
		apis[0].CORS.Enable = false
		LoadAPI(apis...)

		t.Run("CORS disabled", func(t *testing.T) {
			_, _ = g.Run(t, []test.TestCase{
				{Path: "/cors-api/oauth/token", Headers: headers, HeadersNotMatch: headersMatch, Code: http.StatusForbidden},
			}...)
		})

		t.Run("CORS enabled", func(t *testing.T) {
			apis[0].CORS.Enable = true
			LoadAPI(apis...)

			_, _ = g.Run(t, []test.TestCase{
				{Path: "/cors-api/oauth/token", Headers: headers, HeadersMatch: headersMatch, Code: http.StatusForbidden},
			}...)
		})
	})
}
<commit_msg>Add test for rate limit status of an API (#3491)<commit_after>package gateway

import (
	"fmt"
	"net/http"
	"sync/atomic"
	"testing"

	"github.com/TykTechnologies/tyk/user"

	"github.com/TykTechnologies/tyk/config"

	"github.com/stretchr/testify/assert"

	"github.com/TykTechnologies/tyk/test"

	"github.com/TykTechnologies/tyk/trace"
)

func TestOpenTracing(t *testing.T) {
	ts := StartTest()
	defer ts.Close()
	trace.SetupTracing("test", nil)
	defer trace.Close()

	t.Run("ensure the manager is enabled", func(ts *testing.T) {
		if !trace.IsEnabled() {
			ts.Error("expected tracing manager should be enabled")
		}
	})

	t.Run("ensure services are initialized", func(ts *testing.T) {
		var s atomic.Value
		trace.SetInit(func(name string, service string, opts map[string]interface{}, logger trace.Logger) (trace.Tracer, error) {
			s.Store(service)
			return trace.NoopTracer{}, nil
		})
		name := "trace"
		BuildAndLoadAPI(
			func(spec *APISpec) {
				spec.Name = name
				spec.UseOauth2 = true
			},
		)
		var n string
		if v := s.Load(); v != nil {
			n = v.(string)
		}
		if name != n {
			ts.Errorf("expected %s got %s", name, n)
		}
	})
}

func TestInternalAPIUsage(t *testing.T) {
	g := StartTest()
	defer g.Close()

	internal := BuildAPI(func(spec *APISpec) {
		spec.Name = "internal"
		spec.APIID = "test1"
		spec.Proxy.ListenPath = "/"
	})[0]

	normal := BuildAPI(func(spec *APISpec) {
		spec.Name = "normal"
		spec.APIID = "test2"
		spec.Proxy.TargetURL = fmt.Sprintf("tyk://%s", internal.Name)
		spec.Proxy.ListenPath = "/normal-api"
	})[0]

	LoadAPI(internal, normal)

	t.Run("with name", func(t *testing.T) {
		_, _ = g.Run(t, []test.TestCase{
			{Path: "/normal-api", Code: http.StatusOK},
		}...)
	})

	t.Run("with api id", func(t *testing.T) {
		normal.Proxy.TargetURL = fmt.Sprintf("tyk://%s", internal.APIID)

		LoadAPI(internal, normal)

		_, _ = g.Run(t, []test.TestCase{
			{Path: "/normal-api", Code: http.StatusOK},
		}...)
	})
}

func TestFuzzyFindAPI(t *testing.T) {
	BuildAndLoadAPI(func(spec *APISpec) {
		spec.Name = "IgnoreCase"
		spec.APIID = "123456"
		spec.Proxy.ListenPath = "/"
	})

	spec := fuzzyFindAPI("ignoreCase")
	assert.Equal(t, "123456", spec.APIID)
}

func TestGraphQLPlayground(t *testing.T) {
	g := StartTest()
	defer g.Close()

	defer ResetTestConfig()

	const apiName = "graphql-api"

	api := BuildAPI(func(spec *APISpec) {
		spec.APIID = "APIID"
		spec.Proxy.ListenPath = fmt.Sprintf("/%s/", apiName)
		spec.GraphQL.Enabled = true
		spec.GraphQL.GraphQLPlayground.Enabled = true
	})[0]

	for _, env := range []string{"on-premise", "cloud"} {
		if env == "cloud" {
			api.Proxy.ListenPath = fmt.Sprintf("/%s/", api.APIID)
			api.Slug = apiName
			globalConf := config.Global()
			globalConf.Cloud = true
			config.SetGlobal(globalConf)
		}

		t.Run(env, func(t *testing.T) {
			t.Run("path is empty", func(t *testing.T) {
				api.GraphQL.GraphQLPlayground.Path = ""
				LoadAPI(api)
				_, _ = g.Run(t, []test.TestCase{
					{Path: api.Proxy.ListenPath, BodyMatch: `<link rel="stylesheet" href="playground.css" />`},
					{Path: api.Proxy.ListenPath, BodyMatch: `endpoint: "\\/` + apiName + `\\/"`},
					{Path: api.Proxy.ListenPath + "playground.css", BodyMatch: "body{margin:0;padding:0;font-family:.*"},
				}...)
			})

			t.Run("path is /", func(t *testing.T) {
				api.GraphQL.GraphQLPlayground.Path = "/"
				LoadAPI(api)
				_, _ = g.Run(t, []test.TestCase{
					{Path: api.Proxy.ListenPath, BodyMatch: `<link rel="stylesheet" href="playground.css" />`},
					{Path: api.Proxy.ListenPath, BodyMatch: `endpoint: "\\/` + apiName + `\\/"`},
					{Path: api.Proxy.ListenPath + "playground.css", BodyMatch: "body{margin:0;padding:0;font-family:.*"},
				}...)
			})

			t.Run("path is /playground", func(t *testing.T) {
				api.GraphQL.GraphQLPlayground.Path = "/playground"
				LoadAPI(api)
				_, _ = g.Run(t, []test.TestCase{
					{Path: api.Proxy.ListenPath + "playground", BodyMatch: `<link rel="stylesheet" href="playground/playground.css" />`},
					{Path: api.Proxy.ListenPath + "playground", BodyMatch: `endpoint: "\\/` + apiName + `\\/"`},
					{Path: api.Proxy.ListenPath + "playground/playground.css", BodyMatch: "body{margin:0;padding:0;font-family:.*"},
				}...)
			})
		})
	}
}

func TestCORS(t *testing.T) {
	g := StartTest()
	defer g.Close()

	apis := BuildAndLoadAPI(func(spec *APISpec) {
		spec.Name = "CORS test API"
		spec.APIID = "cors-api"
		spec.Proxy.ListenPath = "/cors-api/"
		spec.CORS.Enable = false
		spec.CORS.ExposedHeaders = []string{"Custom-Header"}
		spec.CORS.AllowedOrigins = []string{"*"}
	}, func(spec *APISpec) {
		spec.Name = "Another API"
		spec.APIID = "another-api"
		spec.Proxy.ListenPath = "/another-api/"
		spec.CORS.ExposedHeaders = []string{"Custom-Header"}
		spec.CORS.AllowedOrigins = []string{"*"}
	})

	headers := map[string]string{
		"Origin": "my-custom-origin",
	}

	headersMatch := map[string]string{
		"Access-Control-Allow-Origin":   "*",
		"Access-Control-Expose-Headers": "Custom-Header",
	}

	t.Run("CORS disabled", func(t *testing.T) {
		_, _ = g.Run(t, []test.TestCase{
			{Path: "/cors-api/", Headers: headers, HeadersNotMatch: headersMatch, Code: http.StatusOK},
		}...)
	})

	t.Run("CORS enabled", func(t *testing.T) {
		apis[0].CORS.Enable = true
		LoadAPI(apis...)

		_, _ = g.Run(t, []test.TestCase{
			{Path: "/cors-api/", Headers: headers, HeadersMatch: headersMatch, Code: http.StatusOK},
		}...)

		_, _ = g.Run(t, []test.TestCase{
			{Path: "/another-api/", Headers: headers, HeadersNotMatch: headersMatch, Code: http.StatusOK},
		}...)
	})

	t.Run("oauth endpoints", func(t *testing.T) {
		apis[0].UseOauth2 = true
		apis[0].CORS.Enable = false
		LoadAPI(apis...)

		t.Run("CORS disabled", func(t *testing.T) {
			_, _ = g.Run(t, []test.TestCase{
				{Path: "/cors-api/oauth/token", Headers: headers, HeadersNotMatch: headersMatch, Code: http.StatusForbidden},
			}...)
		})

		t.Run("CORS enabled", func(t *testing.T) {
			apis[0].CORS.Enable = true
			LoadAPI(apis...)

			_, _ = g.Run(t, []test.TestCase{
				{Path: "/cors-api/oauth/token", Headers: headers, HeadersMatch: headersMatch, Code: http.StatusForbidden},
			}...)
		})
	})
}

func TestTykRateLimitsStatusOfAPI(t *testing.T) {
	g := StartTest()
	defer g.Close()

	const (
		quotaMax       = 20
		quotaRemaining = 10
		rate           = 10
		per            = 3
	)

	BuildAndLoadAPI(func(spec *APISpec) {
		spec.APIID = "test"
		spec.Proxy.ListenPath = "/my-api/"
		spec.UseKeylessAccess = false
	})
	_, key := g.CreateSession(func(s *user.SessionState) {
		s.QuotaMax = quotaMax
		s.QuotaRemaining = quotaRemaining
		s.Rate = rate
		s.Per = per

		s.AccessRights = map[string]user.AccessDefinition{"test": {
			APIID: "test", Versions: []string{"v1"},
		}}
	})

	authHeader := map[string]string{
		"Authorization": key,
	}

	bodyMatch := fmt.Sprintf(`{"quota":{"quota_max":%d,"quota_remaining":%d,"quota_renews":.*},"rate_limit":{"requests":%d,"per_unit":%d}}`,
		quotaMax, quotaRemaining, rate, per)

	_, _ = g.Run(t, test.TestCase{Path: "/my-api/tyk/rate-limits/", Headers: authHeader, BodyMatch: bodyMatch, Code: http.StatusOK})
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"io"
	"io/ioutil"
	"math"
	"strings"
	"testing"
	"time"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/mutable"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestObjectSyncer(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// fakeObjectCreator
////////////////////////////////////////////////////////////////////////

// An objectCreator that records the arguments it is called with, returning
// canned results.
type fakeObjectCreator struct {
	called bool

	// Supplied arguments
	srcObject *gcs.Object
	contents  []byte

	// Canned results
	o   *gcs.Object
	err error
}

func (oc *fakeObjectCreator) Create(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	// Have we been called more than once?
	AssertFalse(oc.called)
	oc.called = true

	// Record args.
	oc.srcObject = srcObject
	oc.contents, err = ioutil.ReadAll(r)
	AssertEq(nil, err)

	// Return results.
	o, err = oc.o, oc.err
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const srcObjectContents = "taco"
const appendThreshold = int64(len(srcObjectContents))

type ObjectSyncerTest struct {
	ctx context.Context

	fullCreator   fakeObjectCreator
	appendCreator fakeObjectCreator

	bucket gcs.Bucket
	leaser lease.FileLeaser
	syncer ObjectSyncer
	clock  timeutil.SimulatedClock

	srcObject *gcs.Object
	content   mutable.Content
}

var _ SetUpInterface = &ObjectSyncerTest{}

func init() { RegisterTestSuite(&ObjectSyncerTest{}) }

func (t *ObjectSyncerTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Set up dependencies.
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")
	t.leaser = lease.NewFileLeaser("", math.MaxInt32, math.MaxInt32)
	t.syncer = newObjectSyncer(
		appendThreshold,
		&t.fullCreator,
		&t.appendCreator)

	t.clock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local))

	// Set up a source object.
	t.srcObject, err = t.bucket.CreateObject(
		t.ctx,
		&gcs.CreateObjectRequest{
			Name:     "foo",
			Contents: strings.NewReader(srcObjectContents),
		})

	AssertEq(nil, err)

	// Wrap a mutable.Content around it.
	t.content = mutable.NewContent(
		NewReadProxy(
			t.srcObject,
			nil,            // Initial read lease
			math.MaxUint64, // Chunk size
			t.leaser,
			t.bucket),
		&t.clock)

	// Return errors from the fakes by default.
	t.fullCreator.err = errors.New("Fake error")
	t.appendCreator.err = errors.New("Fake error")
}

func (t *ObjectSyncerTest) call() (
	rl lease.ReadLease, o *gcs.Object, err error) {
	rl, o, err = t.syncer.SyncObject(t.ctx, t.srcObject, t.content)
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ObjectSyncerTest) NotDirty() {
	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(nil, rl)
	ExpectEq(nil, o)

	// Neither creater should have been called.
	ExpectFalse(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SmallerThanSource() {
	// Truncate downward.
	err := t.content.Truncate(t.ctx, int64(len(srcObjectContents)-1))
	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SameSizeAsSource() {
	// Dirty a byte without changing the length.
	_, err := t.content.WriteAt(
		t.ctx,
		[]byte("a"),
		int64(len(srcObjectContents)-1))

	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) LargerThanSource_ThresholdInSource() {
	var err error

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+100))
	AssertEq(nil, err)

	// But dirty a byte within the initial content.
	_, err = t.content.WriteAt(
		t.ctx,
		[]byte("a"),
		int64(len(srcObjectContents)-1))

	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SourceTooShortForAppend() {
	var err error

	// Recreate the syncer with a higher append threshold.
	t.syncer = newObjectSyncer(
		int64(len(srcObjectContents)+1),
		&t.fullCreator,
		&t.appendCreator)

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+1))
	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SourceComponentCountTooHigh() {
	var err error

	// Simulate a large component count.
	t.srcObject.ComponentCount = gcs.MaxComponentCount

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+1))
	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) LargerThanSource_ThresholdAtEndOfSource() {
	var err error

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+1))
	AssertEq(nil, err)

	// The append creator should be called.
	t.call()

	ExpectFalse(t.fullCreator.called)
	ExpectTrue(t.appendCreator.called)
}

func (t *ObjectSyncerTest) CallsFullCreator() {
	var err error
	AssertLt(2, t.srcObject.Size)

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	t.call()

	AssertTrue(t.fullCreator.called)
	ExpectEq(t.srcObject, t.fullCreator.srcObject)
	ExpectEq(srcObjectContents[:2], string(t.fullCreator.contents))
}

func (t *ObjectSyncerTest) FullCreatorFails() {
	var err error
	t.fullCreator.err = errors.New("taco")

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectThat(err, Error(HasSubstr("fullCreator.Create")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ObjectSyncerTest) FullCreatorReturnsPreconditionError() {
	var err error
	t.fullCreator.err = &gcs.PreconditionError{}

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectEq(t.fullCreator.err, err)
}

func (t *ObjectSyncerTest) FullCreatorSucceeds() {
	var err error
	t.fullCreator.o = &gcs.Object{}
	t.fullCreator.err = nil

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(t.fullCreator.o, o)

	// Check the read lease.
	_, err = rl.Seek(0, 0)
	AssertEq(nil, err)

	buf, err := ioutil.ReadAll(rl)
	AssertEq(nil, err)
	ExpectEq(srcObjectContents[:2], string(buf))
}

func (t *ObjectSyncerTest) CallsAppendCreator() {
	var err error

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	t.call()

	AssertTrue(t.appendCreator.called)
	ExpectEq(t.srcObject, t.appendCreator.srcObject)
	ExpectEq("burrito", string(t.appendCreator.contents))
}

func (t *ObjectSyncerTest) AppendCreatorFails() {
	var err error
	t.appendCreator.err = errors.New("taco")

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectThat(err, Error(HasSubstr("appendCreator.Create")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ObjectSyncerTest) AppendCreatorReturnsPreconditionError() {
	var err error
	t.appendCreator.err = &gcs.PreconditionError{}

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectEq(t.appendCreator.err, err)
}

func (t *ObjectSyncerTest) AppendCreatorSucceeds() {
	var err error
	t.appendCreator.o = &gcs.Object{}
	t.appendCreator.err = nil

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(t.appendCreator.o, o)

	// Check the read lease.
	_, err = rl.Seek(0, 0)
	AssertEq(nil, err)

	buf, err := ioutil.ReadAll(rl)
	AssertEq(nil, err)
	ExpectEq(srcObjectContents+"burrito", string(buf))
}
<commit_msg>Fixed two out of date tests.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"io"
	"io/ioutil"
	"math"
	"strings"
	"testing"
	"time"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/mutable"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestObjectSyncer(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// fakeObjectCreator
////////////////////////////////////////////////////////////////////////

// An objectCreator that records the arguments it is called with, returning
// canned results.
type fakeObjectCreator struct {
	called bool

	// Supplied arguments
	srcObject *gcs.Object
	contents  []byte

	// Canned results
	o   *gcs.Object
	err error
}

func (oc *fakeObjectCreator) Create(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	// Have we been called more than once?
	AssertFalse(oc.called)
	oc.called = true

	// Record args.
	oc.srcObject = srcObject
	oc.contents, err = ioutil.ReadAll(r)
	AssertEq(nil, err)

	// Return results.
	o, err = oc.o, oc.err
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const srcObjectContents = "taco"
const appendThreshold = int64(len(srcObjectContents))

type ObjectSyncerTest struct {
	ctx context.Context

	fullCreator   fakeObjectCreator
	appendCreator fakeObjectCreator

	bucket gcs.Bucket
	leaser lease.FileLeaser
	syncer ObjectSyncer
	clock  timeutil.SimulatedClock

	srcObject *gcs.Object
	content   mutable.Content
}

var _ SetUpInterface = &ObjectSyncerTest{}

func init() { RegisterTestSuite(&ObjectSyncerTest{}) }

func (t *ObjectSyncerTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Set up dependencies.
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")
	t.leaser = lease.NewFileLeaser("", math.MaxInt32, math.MaxInt32)
	t.syncer = newObjectSyncer(
		appendThreshold,
		&t.fullCreator,
		&t.appendCreator)

	t.clock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local))

	// Set up a source object.
	t.srcObject, err = t.bucket.CreateObject(
		t.ctx,
		&gcs.CreateObjectRequest{
			Name:     "foo",
			Contents: strings.NewReader(srcObjectContents),
		})

	AssertEq(nil, err)

	// Wrap a mutable.Content around it.
	t.content = mutable.NewContent(
		NewReadProxy(
			t.srcObject,
			nil,            // Initial read lease
			math.MaxUint64, // Chunk size
			t.leaser,
			t.bucket),
		&t.clock)

	// Return errors from the fakes by default.
	t.fullCreator.err = errors.New("Fake error")
	t.appendCreator.err = errors.New("Fake error")
}

func (t *ObjectSyncerTest) call() (
	rl lease.ReadLease, o *gcs.Object, err error) {
	rl, o, err = t.syncer.SyncObject(t.ctx, t.srcObject, t.content)
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ObjectSyncerTest) NotDirty() {
	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(nil, rl)
	ExpectEq(nil, o)

	// Neither creater should have been called.
	ExpectFalse(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SmallerThanSource() {
	// Truncate downward.
	err := t.content.Truncate(t.ctx, int64(len(srcObjectContents)-1))
	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SameSizeAsSource() {
	// Dirty a byte without changing the length.
	_, err := t.content.WriteAt(
		t.ctx,
		[]byte("a"),
		int64(len(srcObjectContents)-1))

	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) LargerThanSource_ThresholdInSource() {
	var err error

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+100))
	AssertEq(nil, err)

	// But dirty a byte within the initial content.
	_, err = t.content.WriteAt(
		t.ctx,
		[]byte("a"),
		int64(len(srcObjectContents)-1))

	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SourceTooShortForAppend() {
	var err error

	// Recreate the syncer with a higher append threshold.
	t.syncer = newObjectSyncer(
		int64(len(srcObjectContents)+1),
		&t.fullCreator,
		&t.appendCreator)

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+1))
	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) SourceComponentCountTooHigh() {
	var err error

	// Simulate a large component count.
	t.srcObject.ComponentCount = gcs.MaxComponentCount

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+1))
	AssertEq(nil, err)

	// The full creator should be called.
	t.call()

	ExpectTrue(t.fullCreator.called)
	ExpectFalse(t.appendCreator.called)
}

func (t *ObjectSyncerTest) LargerThanSource_ThresholdAtEndOfSource() {
	var err error

	// Extend the length of the content.
	err = t.content.Truncate(t.ctx, int64(len(srcObjectContents)+1))
	AssertEq(nil, err)

	// The append creator should be called.
	t.call()

	ExpectFalse(t.fullCreator.called)
	ExpectTrue(t.appendCreator.called)
}

func (t *ObjectSyncerTest) CallsFullCreator() {
	var err error
	AssertLt(2, t.srcObject.Size)

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	t.call()

	AssertTrue(t.fullCreator.called)
	ExpectEq(t.srcObject, t.fullCreator.srcObject)
	ExpectEq(srcObjectContents[:2], string(t.fullCreator.contents))
}

func (t *ObjectSyncerTest) FullCreatorFails() {
	var err error
	t.fullCreator.err = errors.New("taco")

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectThat(err, Error(HasSubstr("Create")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ObjectSyncerTest) FullCreatorReturnsPreconditionError() {
	var err error
	t.fullCreator.err = &gcs.PreconditionError{}

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectEq(t.fullCreator.err, err)
}

func (t *ObjectSyncerTest) FullCreatorSucceeds() {
	var err error
	t.fullCreator.o = &gcs.Object{}
	t.fullCreator.err = nil

	// Truncate downward.
	err = t.content.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(t.fullCreator.o, o)

	// Check the read lease.
	_, err = rl.Seek(0, 0)
	AssertEq(nil, err)

	buf, err := ioutil.ReadAll(rl)
	AssertEq(nil, err)
	ExpectEq(srcObjectContents[:2], string(buf))
}

func (t *ObjectSyncerTest) CallsAppendCreator() {
	var err error

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	t.call()

	AssertTrue(t.appendCreator.called)
	ExpectEq(t.srcObject, t.appendCreator.srcObject)
	ExpectEq("burrito", string(t.appendCreator.contents))
}

func (t *ObjectSyncerTest) AppendCreatorFails() {
	var err error
	t.appendCreator.err = errors.New("taco")

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectThat(err, Error(HasSubstr("Create")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ObjectSyncerTest) AppendCreatorReturnsPreconditionError() {
	var err error
	t.appendCreator.err = &gcs.PreconditionError{}

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	_, _, err = t.call()

	ExpectEq(t.appendCreator.err, err)
}

func (t *ObjectSyncerTest) AppendCreatorSucceeds() {
	var err error
	t.appendCreator.o = &gcs.Object{}
	t.appendCreator.err = nil

	// Append some data.
	_, err = t.content.WriteAt(t.ctx, []byte("burrito"), int64(t.srcObject.Size))
	AssertEq(nil, err)

	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(t.appendCreator.o, o)

	// Check the read lease.
	_, err = rl.Seek(0, 0)
	AssertEq(nil, err)

	buf, err := ioutil.ReadAll(rl)
	AssertEq(nil, err)
	ExpectEq(srcObjectContents+"burrito", string(buf))
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"github.com/docker/swarm/cluster"
	"github.com/docker/swarm/version"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"
)

func serveRequest(c cluster.Cluster, w http.ResponseWriter, req *http.Request) error {
	context := &context{
		cluster: c,
	}

	r := createRouter(context, false)
	r.ServeHTTP(w, req)
	return nil
}

func TestGetVersion(t *testing.T) {
	r := httptest.NewRecorder()
	req, err := http.NewRequest("GET", "/version", nil)
	assert.NoError(t, err)

	assert.NoError(t, serveRequest(nil, r, req))
	assert.Equal(t, r.Code, http.StatusOK)

	v := struct {
		Version string
	}{}

	json.NewDecoder(r.Body).Decode(&v)
	assert.Equal(t, v.Version, "swarm/"+version.VERSION)
}
<commit_msg>api: Remove unused test.<commit_after><|endoftext|>


<commit_before>package api

const (
	DefaultPortName         = "http"
	NavTruststoreFasitAlias = "nav_truststore"
)

func DefaultResourceRequests() []ResourceRequest {
	return []ResourceRequest{
		{
			Alias:        NavTruststoreFasitAlias,
			ResourceType: "certificate",
			PropertyMap:  map[string]string{"keystore": "NAV_TRUSTSTORE_PATH"},
		},
	}
}

func GetDefaultManifest(application string) NaisManifest {

	defaultManifest := NaisManifest{
		Replicas: Replicas{
			Min:                    2,
			Max:                    4,
			CpuThresholdPercentage: 50,
		},
		Port: 8080,
		Prometheus: PrometheusConfig{
			Enabled: false,
			Port:    DefaultPortName,
			Path:    "/metrics",
		},
		Istio: IstioConfig{
			Disabled: false,
		},
		Healthcheck: Healthcheck{
			Liveness: Probe{
				Path:             "isAlive",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
			Readiness: Probe{
				Path:             "isReady",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
		},
		Ingress: Ingress{Disabled: false},
		Resources: ResourceRequirements{
			Limits: ResourceList{
				Cpu:    "500m",
				Memory: "512Mi",
			},
			Requests: ResourceList{
				Cpu:    "200m",
				Memory: "256Mi",
			},
		},
		LeaderElection: false,
	}
	defaultManifest.Image = "docker.adeo.no:5000/" + application

	return defaultManifest
}
<commit_msg>Fixed env var<commit_after>package api

const (
	DefaultPortName         = "http"
	NavTruststoreFasitAlias = "nav_truststore"
)

func DefaultResourceRequests() []ResourceRequest {
	return []ResourceRequest{
		{
			Alias:        NavTruststoreFasitAlias,
			ResourceType: "certificate",
			PropertyMap:  map[string]string{"keystore": "NAV_TRUSTSTORE"},
		},
	}
}

func GetDefaultManifest(application string) NaisManifest {

	defaultManifest := NaisManifest{
		Replicas: Replicas{
			Min:                    2,
			Max:                    4,
			CpuThresholdPercentage: 50,
		},
		Port: 8080,
		Prometheus: PrometheusConfig{
			Enabled: false,
			Port:    DefaultPortName,
			Path:    "/metrics",
		},
		Istio: IstioConfig{
			Disabled: false,
		},
		Healthcheck: Healthcheck{
			Liveness: Probe{
				Path:             "isAlive",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
			Readiness: Probe{
				Path:             "isReady",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
		},
		Ingress: Ingress{Disabled: false},
		Resources: ResourceRequirements{
			Limits: ResourceList{
				Cpu:    "500m",
				Memory: "512Mi",
			},
			Requests: ResourceList{
				Cpu:    "200m",
				Memory: "256Mi",
			},
		},
		LeaderElection: false,
	}
	defaultManifest.Image = "docker.adeo.no:5000/" + application

	return defaultManifest
}
<|endoftext|>