<commit_before>package sudoku

import (
	"log"
	"math/rand"
	"strconv"
	"strings"
)

const NUM_NEIGHBORS = (DIM-1)*3 - (BLOCK_DIM-1)*2

type SymmetryType int

const (
	SYMMETRY_NONE SymmetryType = iota
	SYMMETRY_ANY
	SYMMETRY_HORIZONTAL
	SYMMETRY_VERTICAL
	SYMMETRY_BOTH
)

//Cell represents a single cell within a grid. It maintains information about the number that is filled, the numbers that are
//currently legal given the filled status of its neighbors, and whether any possibilities have been
//explicitly excluded by solve techniques. Cells should not be constructed on their own; create a Grid
//and grab references to the cells from there.
type Cell struct {
	grid *Grid
	//The number if it's explicitly set. Number() will return it if it's explicitly or implicitly set.
	number      int
	Row         int
	Col         int
	Block       int
	neighbors   CellList
	impossibles [DIM]int
	excluded    [DIM]bool
}

func newCell(grid *Grid, row int, col int) Cell {
	//TODO: we should not set the number until neighbors are initialized.
	return Cell{grid: grid, Row: row, Col: col, Block: grid.blockForCell(row, col)}
}

//InGrid returns a reference to a cell in the provided grid that has the same row/column as this cell.
//Effectively, this cell's analogue in the other grid.
func (self *Cell) InGrid(grid *Grid) *Cell {
	//Returns our analogue in the given grid.
	if grid == nil {
		return nil
	}
	return grid.Cell(self.Row, self.Col)
}

func (self *Cell) load(data string) {
	//Format, for now, is just the number itself, or 0 if no number.
	data = strings.Replace(data, ALT_0, "0", -1)
	num, _ := strconv.Atoi(data)
	self.SetNumber(num)
}

//Number returns the number the cell is currently set to.
func (self *Cell) Number() int {
	//A layer of indirection since number needs to be used from the Setter.
	return self.number
}

//SetNumber explicitly sets the number of the cell. This operation could cause the grid to become
//invalid if it conflicts with its neighbors' numbers. This operation will affect the Possiblities()
//of its neighbor cells.
func (self *Cell) SetNumber(number int) {
	//Sets the explicit number. This will affect its neighbors possibles list.
	if self.number == number {
		//No work to do now.
		return
	}
	oldNumber := self.number
	self.number = number
	if oldNumber > 0 {
		for i := 1; i <= DIM; i++ {
			if i == oldNumber {
				continue
			}
			self.setPossible(i)
		}
		self.alertNeighbors(oldNumber, true)
	}
	if number > 0 {
		for i := 1; i <= DIM; i++ {
			if i == number {
				continue
			}
			self.setImpossible(i)
		}
		self.alertNeighbors(number, false)
	}
	if self.grid != nil {
		self.grid.cellModified(self)
		if (oldNumber > 0 && number == 0) || (oldNumber == 0 && number > 0) {
			//Our rank will have changed.
			//TODO: figure out how to test this.
			self.grid.cellRankChanged(self)
		}
	}
}

func (self *Cell) alertNeighbors(number int, possible bool) {
	for _, cell := range self.Neighbors() {
		if possible {
			cell.setPossible(number)
		} else {
			cell.setImpossible(number)
		}
	}
}

func (self *Cell) setPossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	if self.impossibles[number] == 0 {
		log.Println("We were told to mark something that was already possible to possible.")
		return
	}
	self.impossibles[number]--
	if self.impossibles[number] == 0 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become valid.
		self.checkInvalid()
	}

}

func (self *Cell) setImpossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.impossibles[number]++
	if self.impossibles[number] == 1 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become invalid.
		self.checkInvalid()
	}
}

//SetExcluded defines whether a possibility is considered not feasible, even if not directly precluded
//by the Number()s of the cell's neighbors. This is used by advanced HumanSolve techniques that
//cull possibilities that are logically excluded by the state of the grid, in a non-direct way.
//The state of Excluded bits will affect the results of this cell's Possibilities() list.
func (self *Cell) SetExcluded(number int, excluded bool) {
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.excluded[number] = excluded
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

//ResetExcludes sets all excluded bits to false, so that Possibilities() will be based purely
//on direct implications of the Number()s of neighbors. See also SetExcluded.
func (self *Cell) ResetExcludes() {
	for i := 0; i < DIM; i++ {
		self.excluded[i] = false
	}
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

//Possible returns whether or not a given number is legal to fill via SetNumber, given the state of the grid (specifically,
//the cell's neighbors) and the numbers the cell was told to explicitly exclude via SetExclude.
//If the cell is already filled with a number, it will return false for all numbers.
func (self *Cell) Possible(number int) bool {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return false
	}
	return self.impossibles[number] == 0 && !self.excluded[number]
}

//Possibilities returns a list of all current possibilities for this cell: all numbers for which cell.Possible
//returns true.
func (self *Cell) Possibilities() (result []int) {
	//tODO: shouldn't this return an intslice?
	if self.number != 0 {
		return nil
	}
	for i := 1; i <= DIM; i++ {
		if self.Possible(i) {
			result = append(result, i)
		}
	}
	return result
}

func (self *Cell) checkInvalid() {
	if self.grid == nil {
		return
	}
	if self.Invalid() {
		self.grid.cellIsInvalid(self)
	} else {
		self.grid.cellIsValid(self)
	}
}

func (self *Cell) Invalid() bool {
	//Returns true if no numbers are possible.
	//TODO: figure out a way to send this back up to the solver when it happens.
	for i, counter := range self.impossibles {
		if counter == 0 && !self.excluded[i] {
			return false
		}
	}
	return true
}

func (self *Cell) rank() int {
	if self.number != 0 {
		return 0
	}
	count := 0
	for _, counter := range self.impossibles {
		if counter == 0 {
			count++
		}
	}
	return count
}

func (self *Cell) ref() cellRef {
	return cellRef{self.Row, self.Col}
}

//Sets ourselves to a random one of our possibilities.
func (self *Cell) pickRandom() {
	possibilities := self.Possibilities()
	choice := possibilities[rand.Intn(len(possibilities))]
	self.SetNumber(choice)
}

func (self *Cell) implicitNumber() int {
	//Impossibles is in 0-index space, but represents nubmers in 1-indexed space.
	result := -1
	for i, counter := range self.impossibles {
		if counter == 0 {
			//Is there someone else competing for this? If so there's no implicit number
			if result != -1 {
				return 0
			}
			result = i
		}
	}
	//convert from 0-indexed to 1-indexed
	return result + 1
}

func (self *Cell) SymmetricalPartner(symmetry SymmetryType) *Cell {

	if symmetry == SYMMETRY_ANY {
		//TODO: don't chose a type of smmetry that doesn't have a partner
		typesOfSymmetry := []SymmetryType{SYMMETRY_BOTH, SYMMETRY_HORIZONTAL, SYMMETRY_HORIZONTAL, SYMMETRY_VERTICAL}
		symmetry = typesOfSymmetry[rand.Intn(len(typesOfSymmetry))]
	}

	switch symmetry {
	case SYMMETRY_BOTH:
		if cell := self.grid.Cell(DIM-self.Row-1, DIM-self.Col-1); cell != self {
			return cell
		}
	case SYMMETRY_HORIZONTAL:
		if cell := self.grid.Cell(DIM-self.Row-1, self.Col); cell != self {
			return cell
		}
	case SYMMETRY_VERTICAL:
		if cell := self.grid.Cell(self.Row, DIM-self.Col-1); cell != self {
			return cell
		}
	}

	//If the cell was the same as self, or SYMMETRY_NONE
	return nil
}

func (self *Cell) Neighbors() CellList {
	if self.grid == nil || !self.grid.initalized {
		return nil
	}
	if self.neighbors == nil {
		//We don't want duplicates, so we will collect in a map (used as a set) and then reduce.
		neighborsMap := make(map[*Cell]bool)
		for _, cell := range self.grid.Row(self.Row) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Col(self.Col) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Block(self.Block) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		self.neighbors = make([]*Cell, len(neighborsMap))
		i := 0
		for cell, _ := range neighborsMap {
			self.neighbors[i] = cell
			i++
		}
	}
	return self.neighbors

}

func (self *Cell) DataString() string {
	result := strconv.Itoa(self.Number())
	return strings.Replace(result, "0", ALT_0, -1)
}

func (self *Cell) String() string {
	return "Cell[" + strconv.Itoa(self.Row) + "][" + strconv.Itoa(self.Col) + "]:" + strconv.Itoa(self.Number()) + "\n"
}

func (self *Cell) positionInBlock() (top, right, bottom, left bool) {
	if self.grid == nil {
		return
	}
	topRow, topCol, bottomRow, bottomCol := self.grid.blockExtents(self.Block)
	top = self.Row == topRow
	right = self.Col == bottomCol
	bottom = self.Row == bottomRow
	left = self.Col == topCol
	return
}

func (self *Cell) diagramRows() (rows []string) {
	//We'll only draw barriers at our bottom right edge.
	_, right, bottom, _ := self.positionInBlock()
	current := 0
	for r := 0; r < BLOCK_DIM; r++ {
		row := ""
		for c := 0; c < BLOCK_DIM; c++ {
			if self.number != 0 {
				//Print just the number.
				if r == BLOCK_DIM/2 && c == BLOCK_DIM/2 {
					row += strconv.Itoa(self.number)
				} else {
					row += DIAGRAM_NUMBER
				}
			} else {
				//Print the possibles.
				if self.Possible(current + 1) {
					row += strconv.Itoa(current + 1)
				} else {
					row += DIAGRAM_IMPOSSIBLE
				}
			}
			current++
		}
		rows = append(rows, row)
	}

	//Do we need to pad each row with | on the right?
	if !right {
		for i, data := range rows {
			rows[i] = data + DIAGRAM_RIGHT
		}
	}
	//Do we need an extra bottom row?
	if !bottom {
		rows = append(rows, strings.Repeat(DIAGRAM_BOTTOM, BLOCK_DIM))
		// Does it need a + at the end?
		if !right {
			rows[len(rows)-1] = rows[len(rows)-1] + DIAGRAM_CORNER
		}
	}

	return rows
}

func (self *Cell) Diagram() string {
	return strings.Join(self.diagramRows(), "\n")
}
<commit_msg>cell.Possibilities returns an IntSlice now instead of []int<commit_after>package sudoku

import (
	"log"
	"math/rand"
	"strconv"
	"strings"
)

const NUM_NEIGHBORS = (DIM-1)*3 - (BLOCK_DIM-1)*2

type SymmetryType int

const (
	SYMMETRY_NONE SymmetryType = iota
	SYMMETRY_ANY
	SYMMETRY_HORIZONTAL
	SYMMETRY_VERTICAL
	SYMMETRY_BOTH
)

//Cell represents a single cell within a grid. It maintains information about the number that is filled, the numbers that are
//currently legal given the filled status of its neighbors, and whether any possibilities have been
//explicitly excluded by solve techniques. Cells should not be constructed on their own; create a Grid
//and grab references to the cells from there.
type Cell struct {
	grid *Grid
	//The number if it's explicitly set. Number() will return it if it's explicitly or implicitly set.
	number      int
	Row         int
	Col         int
	Block       int
	neighbors   CellList
	impossibles [DIM]int
	excluded    [DIM]bool
}

func newCell(grid *Grid, row int, col int) Cell {
	//TODO: we should not set the number until neighbors are initialized.
	return Cell{grid: grid, Row: row, Col: col, Block: grid.blockForCell(row, col)}
}

//InGrid returns a reference to a cell in the provided grid that has the same row/column as this cell.
//Effectively, this cell's analogue in the other grid.
func (self *Cell) InGrid(grid *Grid) *Cell {
	//Returns our analogue in the given grid.
	if grid == nil {
		return nil
	}
	return grid.Cell(self.Row, self.Col)
}

func (self *Cell) load(data string) {
	//Format, for now, is just the number itself, or 0 if no number.
	data = strings.Replace(data, ALT_0, "0", -1)
	num, _ := strconv.Atoi(data)
	self.SetNumber(num)
}

//Number returns the number the cell is currently set to.
func (self *Cell) Number() int {
	//A layer of indirection since number needs to be used from the Setter.
	return self.number
}

//SetNumber explicitly sets the number of the cell. This operation could cause the grid to become
//invalid if it conflicts with its neighbors' numbers. This operation will affect the Possiblities()
//of its neighbor cells.
func (self *Cell) SetNumber(number int) {
	//Sets the explicit number. This will affect its neighbors possibles list.
	if self.number == number {
		//No work to do now.
		return
	}
	oldNumber := self.number
	self.number = number
	if oldNumber > 0 {
		for i := 1; i <= DIM; i++ {
			if i == oldNumber {
				continue
			}
			self.setPossible(i)
		}
		self.alertNeighbors(oldNumber, true)
	}
	if number > 0 {
		for i := 1; i <= DIM; i++ {
			if i == number {
				continue
			}
			self.setImpossible(i)
		}
		self.alertNeighbors(number, false)
	}
	if self.grid != nil {
		self.grid.cellModified(self)
		if (oldNumber > 0 && number == 0) || (oldNumber == 0 && number > 0) {
			//Our rank will have changed.
			//TODO: figure out how to test this.
			self.grid.cellRankChanged(self)
		}
	}
}

func (self *Cell) alertNeighbors(number int, possible bool) {
	for _, cell := range self.Neighbors() {
		if possible {
			cell.setPossible(number)
		} else {
			cell.setImpossible(number)
		}
	}
}

func (self *Cell) setPossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	if self.impossibles[number] == 0 {
		log.Println("We were told to mark something that was already possible to possible.")
		return
	}
	self.impossibles[number]--
	if self.impossibles[number] == 0 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become valid.
		self.checkInvalid()
	}

}

func (self *Cell) setImpossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.impossibles[number]++
	if self.impossibles[number] == 1 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become invalid.
		self.checkInvalid()
	}
}

//SetExcluded defines whether a possibility is considered not feasible, even if not directly precluded
//by the Number()s of the cell's neighbors. This is used by advanced HumanSolve techniques that
//cull possibilities that are logically excluded by the state of the grid, in a non-direct way.
//The state of Excluded bits will affect the results of this cell's Possibilities() list.
func (self *Cell) SetExcluded(number int, excluded bool) {
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.excluded[number] = excluded
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

//ResetExcludes sets all excluded bits to false, so that Possibilities() will be based purely
//on direct implications of the Number()s of neighbors. See also SetExcluded.
func (self *Cell) ResetExcludes() {
	for i := 0; i < DIM; i++ {
		self.excluded[i] = false
	}
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

//Possible returns whether or not a given number is legal to fill via SetNumber, given the state of the grid (specifically,
//the cell's neighbors) and the numbers the cell was told to explicitly exclude via SetExclude.
//If the cell is already filled with a number, it will return false for all numbers.
func (self *Cell) Possible(number int) bool {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return false
	}
	return self.impossibles[number] == 0 && !self.excluded[number]
}

//Possibilities returns a list of all current possibilities for this cell: all numbers for which cell.Possible
//returns true.
func (self *Cell) Possibilities() (result IntSlice) {
	if self.number != 0 {
		return nil
	}
	for i := 1; i <= DIM; i++ {
		if self.Possible(i) {
			result = append(result, i)
		}
	}
	return result
}

func (self *Cell) checkInvalid() {
	if self.grid == nil {
		return
	}
	if self.Invalid() {
		self.grid.cellIsInvalid(self)
	} else {
		self.grid.cellIsValid(self)
	}
}

func (self *Cell) Invalid() bool {
	//Returns true if no numbers are possible.
	//TODO: figure out a way to send this back up to the solver when it happens.
	for i, counter := range self.impossibles {
		if counter == 0 && !self.excluded[i] {
			return false
		}
	}
	return true
}

func (self *Cell) rank() int {
	if self.number != 0 {
		return 0
	}
	count := 0
	for _, counter := range self.impossibles {
		if counter == 0 {
			count++
		}
	}
	return count
}

func (self *Cell) ref() cellRef {
	return cellRef{self.Row, self.Col}
}

//Sets ourselves to a random one of our possibilities.
func (self *Cell) pickRandom() {
	possibilities := self.Possibilities()
	choice := possibilities[rand.Intn(len(possibilities))]
	self.SetNumber(choice)
}

func (self *Cell) implicitNumber() int {
	//Impossibles is in 0-index space, but represents nubmers in 1-indexed space.
	result := -1
	for i, counter := range self.impossibles {
		if counter == 0 {
			//Is there someone else competing for this? If so there's no implicit number
			if result != -1 {
				return 0
			}
			result = i
		}
	}
	//convert from 0-indexed to 1-indexed
	return result + 1
}

func (self *Cell) SymmetricalPartner(symmetry SymmetryType) *Cell {

	if symmetry == SYMMETRY_ANY {
		//TODO: don't chose a type of smmetry that doesn't have a partner
		typesOfSymmetry := []SymmetryType{SYMMETRY_BOTH, SYMMETRY_HORIZONTAL, SYMMETRY_HORIZONTAL, SYMMETRY_VERTICAL}
		symmetry = typesOfSymmetry[rand.Intn(len(typesOfSymmetry))]
	}

	switch symmetry {
	case SYMMETRY_BOTH:
		if cell := self.grid.Cell(DIM-self.Row-1, DIM-self.Col-1); cell != self {
			return cell
		}
	case SYMMETRY_HORIZONTAL:
		if cell := self.grid.Cell(DIM-self.Row-1, self.Col); cell != self {
			return cell
		}
	case SYMMETRY_VERTICAL:
		if cell := self.grid.Cell(self.Row, DIM-self.Col-1); cell != self {
			return cell
		}
	}

	//If the cell was the same as self, or SYMMETRY_NONE
	return nil
}

func (self *Cell) Neighbors() CellList {
	if self.grid == nil || !self.grid.initalized {
		return nil
	}
	if self.neighbors == nil {
		//We don't want duplicates, so we will collect in a map (used as a set) and then reduce.
		neighborsMap := make(map[*Cell]bool)
		for _, cell := range self.grid.Row(self.Row) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Col(self.Col) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Block(self.Block) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		self.neighbors = make([]*Cell, len(neighborsMap))
		i := 0
		for cell, _ := range neighborsMap {
			self.neighbors[i] = cell
			i++
		}
	}
	return self.neighbors

}

func (self *Cell) DataString() string {
	result := strconv.Itoa(self.Number())
	return strings.Replace(result, "0", ALT_0, -1)
}

func (self *Cell) String() string {
	return "Cell[" + strconv.Itoa(self.Row) + "][" + strconv.Itoa(self.Col) + "]:" + strconv.Itoa(self.Number()) + "\n"
}

func (self *Cell) positionInBlock() (top, right, bottom, left bool) {
	if self.grid == nil {
		return
	}
	topRow, topCol, bottomRow, bottomCol := self.grid.blockExtents(self.Block)
	top = self.Row == topRow
	right = self.Col == bottomCol
	bottom = self.Row == bottomRow
	left = self.Col == topCol
	return
}

func (self *Cell) diagramRows() (rows []string) {
	//We'll only draw barriers at our bottom right edge.
	_, right, bottom, _ := self.positionInBlock()
	current := 0
	for r := 0; r < BLOCK_DIM; r++ {
		row := ""
		for c := 0; c < BLOCK_DIM; c++ {
			if self.number != 0 {
				//Print just the number.
				if r == BLOCK_DIM/2 && c == BLOCK_DIM/2 {
					row += strconv.Itoa(self.number)
				} else {
					row += DIAGRAM_NUMBER
				}
			} else {
				//Print the possibles.
				if self.Possible(current + 1) {
					row += strconv.Itoa(current + 1)
				} else {
					row += DIAGRAM_IMPOSSIBLE
				}
			}
			current++
		}
		rows = append(rows, row)
	}

	//Do we need to pad each row with | on the right?
	if !right {
		for i, data := range rows {
			rows[i] = data + DIAGRAM_RIGHT
		}
	}
	//Do we need an extra bottom row?
	if !bottom {
		rows = append(rows, strings.Repeat(DIAGRAM_BOTTOM, BLOCK_DIM))
		// Does it need a + at the end?
		if !right {
			rows[len(rows)-1] = rows[len(rows)-1] + DIAGRAM_CORNER
		}
	}

	return rows
}

func (self *Cell) Diagram() string {
	return strings.Join(self.diagramRows(), "\n")
}
<|endoftext|>


<commit_before>package cmd

import (
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/noborus/trdsql"
)

// input format flag
type inputFlag struct {
	CSV  bool
	LTSV bool
	JSON bool
	TBLN bool
}

// Output format flag
type outputFlag struct {
	CSV  bool
	LTSV bool
	AT   bool
	MD   bool
	VF   bool
	RAW  bool
	JSON bool
	TBLN bool
}

func inputFormat(i inputFlag) trdsql.Format {
	switch {
	case i.CSV:
		return trdsql.CSV
	case i.LTSV:
		return trdsql.LTSV
	case i.JSON:
		return trdsql.JSON
	case i.TBLN:
		return trdsql.TBLN
	default:
		return trdsql.GUESS
	}
}

func outputFormat(o outputFlag) trdsql.Format {
	switch {
	case o.LTSV:
		return trdsql.LTSV
	case o.JSON:
		return trdsql.JSON
	case o.RAW:
		return trdsql.RAW
	case o.MD:
		return trdsql.MD
	case o.AT:
		return trdsql.AT
	case o.VF:
		return trdsql.VF
	case o.TBLN:
		return trdsql.TBLN
	case o.CSV:
		return trdsql.CSV
	default:
		return trdsql.CSV
	}
}

// Debug flag for a detailed output
var Debug bool

// Run executes the main routine.
// The return value is the exit code.
func Run(args []string) int {
	var (
		usage     bool
		version   bool
		dbList    bool
		config    string
		cDB       string
		cDriver   string
		cDSN      string
		guess     bool
		queryFile string
		analyze   string

		inFlag      inputFlag
		inDelimiter string
		inHeader    bool
		inSkip      int
		inPreRead   int

		outFlag      outputFlag
		outDelimiter string
		outHeader    bool
	)

	flags := flag.NewFlagSet(trdsql.AppName, flag.ExitOnError)

	flags.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s [OPTIONS] [SQL(SELECT...)]
`, os.Args[0])
		fmt.Fprintf(os.Stderr, `'See %s -help'
`, os.Args[0])
	}
	flags.StringVar(&config, "config", config, "Configuration file location.")
	flags.StringVar(&cDB, "db", "", "Specify db name of the setting.")
	flags.BoolVar(&dbList, "dblist", false, "display db information.")
	flags.StringVar(&cDriver, "driver", "", "database driver.  [ "+strings.Join(sql.Drivers(), " | ")+" ]")
	flags.StringVar(&cDSN, "dsn", "", "database connection option.")
	flags.BoolVar(&guess, "ig", true, "Guess format from extension.")
	flags.StringVar(&queryFile, "q", "", "Read query from the provided filename.")
	flags.StringVar(&analyze, "a", "", "Analyze file and suggest SQL.")
	flags.BoolVar(&usage, "help", false, "display usage information.")
	flags.BoolVar(&version, "version", false, "display version information.")
	flags.BoolVar(&Debug, "debug", false, "debug print.")

	flags.StringVar(&inDelimiter, "id", ",", "Field delimiter for input.")
	flags.BoolVar(&inHeader, "ih", false, "The first line is interpreted as column names(CSV only).")
	flags.IntVar(&inSkip, "is", 0, "Skip header row.")
	flags.IntVar(&inPreRead, "ir", 1, "Number of row preread for column determination.")

	flags.BoolVar(&inFlag.CSV, "icsv", false, "CSV format for input.")
	flags.BoolVar(&inFlag.LTSV, "iltsv", false, "LTSV format for input.")
	flags.BoolVar(&inFlag.JSON, "ijson", false, "JSON format for input.")
	flags.BoolVar(&inFlag.TBLN, "itbln", false, "TBLN format for input.")

	flags.StringVar(&outDelimiter, "od", ",", "Field delimiter for output.")
	flags.BoolVar(&outHeader, "oh", false, "Output column name as header.")

	flags.BoolVar(&outFlag.CSV, "ocsv", true, "CSV format for output.")
	flags.BoolVar(&outFlag.LTSV, "oltsv", false, "LTSV format for output.")
	flags.BoolVar(&outFlag.AT, "oat", false, "ASCII Table format for output.")
	flags.BoolVar(&outFlag.MD, "omd", false, "Mark Down format for output.")
	flags.BoolVar(&outFlag.VF, "ovf", false, "Vertical format for output.")
	flags.BoolVar(&outFlag.RAW, "oraw", false, "Raw format for output.")
	flags.BoolVar(&outFlag.JSON, "ojson", false, "JSON format for output.")
	flags.BoolVar(&outFlag.TBLN, "otbln", false, "TBLN format for output.")

	err := flags.Parse(args[1:])
	if err != nil {
		log.Println("ERROR:", err)
		return 1
	}

	if version {
		fmt.Println(trdsql.Version)
		return 0
	}

	if Debug {
		trdsql.EnableDebug()
	}

	cfgFile := configOpen(config)
	cfg, err := loadConfig(cfgFile)
	if err != nil {
		if config != "" {
			log.Printf("ERROR: [%s]%s", config, err)
			return 1
		}
	}
	if dbList {
		for od, odb := range cfg.Database {
			fmt.Printf("%s:%s\n", od, odb.Driver)
		}
		return 0
	}
	driver, dsn := getDB(cfg, cDB, cDriver, cDSN)

	// If flags.Args [0] is a file name,
	// it is regarded as an analysis file name.
	fileName := flags.Arg(0)
	_, nonEx := os.Stat(fileName)
	if nonEx == nil {
		analyze = fileName
	}
	if analyze != "" {
		command := getCommand(os.Args)
		if inHeader && inPreRead == 1 {
			inPreRead = 2
		}
		opts := trdsql.NewReadOpts(
			trdsql.InFormat(inputFormat(inFlag)),
			trdsql.InDelimiter(inDelimiter),
			trdsql.InHeader(inHeader),
			trdsql.InSkip(inSkip),
			trdsql.InPreRead(inPreRead))
		err := trdsql.Analyze(analyze, command, driver, opts)
		if err != nil {
			log.Println("ERROR:", err)
			return 1
		}
		return 0
	}

	query, err := getQuery(flags.Args(), queryFile)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return 1
	}

	if usage || (len(query) == 0) {
		fmt.Fprintf(os.Stderr, `
Usage: %s [OPTIONS] [SQL(SELECT...)]

Options:
`, os.Args[0])
		flags.PrintDefaults()
		return 2
	}

	importer := trdsql.NewImporter(
		trdsql.InFormat(inputFormat(inFlag)),
		trdsql.InDelimiter(inDelimiter),
		trdsql.InHeader(inHeader),
		trdsql.InSkip(inSkip),
		trdsql.InPreRead(inPreRead),
	)

	w := trdsql.NewWriter(
		trdsql.OutFormat(outputFormat(outFlag)),
		trdsql.OutDelimiter(outDelimiter),
		trdsql.OutHeader(outHeader),
	)
	exporter := trdsql.NewExporter(w)

	trd := trdsql.NewTRDSQL(importer, exporter)

	if driver != "" {
		trd.Driver = driver
	}
	if dsn != "" {
		trd.Dsn = dsn
	}

	err = trd.Exec(query)
	if err != nil {
		log.Fatal(err)
	}
	return 0
}

func getQuery(args []string, fileName string) (string, error) {
	query := ""
	if fileName != "" {
		sqlByte, err := ioutil.ReadFile(fileName)
		if err != nil {
			return "", err
		}
		query = string(sqlByte)
	} else {
		query = strings.Join(args, " ")
	}
	if strings.HasSuffix(query, ";") {
		query = query[:len(query)-1]
	}
	return query, nil
}

func getDB(cfg *config, cDB string, cDriver string, cDSN string) (string, string) {
	if cDB == "" {
		cDB = cfg.Db
	}
	if Debug {
		for od, odb := range cfg.Database {
			if cDB == od {
				log.Printf(">[driver: %s:%s:%s]", od, odb.Driver, odb.Dsn)
			} else {
				log.Printf(" [driver: %s:%s:%s]", od, odb.Driver, odb.Dsn)
			}
		}
	}
	if cDriver != "" {
		return cDriver, cDSN
	}
	if cDSN != "" {
		return "", cDSN
	}
	if cDB != "" {
		if cfg.Database[cDB].Driver == "" {
			log.Printf("ERROR: db[%s] does not found", cDB)
		} else {
			return cfg.Database[cDB].Driver, cfg.Database[cDB].Dsn
		}
	}
	return "", ""
}

func getCommand(args []string) string {
	command := trdsql.AppName
	aFlag := false
	for i, arg := range args {
		if i == 0 {
			continue
		}
		if aFlag {
			aFlag = false
			continue
		}
		if arg == "-a" {
			aFlag = true
			continue
		}
		command += " " + arg
	}
	return command
}
<commit_msg>Enclose arguments in double quotes if necessary<commit_after>package cmd

import (
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"github.com/noborus/trdsql"
)

// input format flag
type inputFlag struct {
	CSV  bool
	LTSV bool
	JSON bool
	TBLN bool
}

// Output format flag
type outputFlag struct {
	CSV  bool
	LTSV bool
	AT   bool
	MD   bool
	VF   bool
	RAW  bool
	JSON bool
	TBLN bool
}

func inputFormat(i inputFlag) trdsql.Format {
	switch {
	case i.CSV:
		return trdsql.CSV
	case i.LTSV:
		return trdsql.LTSV
	case i.JSON:
		return trdsql.JSON
	case i.TBLN:
		return trdsql.TBLN
	default:
		return trdsql.GUESS
	}
}

func outputFormat(o outputFlag) trdsql.Format {
	switch {
	case o.LTSV:
		return trdsql.LTSV
	case o.JSON:
		return trdsql.JSON
	case o.RAW:
		return trdsql.RAW
	case o.MD:
		return trdsql.MD
	case o.AT:
		return trdsql.AT
	case o.VF:
		return trdsql.VF
	case o.TBLN:
		return trdsql.TBLN
	case o.CSV:
		return trdsql.CSV
	default:
		return trdsql.CSV
	}
}

// Debug flag for a detailed output
var Debug bool

// Run executes the main routine.
// The return value is the exit code.
func Run(args []string) int {
	var (
		usage     bool
		version   bool
		dbList    bool
		config    string
		cDB       string
		cDriver   string
		cDSN      string
		guess     bool
		queryFile string
		analyze   string

		inFlag      inputFlag
		inDelimiter string
		inHeader    bool
		inSkip      int
		inPreRead   int

		outFlag      outputFlag
		outDelimiter string
		outHeader    bool
	)

	flags := flag.NewFlagSet(trdsql.AppName, flag.ExitOnError)

	flags.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s [OPTIONS] [SQL(SELECT...)]
`, os.Args[0])
		fmt.Fprintf(os.Stderr, `'See %s -help'
`, os.Args[0])
	}
	flags.StringVar(&config, "config", config, "Configuration file location.")
	flags.StringVar(&cDB, "db", "", "Specify db name of the setting.")
	flags.BoolVar(&dbList, "dblist", false, "display db information.")
	flags.StringVar(&cDriver, "driver", "", "database driver.  [ "+strings.Join(sql.Drivers(), " | ")+" ]")
	flags.StringVar(&cDSN, "dsn", "", "database connection option.")
	flags.BoolVar(&guess, "ig", true, "Guess format from extension.")
	flags.StringVar(&queryFile, "q", "", "Read query from the provided filename.")
	flags.StringVar(&analyze, "a", "", "Analyze file and suggest SQL.")
	flags.BoolVar(&usage, "help", false, "display usage information.")
	flags.BoolVar(&version, "version", false, "display version information.")
	flags.BoolVar(&Debug, "debug", false, "debug print.")

	flags.StringVar(&inDelimiter, "id", ",", "Field delimiter for input.")
	flags.BoolVar(&inHeader, "ih", false, "The first line is interpreted as column names(CSV only).")
	flags.IntVar(&inSkip, "is", 0, "Skip header row.")
	flags.IntVar(&inPreRead, "ir", 1, "Number of row preread for column determination.")

	flags.BoolVar(&inFlag.CSV, "icsv", false, "CSV format for input.")
	flags.BoolVar(&inFlag.LTSV, "iltsv", false, "LTSV format for input.")
	flags.BoolVar(&inFlag.JSON, "ijson", false, "JSON format for input.")
	flags.BoolVar(&inFlag.TBLN, "itbln", false, "TBLN format for input.")

	flags.StringVar(&outDelimiter, "od", ",", "Field delimiter for output.")
	flags.BoolVar(&outHeader, "oh", false, "Output column name as header.")

	flags.BoolVar(&outFlag.CSV, "ocsv", true, "CSV format for output.")
	flags.BoolVar(&outFlag.LTSV, "oltsv", false, "LTSV format for output.")
	flags.BoolVar(&outFlag.AT, "oat", false, "ASCII Table format for output.")
	flags.BoolVar(&outFlag.MD, "omd", false, "Mark Down format for output.")
	flags.BoolVar(&outFlag.VF, "ovf", false, "Vertical format for output.")
	flags.BoolVar(&outFlag.RAW, "oraw", false, "Raw format for output.")
	flags.BoolVar(&outFlag.JSON, "ojson", false, "JSON format for output.")
	flags.BoolVar(&outFlag.TBLN, "otbln", false, "TBLN format for output.")

	err := flags.Parse(args[1:])
	if err != nil {
		log.Println("ERROR:", err)
		return 1
	}

	if version {
		fmt.Println(trdsql.Version)
		return 0
	}

	if Debug {
		trdsql.EnableDebug()
	}

	cfgFile := configOpen(config)
	cfg, err := loadConfig(cfgFile)
	if err != nil {
		if config != "" {
			log.Printf("ERROR: [%s]%s", config, err)
			return 1
		}
	}
	if dbList {
		for od, odb := range cfg.Database {
			fmt.Printf("%s:%s\n", od, odb.Driver)
		}
		return 0
	}
	driver, dsn := getDB(cfg, cDB, cDriver, cDSN)

	// If flags.Args [0] is a file name,
	// it is regarded as an analysis file name.
	fileName := flags.Arg(0)
	_, nonEx := os.Stat(fileName)
	if nonEx == nil {
		analyze = fileName
	}
	if analyze != "" {
		command := getCommand(os.Args)
		if inHeader && inPreRead == 1 {
			inPreRead = 2
		}
		opts := trdsql.NewReadOpts(
			trdsql.InFormat(inputFormat(inFlag)),
			trdsql.InDelimiter(inDelimiter),
			trdsql.InHeader(inHeader),
			trdsql.InSkip(inSkip),
			trdsql.InPreRead(inPreRead))
		err := trdsql.Analyze(analyze, command, driver, opts)
		if err != nil {
			log.Println("ERROR:", err)
			return 1
		}
		return 0
	}

	query, err := getQuery(flags.Args(), queryFile)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return 1
	}

	if usage || (len(query) == 0) {
		fmt.Fprintf(os.Stderr, `
Usage: %s [OPTIONS] [SQL(SELECT...)]

Options:
`, os.Args[0])
		flags.PrintDefaults()
		return 2
	}

	importer := trdsql.NewImporter(
		trdsql.InFormat(inputFormat(inFlag)),
		trdsql.InDelimiter(inDelimiter),
		trdsql.InHeader(inHeader),
		trdsql.InSkip(inSkip),
		trdsql.InPreRead(inPreRead),
	)

	w := trdsql.NewWriter(
		trdsql.OutFormat(outputFormat(outFlag)),
		trdsql.OutDelimiter(outDelimiter),
		trdsql.OutHeader(outHeader),
	)
	exporter := trdsql.NewExporter(w)

	trd := trdsql.NewTRDSQL(importer, exporter)

	if driver != "" {
		trd.Driver = driver
	}
	if dsn != "" {
		trd.Dsn = dsn
	}

	err = trd.Exec(query)
	if err != nil {
		log.Fatal(err)
	}
	return 0
}

func getQuery(args []string, fileName string) (string, error) {
	query := ""
	if fileName != "" {
		sqlByte, err := ioutil.ReadFile(fileName)
		if err != nil {
			return "", err
		}
		query = string(sqlByte)
	} else {
		query = strings.Join(args, " ")
	}
	if strings.HasSuffix(query, ";") {
		query = query[:len(query)-1]
	}
	return query, nil
}

func getDB(cfg *config, cDB string, cDriver string, cDSN string) (string, string) {
	if cDB == "" {
		cDB = cfg.Db
	}
	if Debug {
		for od, odb := range cfg.Database {
			if cDB == od {
				log.Printf(">[driver: %s:%s:%s]", od, odb.Driver, odb.Dsn)
			} else {
				log.Printf(" [driver: %s:%s:%s]", od, odb.Driver, odb.Dsn)
			}
		}
	}
	if cDriver != "" {
		return cDriver, cDSN
	}
	if cDSN != "" {
		return "", cDSN
	}
	if cDB != "" {
		if cfg.Database[cDB].Driver == "" {
			log.Printf("ERROR: db[%s] does not found", cDB)
		} else {
			return cfg.Database[cDB].Driver, cfg.Database[cDB].Dsn
		}
	}
	return "", ""
}

func getCommand(args []string) string {
	command := args[0]
	omitFlag := false
	for _, arg := range args[1:] {
		if omitFlag {
			omitFlag = false
			continue
		}
		if arg == "-a" {
			omitFlag = true
			continue
		}
		if arg[0] != '-' || len(arg) == 1 {
			arg = quotedArg(arg)
		}
		command += " " + arg
	}
	return command
}

var argQuote = regexp.MustCompile(`^[a-zA-Z0-9_]+$`)

func quotedArg(arg string) string {
	if argQuote.MatchString(arg) {
		return arg
	}
	return `"` + arg + `"`
}
<|endoftext|>


<commit_before>package cmd

import (
	"flag"

	"github.com/asim/go-micro/registry"
	"github.com/asim/go-micro/server"
	"github.com/asim/go-micro/store"
)

var (
	flagBindAddress string
	flagRegistry    string
	flagStore       string
)

func init() {
	flag.StringVar(&flagBindAddress, "bind_address", ":0", "Bind address for the server. 127.0.0.1:8080")
	flag.StringVar(&flagRegistry, "registry", "consul", "Registry for discovery. kubernetes, consul, etc")
	flag.StringVar(&flagStore, "store", "consul", "Store used as a basic key/value store using consul, memcached, etc")
}

func Init() {
	flag.Parse()

	server.Address = flagBindAddress

	switch flagRegistry {
	case "kubernetes":
		registry.DefaultRegistry = registry.NewKubernetesRegistry()
	}

	switch flagStore {
	case "memcached":
		store.DefaultStore = store.NewMemcacheStore()
	case "memory":
		store.DefaultStore = store.NewMemoryStore()
	case "etcd":
		store.DefaultStore = store.NewEtcdStore()
	}
}
<commit_msg>Switch to codegangsta/cli to be inline with micro<commit_after>package cmd

import (
	"os"
	"text/tabwriter"
	"text/template"

	"github.com/asim/go-micro/registry"
	"github.com/asim/go-micro/server"
	"github.com/asim/go-micro/store"
	"github.com/codegangsta/cli"
)

var (
	Flags = []cli.Flag{
		cli.StringFlag{Name: "bind_address", Value: ":0", Usage: "Bind address for the server. 127.0.0.1:8080"},
		cli.StringFlag{Name: "registry", Value: "consul", Usage: "Registry for discovery. kubernetes, consul, etc"},
		cli.StringFlag{Name: "store", Value: "consul", Usage: "Store used as a basic key/value store using consul, memcached, etc"},
	}
)

func Setup(c *cli.Context) error {
	server.Address = c.String("bind_address")

	switch c.String("registry") {
	case "kubernetes":
		registry.DefaultRegistry = registry.NewKubernetesRegistry()
	}

	switch c.String("store") {
	case "memcached":
		store.DefaultStore = store.NewMemcacheStore()
	case "memory":
		store.DefaultStore = store.NewMemoryStore()
	case "etcd":
		store.DefaultStore = store.NewEtcdStore()
	}

	return nil
}

func Init() {
	cli.AppHelpTemplate = `
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
`

	cli.HelpPrinter = func(templ string, data interface{}) {
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
		t := template.Must(template.New("help").Parse(templ))
		err := t.Execute(w, data)
		if err != nil {
			panic(err)
		}
		w.Flush()
		os.Exit(2)
	}

	app := cli.NewApp()
	app.HideVersion = true
	app.Usage = "a go micro app"
	app.Action = func(c *cli.Context) {}
	app.Before = Setup
	app.Flags = Flags
	app.RunAndExitOnError()
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/url"
	"sync"
	"time"

	xrpc "github.com/minio/minio/cmd/rpc"
	xnet "github.com/minio/minio/pkg/net"
)

// DefaultSkewTime - skew time is 15 minutes between minio peers.
const DefaultSkewTime = 15 * time.Minute

var errRPCRetry = fmt.Errorf("rpc: retry error")

func isNetError(err error) bool {
	if err == nil {
		return false
	}

	if uerr, isURLError := err.(*url.Error); isURLError {
		if uerr.Timeout() {
			return true
		}

		err = uerr.Err
	}

	_, isNetOpError := err.(*net.OpError)
	return isNetOpError
}

// RPCVersion - RPC semantic version based on semver 2.0.0 https://semver.org/.
type RPCVersion struct {
	Major uint64
	Minor uint64
	Patch uint64
}

// Compare - compares given version with this version.
func (v RPCVersion) Compare(o RPCVersion) int {
	compare := func(v1, v2 uint64) int {
		if v1 == v2 {
			return 0
		}

		if v1 > v2 {
			return 1
		}

		return -1
	}

	if r := compare(v.Major, o.Major); r != 0 {
		return r
	}

	if r := compare(v.Minor, o.Minor); r != 0 {
		return r
	}

	return compare(v.Patch, o.Patch)
}

func (v RPCVersion) String() string {
	return fmt.Sprintf("%v.%v.%v", v.Major, v.Minor, v.Patch)
}

// AuthArgs - base argument for any RPC call for authentication.
type AuthArgs struct {
	Token       string
	RPCVersion  RPCVersion
	RequestTime time.Time
}

// Authenticate - checks if given arguments are valid to allow RPC call.
// This is xrpc.Authenticator and is called in RPC server.
func (args AuthArgs) Authenticate() error {
	// Check whether request time is within acceptable skew time.
	utcNow := time.Now().UTC()
	if args.RequestTime.Sub(utcNow) > DefaultSkewTime || utcNow.Sub(args.RequestTime) > DefaultSkewTime {
		return fmt.Errorf("client time %v is too apart with server time %v", args.RequestTime, utcNow)
	}

	if globalRPCAPIVersion.Compare(args.RPCVersion) != 0 {
		return fmt.Errorf("version mismatch. expected: %v, received: %v", globalRPCAPIVersion, args.RPCVersion)
	}

	if !isAuthTokenValid(args.Token) {
		return errAuthentication
	}

	return nil
}

// SetAuthArgs - sets given authentication arguments to this args. This is called in RPC client.
func (args *AuthArgs) SetAuthArgs(authArgs AuthArgs) {
	*args = authArgs
}

// VoidReply - void (empty) RPC reply.
type VoidReply struct{}

// RPCClientArgs - RPC client arguments.
type RPCClientArgs struct {
	NewAuthTokenFunc func() string
	RPCVersion       RPCVersion
	ServiceName      string
	ServiceURL       *xnet.URL
	TLSConfig        *tls.Config
}

// validate - checks whether given args are valid or not.
func (args RPCClientArgs) validate() error {
	if args.NewAuthTokenFunc == nil {
		return fmt.Errorf("NewAuthTokenFunc must not be empty")
	}

	if args.ServiceName == "" {
		return fmt.Errorf("ServiceName must not be empty")
	}

	if args.ServiceURL.Scheme != "http" && args.ServiceURL.Scheme != "https" {
		return fmt.Errorf("unknown RPC URL %v", args.ServiceURL)
	}

	if args.ServiceURL.User != nil || args.ServiceURL.ForceQuery || args.ServiceURL.RawQuery != "" || args.ServiceURL.Fragment != "" {
		return fmt.Errorf("unknown RPC URL %v", args.ServiceURL)
	}

	if args.ServiceURL.Scheme == "https" && args.TLSConfig == nil {
		return fmt.Errorf("tls configuration must not be empty for https url %v", args.ServiceURL)
	}

	return nil
}

// RPCClient - base RPC client.
type RPCClient struct {
	sync.RWMutex
	args        RPCClientArgs
	authToken   string
	rpcClient   *xrpc.Client
	retryTicker *time.Ticker
}

func (client *RPCClient) setRetryTicker(ticker *time.Ticker) {
	client.Lock()
	defer client.Unlock()

	if client.retryTicker != nil {
		client.retryTicker.Stop()
	}

	client.retryTicker = ticker
}

// Call - calls servicemethod on remote server.
func (client *RPCClient) Call(serviceMethod string, args interface {
	SetAuthArgs(args AuthArgs)
}, reply interface{}) (err error) {
	lockedCall := func() error {
		client.RLock()
		defer client.RUnlock()

		if client.retryTicker != nil {
			select {
			case <-client.retryTicker.C:
			default:
				return errRPCRetry
			}
		}

		// Make RPC call.
		args.SetAuthArgs(AuthArgs{client.authToken, client.args.RPCVersion, time.Now().UTC()})
		return client.rpcClient.Call(serviceMethod, args, reply)
	}

	call := func() error {
		err = lockedCall()

		if err == errRPCRetry {
			return err
		}

		if isNetError(err) {
			client.setRetryTicker(time.NewTicker(xrpc.DefaultRPCTimeout))
		} else {
			client.setRetryTicker(nil)
		}

		return err
	}

	// If authentication error is received, retry the same call only once
	// with new authentication token.
	if err = call(); err == nil {
		return nil
	}
	if err.Error() != errAuthentication.Error() {
		return err
	}

	client.Lock()
	client.authToken = client.args.NewAuthTokenFunc()
	client.Unlock()
	return call()
}

// Close - closes underneath RPC client.
func (client *RPCClient) Close() error {
	client.Lock()
	defer client.Unlock()

	client.authToken = ""
	return client.rpcClient.Close()
}

// ServiceURL - returns service URL used for RPC call.
func (client *RPCClient) ServiceURL() *xnet.URL {
	// Take copy of ServiceURL
	u := *(client.args.ServiceURL)
	return &u
}

// NewRPCClient - returns new RPC client.
func NewRPCClient(args RPCClientArgs) (*RPCClient, error) {
	if err := args.validate(); err != nil {
		return nil, err
	}

	return &RPCClient{
		args:      args,
		authToken: args.NewAuthTokenFunc(),
		rpcClient: xrpc.NewClient(args.ServiceURL, args.TLSConfig, xrpc.DefaultRPCTimeout),
	}, nil
}
<commit_msg>Add granular locking in retryTicker (#6236)<commit_after>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/url"
	"sync"
	"time"

	xrpc "github.com/minio/minio/cmd/rpc"
	xnet "github.com/minio/minio/pkg/net"
)

// DefaultSkewTime - skew time is 15 minutes between minio peers.
const DefaultSkewTime = 15 * time.Minute

var errRPCRetry = fmt.Errorf("rpc: retry error")

func isNetError(err error) bool {
	if err == nil {
		return false
	}

	if uerr, isURLError := err.(*url.Error); isURLError {
		if uerr.Timeout() {
			return true
		}

		err = uerr.Err
	}

	_, isNetOpError := err.(*net.OpError)
	return isNetOpError
}

// RPCVersion - RPC semantic version based on semver 2.0.0 https://semver.org/.
type RPCVersion struct {
	Major uint64
	Minor uint64
	Patch uint64
}

// Compare - compares given version with this version.
func (v RPCVersion) Compare(o RPCVersion) int {
	compare := func(v1, v2 uint64) int {
		if v1 == v2 {
			return 0
		}

		if v1 > v2 {
			return 1
		}

		return -1
	}

	if r := compare(v.Major, o.Major); r != 0 {
		return r
	}

	if r := compare(v.Minor, o.Minor); r != 0 {
		return r
	}

	return compare(v.Patch, o.Patch)
}

func (v RPCVersion) String() string {
	return fmt.Sprintf("%v.%v.%v", v.Major, v.Minor, v.Patch)
}

// AuthArgs - base argument for any RPC call for authentication.
type AuthArgs struct {
	Token       string
	RPCVersion  RPCVersion
	RequestTime time.Time
}

// Authenticate - checks if given arguments are valid to allow RPC call.
// This is xrpc.Authenticator and is called in RPC server.
func (args AuthArgs) Authenticate() error {
	// Check whether request time is within acceptable skew time.
	utcNow := time.Now().UTC()
	if args.RequestTime.Sub(utcNow) > DefaultSkewTime || utcNow.Sub(args.RequestTime) > DefaultSkewTime {
		return fmt.Errorf("client time %v is too apart with server time %v", args.RequestTime, utcNow)
	}

	if globalRPCAPIVersion.Compare(args.RPCVersion) != 0 {
		return fmt.Errorf("version mismatch. expected: %v, received: %v", globalRPCAPIVersion, args.RPCVersion)
	}

	if !isAuthTokenValid(args.Token) {
		return errAuthentication
	}

	return nil
}

// SetAuthArgs - sets given authentication arguments to this args. This is called in RPC client.
func (args *AuthArgs) SetAuthArgs(authArgs AuthArgs) {
	*args = authArgs
}

// VoidReply - void (empty) RPC reply.
type VoidReply struct{}

// RPCClientArgs - RPC client arguments.
type RPCClientArgs struct {
	NewAuthTokenFunc func() string
	RPCVersion       RPCVersion
	ServiceName      string
	ServiceURL       *xnet.URL
	TLSConfig        *tls.Config
}

// validate - checks whether given args are valid or not.
func (args RPCClientArgs) validate() error {
	if args.NewAuthTokenFunc == nil {
		return fmt.Errorf("NewAuthTokenFunc must not be empty")
	}

	if args.ServiceName == "" {
		return fmt.Errorf("ServiceName must not be empty")
	}

	if args.ServiceURL.Scheme != "http" && args.ServiceURL.Scheme != "https" {
		return fmt.Errorf("unknown RPC URL %v", args.ServiceURL)
	}

	if args.ServiceURL.User != nil || args.ServiceURL.ForceQuery || args.ServiceURL.RawQuery != "" || args.ServiceURL.Fragment != "" {
		return fmt.Errorf("unknown RPC URL %v", args.ServiceURL)
	}

	if args.ServiceURL.Scheme == "https" && args.TLSConfig == nil {
		return fmt.Errorf("tls configuration must not be empty for https url %v", args.ServiceURL)
	}

	return nil
}

// RPCClient - base RPC client.
type RPCClient struct {
	sync.RWMutex
	args        RPCClientArgs
	authToken   string
	rpcClient   *xrpc.Client
	retryTicker *time.Ticker
}

func (client *RPCClient) setRetryTicker(ticker *time.Ticker) {
	if ticker == nil {
		client.RLock()
		isNil := client.retryTicker == nil
		client.RUnlock()
		if isNil {
			return
		}
	}

	client.Lock()
	defer client.Unlock()

	if client.retryTicker != nil {
		client.retryTicker.Stop()
	}

	client.retryTicker = ticker
}

// Call - calls servicemethod on remote server.
func (client *RPCClient) Call(serviceMethod string, args interface {
	SetAuthArgs(args AuthArgs)
}, reply interface{}) (err error) {
	lockedCall := func() error {
		client.RLock()
		retryTicker := client.retryTicker
		client.RUnlock()
		if retryTicker != nil {
			select {
			case <-retryTicker.C:
			default:
				return errRPCRetry
			}
		}

		client.RLock()
		authToken := client.authToken
		client.RUnlock()

		// Make RPC call.
		args.SetAuthArgs(AuthArgs{authToken, client.args.RPCVersion, time.Now().UTC()})
		return client.rpcClient.Call(serviceMethod, args, reply)
	}

	call := func() error {
		err = lockedCall()

		if err == errRPCRetry {
			return err
		}

		if isNetError(err) {
			client.setRetryTicker(time.NewTicker(xrpc.DefaultRPCTimeout))
		} else {
			client.setRetryTicker(nil)
		}

		return err
	}

	// If authentication error is received, retry the same call only once
	// with new authentication token.
	if err = call(); err == nil {
		return nil
	}
	if err.Error() != errAuthentication.Error() {
		return err
	}

	client.Lock()
	client.authToken = client.args.NewAuthTokenFunc()
	client.Unlock()
	return call()
}

// Close - closes underneath RPC client.
func (client *RPCClient) Close() error {
	client.Lock()
	defer client.Unlock()

	client.authToken = ""
	return client.rpcClient.Close()
}

// ServiceURL - returns service URL used for RPC call.
func (client *RPCClient) ServiceURL() *xnet.URL {
	// Take copy of ServiceURL
	u := *(client.args.ServiceURL)
	return &u
}

// NewRPCClient - returns new RPC client.
func NewRPCClient(args RPCClientArgs) (*RPCClient, error) {
	if err := args.validate(); err != nil {
		return nil, err
	}

	return &RPCClient{
		args:      args,
		authToken: args.NewAuthTokenFunc(),
		rpcClient: xrpc.NewClient(args.ServiceURL, args.TLSConfig, xrpc.DefaultRPCTimeout),
	}, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os/exec"

	"github.com/markbates/grift/grift"
	"github.com/pkg/errors"
)

var currentGrift *grifter

func Run(name string, args []string) error {
	defer func() {
		currentGrift.TearDown()
	}()
	if len(args) == 1 {
		switch args[0] {
		case "jim":
			jimTribute()
			return nil
		case "init":
			generateInit()
			return nil
		case "--version", "-v":
			fmt.Println(Version)
			return nil
		case "--help", "-h":
			fmt.Println("grift <task name> [task arguments]")
			return nil
		}
	}

	err := setup(name)
	if err != nil {
		return errors.WithStack(err)
	}

	err = run(args)
	if err != nil {
		return errors.WithStack(err)
	}

	return nil
}

func run(args []string) error {
	rargs := []string{"run"}
	// Test for special cases requiring sqlite build tag
	if b, err := ioutil.ReadFile("database.yml"); err == nil {
		if bytes.Contains(b, []byte("sqlite")) {
			rargs = append(rargs, "-tags", "sqlite")
		}
	}
	rargs = append(rargs, exePath)
	rargs = append(rargs, args...)
	if err := grift.RunSource(exec.Command("go", rargs...)); err != nil {
		return errors.WithStack(err)
	}
	return nil
}

func list() error {
	rargs := []string{"run", exePath, "list"}
	return grift.RunSource(exec.Command("go", rargs...))
}

func setup(name string) error {
	var err error
	currentGrift, err = newGrifter(name)
	if err != nil {
		return errors.WithStack(err)
	}
	err = currentGrift.Setup()
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}
<commit_msg>Add support for sqlite build tag for config/database.yml<commit_after>package cmd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os/exec"

	"github.com/markbates/grift/grift"
	"github.com/pkg/errors"
)

var currentGrift *grifter

func Run(name string, args []string) error {
	defer func() {
		currentGrift.TearDown()
	}()
	if len(args) == 1 {
		switch args[0] {
		case "jim":
			jimTribute()
			return nil
		case "init":
			generateInit()
			return nil
		case "--version", "-v":
			fmt.Println(Version)
			return nil
		case "--help", "-h":
			fmt.Println("grift <task name> [task arguments]")
			return nil
		}
	}

	err := setup(name)
	if err != nil {
		return errors.WithStack(err)
	}

	err = run(args)
	if err != nil {
		return errors.WithStack(err)
	}

	return nil
}

func run(args []string) error {
	rargs := []string{"run"}
	// Test for special cases requiring sqlite build tag
	b, err := ioutil.ReadFile("database.yml")
	if err != nil {
		b, err = ioutil.ReadFile("config/database.yml")
	}
	if err == nil {
		if bytes.Contains(b, []byte("sqlite")) {
			rargs = append(rargs, "-tags", "sqlite")
		}
	}
	rargs = append(rargs, exePath)
	rargs = append(rargs, args...)
	if err := grift.RunSource(exec.Command("go", rargs...)); err != nil {
		return errors.WithStack(err)
	}
	return nil
}

func list() error {
	rargs := []string{"run", exePath, "list"}
	return grift.RunSource(exec.Command("go", rargs...))
}

func setup(name string) error {
	var err error
	currentGrift, err = newGrifter(name)
	if err != nil {
		return errors.WithStack(err)
	}
	err = currentGrift.Setup()
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/url"
	"sort"
	"strings"

	"github.com/achiku/varfmt"
	hschema "github.com/lestrrat/go-jshschema"
	schema "github.com/lestrrat/go-jsschema"
	jsval "github.com/lestrrat/go-jsval"
	"github.com/lestrrat/go-jsval/builder"
	"github.com/pkg/errors"
)

// PropType proper type
type PropType int

// Property types
const (
	PropTypeScalar PropType = iota
	PropTypeArray
	PropTypeObject
)

// Parser convertor
type Parser struct {
	schema  *schema.Schema
	pkgName string
}

// NewParser creates parser
func NewParser(sh *schema.Schema, pkgName string) *Parser {
	return &Parser{
		schema:  sh,
		pkgName: pkgName,
	}
}

func resolveSchema(sch *schema.Schema, root *schema.Schema) (*schema.Schema, error) {
	if sch.IsResolved() {
		return sch, nil
	}
	sh, err := sch.Resolve(root)
	if err != nil {
		return nil, err
	}
	// FIXME: recursively resolving schema. may need to limit # of recursion.
	return resolveSchema(sh, root)
}

func typesToStrings(types schema.PrimitiveTypes) []string {
	var vals []string
	for _, tt := range types {
		vals = append(vals, tt.String())
	}
	return vals
}

func sortProperties(props []*Property) []*Property {
	pMap := make(map[string]*Property)
	for _, p := range props {
		pMap[p.Name] = p
	}
	var names []string
	for n := range pMap {
		names = append(names, n)
	}
	sort.Strings(names)

	var sorted []*Property
	for _, n := range names {
		sorted = append(sorted, pMap[n])
	}
	return sorted
}

func sortActions(acs []Action) []Action {
	aMap := make(map[string]Action)
	for _, a := range acs {
		aMap[a.Method+a.Href] = a
	}
	var refs []string
	for r := range aMap {
		refs = append(refs, r)
	}
	sort.Strings(refs)
	var sorted []Action
	for _, r := range refs {
		sorted = append(sorted, aMap[r])
	}
	return sorted
}

func sortValidator(vals []*jsval.JSVal) []*jsval.JSVal {
	vMap := make(map[string]*jsval.JSVal)
	for _, v := range vals {
		vMap[v.Name] = v
	}
	var names []string
	for n := range vMap {
		names = append(names, n)
	}
	sort.Strings(names)
	var sorted []*jsval.JSVal
	for _, n := range names {
		sorted = append(sorted, vMap[n])
	}
	return sorted
}

// NewProperty new property
func NewProperty(name string, tp *schema.Schema, df *schema.Schema, root *schema.Schema) (*Property, error) {
	// save reference before resolving ref
	ref := tp.Reference
	fieldSchema, err := resolveSchema(tp, root)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to resolve, %s", name)
	}
	fld := &Property{
		Name:      name,
		Format:    string(fieldSchema.Format),
		Types:     fieldSchema.Type,
		Required:  df.IsPropRequired(name),
		Pattern:   fieldSchema.Pattern,
		Reference: ref,
		Schema:    fieldSchema,
	}
	switch {
	case fieldSchema.Type.Contains(schema.ArrayType):
		// if this field is an array
		// currently this tool supports only one itme per array field
		if len(fieldSchema.Items.Schemas) != 1 {
			return nil, errors.Errorf("array type has to have an item: %s", name)
		}
		item := fieldSchema.Items.Schemas[0]
		resolvedItem, err := resolveSchema(item, root)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to resolve: %s", name)
		}
		switch {
		case isMainResource(item.Reference) && resolvedItem.Type.Contains(schema.ObjectType):
			// reference to main resource object
			// log.Printf("ref to main resource: %s: %s", name, item.Reference)
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
			fld.SecondReference = item.Reference
		case item.Properties == nil && fieldSchema.Properties != nil:
			// field schema already has properties = inline object
			// log.Printf("inline obj: %s: %v", name, fieldSchema.Properties)
			var inlineFields []*Property
			for k, prop := range fieldSchema.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
		case item.Reference == "" && item.Properties == nil:
			// no reference, no item properties = primitive type
			// log.Printf("primitive type: %s %s", name, item.Type)
			fld.SecondTypes = item.Type
		case item.Reference != "" && !resolvedItem.Type.Contains(schema.ObjectType):
			// reference to primitive = resolved primitive type
			// log.Printf("resolved primitive type: %s %s", name, resolvedItem.Type)
			fld.SecondTypes = resolvedItem.Type
		case item.Reference == "" && item.Properties != nil:
			// no reference, item properties = inline object
			// parse properties, and recursively create inline fields
			// log.Printf("resolved inline obj: %s: %v", name, item.Properties)
			var inlineFields []*Property
			for k, prop := range item.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
		case !isMainResource(item.Reference):
			// log.Printf("resolved inline obj: %s: %v", name, resolvedItem.Properties)
			var inlineFields []*Property
			for k, prop := range resolvedItem.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
		}
		fld.PropType = PropTypeArray
	case fieldSchema.Type.Contains(schema.ObjectType):
		// if this field is a object
		switch {
		case fieldSchema.Reference == "" && fieldSchema.Properties != nil:
			// inline object without definitions
			var inlineFields []*Property
			for k, prop := range fieldSchema.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
		}
		fld.PropType = PropTypeObject
	default:
		// if this field is a scalar
		fld.PropType = PropTypeScalar
	}
	return fld, nil
}

// ParseValidators parse validator
func (p *Parser) ParseValidators() (Validators, error) {
	vals := make(Validators)
	for _, df := range p.schema.Definitions {
		for name, tp := range df.Properties {
			fs, err := resolveSchema(tp, p.schema)
			if err != nil {
				return nil, err
			}
			if fs.Pattern != nil &&
				!fs.Type.Contains(schema.ObjectType) && !fs.Type.Contains(schema.ArrayType) {
				v := Validator{
					Name:         name,
					RegexpString: fs.Pattern.String(),
				}
				vals[name] = v
			}
		}
	}
	return vals, nil
}

// ParseResources parse plain resource
func (p *Parser) ParseResources() (map[string]Resource, error) {
	res := make(map[string]Resource)
	// parse resource itself
	for id, df := range p.schema.Definitions {
		rs := Resource{
			Name:      id,
			Title:     df.Title,
			Schema:    df,
			IsPrimary: true,
		}
		// parse resource field
		var flds []*Property
		for name, tp := range df.Properties {
			fld, err := NewProperty(name, tp, df, p.schema)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to parse %s", id)
			}
			fld.InlineProperties = sortProperties(fld.InlineProperties)
			flds = append(flds, fld)
		}
		rs.Properties = sortProperties(flds)
		res[id] = rs
	}
	return res, nil
}

// ParseActions parse endpoints
func (p *Parser) ParseActions(res map[string]Resource) (map[string][]Action, error) {
	eptsMap := make(map[string][]Action)
	for id, df := range p.schema.Definitions {
		// use json hyper schema to parse links
		hsc := hschema.New()
		if err := hsc.Extract(df.Extras); err != nil {
			return nil, errors.Wrapf(err, "failed to extract links for (%s)", id)
		}
		// parse endpoints
		var eps []Action
		for _, e := range hsc.Links {
			href, err := url.QueryUnescape(e.Href)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to unescape %s", e.Href)
			}
			var encoding string
			if e.EncType == "" {
				encoding = "application/json"
			} else {
				encoding = e.EncType
			}
			ep := Action{
				Encoding: encoding,
				Href:     href,
				Method:   e.Method,
				Title:    e.Title,
				Rel:      e.Rel,
			}
			// parse request if exists
			if e.Schema != nil {
				var flds []*Property
				for name, tp := range e.Schema.Properties {
					fld, err := NewProperty(name, tp, df, p.schema)
					if err != nil {
						return nil, errors.Wrapf(err, "failed to parse %s", id)
					}
					flds = append(flds, fld)
				}
				ep.Request = &Resource{
					Name:       id,
					Properties: sortProperties(flds),
					Title:      e.Schema.Title,
					IsPrimary:  false,
				}
			}
			// parse response if exists
			if e.TargetSchema != nil {
				// http://json-schema.org/latest/json-schema-hypermedia.html#rfc.section.5.4
				switch {
				case e.TargetSchema.Reference == "":
					var flds []*Property
					for name, tp := range e.TargetSchema.Properties {
						fld, err := NewProperty(name, tp, df, p.schema)
						if err != nil {
							return nil, errors.Wrapf(err, "failed to parse %s", id)
						}
						flds = append(flds, fld)
					}
					ep.Response = &Resource{
						Name:       id,
						Properties: sortProperties(flds),
						Title:      e.TargetSchema.Title,
						Schema:     e.TargetSchema,
						IsPrimary:  false,
					}
				case e.TargetSchema.Reference != "" && IsRefToMainResource(e.TargetSchema.Reference):
					ep.Response = &Resource{
						Name:      id,
						Title:     e.TargetSchema.Title,
						Schema:    e.TargetSchema,
						IsPrimary: false,
					}
				case e.TargetSchema.Reference != "" && !IsRefToMainResource(e.TargetSchema.Reference):
					fld, err := NewProperty(e.TargetSchema.ID, e.TargetSchema, df, p.schema)
					if err != nil {
						return nil, errors.Wrapf(err, "failed to parse %s", id)
					}
					ep.Response = &Resource{
						Name:       id,
						Properties: sortProperties(fld.InlineProperties),
						Title:      e.TargetSchema.Title,
						Schema:     e.TargetSchema,
						IsPrimary:  false,
					}
				}
			} else {
				// if targetSchema is not set, use default resource for this link
				resp, ok := res[id]
				if !ok {
					return nil, errors.Errorf("resource not found: %s", id)
				}
				ep.Response = &resp
			}
			eps = append(eps, ep)
		}
		eptsMap[id] = sortActions(eps)
	}
	return eptsMap, nil
}

// ParseJsValValidators parse validator
func (p *Parser) ParseJsValValidators() ([]*jsval.JSVal, error) {
	var validators []*jsval.JSVal
	for id, df := range p.schema.Definitions {
		// use json hyper schema to parse links
		hsc := hschema.New()
		if err := hsc.Extract(df.Extras); err != nil {
			return nil, errors.Wrapf(err, "failed to extract links for (%s)", id)
		}

		for _, e := range hsc.Links {
			var v *jsval.JSVal
			if e.Schema == nil {
				v = jsval.New()
				v.SetRoot(jsval.Any())
			} else {
				sh, err := resolveSchema(e.Schema, p.schema)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to resolve, %s", id)
				}
				b := builder.New()
				v, err = b.BuildWithCtx(sh, p.schema)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to build validator: %s", id)
				}
			}
			v.Name = varfmt.PublicVarName(
				strings.Replace(id+strings.Title(e.Rel), "-", "_", -1) + "Validator")
			validators = append(validators, v)
		}
	}
	return sortValidator(validators), nil
}

func isMainResource(ref string) bool {
	if ref == "" {
		return false
	}
	tmp := strings.Replace(ref, "#/definitions/", "", 1)
	return !strings.Contains(tmp, "/")
}
<commit_msg>fix required request properties<commit_after>package main

import (
	"net/url"
	"sort"
	"strings"

	"github.com/achiku/varfmt"
	hschema "github.com/lestrrat/go-jshschema"
	schema "github.com/lestrrat/go-jsschema"
	jsval "github.com/lestrrat/go-jsval"
	"github.com/lestrrat/go-jsval/builder"
	"github.com/pkg/errors"
)

// PropType proper type
type PropType int

// Property types
const (
	PropTypeScalar PropType = iota
	PropTypeArray
	PropTypeObject
)

// Parser convertor
type Parser struct {
	schema  *schema.Schema
	pkgName string
}

// NewParser creates parser
func NewParser(sh *schema.Schema, pkgName string) *Parser {
	return &Parser{
		schema:  sh,
		pkgName: pkgName,
	}
}

func resolveSchema(sch *schema.Schema, root *schema.Schema) (*schema.Schema, error) {
	if sch.IsResolved() {
		return sch, nil
	}
	sh, err := sch.Resolve(root)
	if err != nil {
		return nil, err
	}
	// FIXME: recursively resolving schema. may need to limit # of recursion.
	return resolveSchema(sh, root)
}

func typesToStrings(types schema.PrimitiveTypes) []string {
	var vals []string
	for _, tt := range types {
		vals = append(vals, tt.String())
	}
	return vals
}

func sortProperties(props []*Property) []*Property {
	pMap := make(map[string]*Property)
	for _, p := range props {
		pMap[p.Name] = p
	}
	var names []string
	for n := range pMap {
		names = append(names, n)
	}
	sort.Strings(names)

	var sorted []*Property
	for _, n := range names {
		sorted = append(sorted, pMap[n])
	}
	return sorted
}

func sortActions(acs []Action) []Action {
	aMap := make(map[string]Action)
	for _, a := range acs {
		aMap[a.Method+a.Href] = a
	}
	var refs []string
	for r := range aMap {
		refs = append(refs, r)
	}
	sort.Strings(refs)
	var sorted []Action
	for _, r := range refs {
		sorted = append(sorted, aMap[r])
	}
	return sorted
}

func sortValidator(vals []*jsval.JSVal) []*jsval.JSVal {
	vMap := make(map[string]*jsval.JSVal)
	for _, v := range vals {
		vMap[v.Name] = v
	}
	var names []string
	for n := range vMap {
		names = append(names, n)
	}
	sort.Strings(names)
	var sorted []*jsval.JSVal
	for _, n := range names {
		sorted = append(sorted, vMap[n])
	}
	return sorted
}

// NewProperty new property
func NewProperty(name string, tp *schema.Schema, df *schema.Schema, root *schema.Schema) (*Property, error) {
	// save reference before resolving ref
	ref := tp.Reference
	fieldSchema, err := resolveSchema(tp, root)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to resolve, %s", name)
	}
	fld := &Property{
		Name:      name,
		Format:    string(fieldSchema.Format),
		Types:     fieldSchema.Type,
		Required:  df.IsPropRequired(name),
		Pattern:   fieldSchema.Pattern,
		Reference: ref,
		Schema:    fieldSchema,
	}
	switch {
	case fieldSchema.Type.Contains(schema.ArrayType):
		// if this field is an array
		// currently this tool supports only one itme per array field
		if len(fieldSchema.Items.Schemas) != 1 {
			return nil, errors.Errorf("array type has to have an item: %s", name)
		}
		item := fieldSchema.Items.Schemas[0]
		resolvedItem, err := resolveSchema(item, root)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to resolve: %s", name)
		}
		switch {
		case isMainResource(item.Reference) && resolvedItem.Type.Contains(schema.ObjectType):
			// reference to main resource object
			// log.Printf("ref to main resource: %s: %s", name, item.Reference)
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
			fld.SecondReference = item.Reference
		case item.Properties == nil && fieldSchema.Properties != nil:
			// field schema already has properties = inline object
			// log.Printf("inline obj: %s: %v", name, fieldSchema.Properties)
			var inlineFields []*Property
			for k, prop := range fieldSchema.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
		case item.Reference == "" && item.Properties == nil:
			// no reference, no item properties = primitive type
			// log.Printf("primitive type: %s %s", name, item.Type)
			fld.SecondTypes = item.Type
		case item.Reference != "" && !resolvedItem.Type.Contains(schema.ObjectType):
			// reference to primitive = resolved primitive type
			// log.Printf("resolved primitive type: %s %s", name, resolvedItem.Type)
			fld.SecondTypes = resolvedItem.Type
		case item.Reference == "" && item.Properties != nil:
			// no reference, item properties = inline object
			// parse properties, and recursively create inline fields
			// log.Printf("resolved inline obj: %s: %v", name, item.Properties)
			var inlineFields []*Property
			for k, prop := range item.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
		case !isMainResource(item.Reference):
			// log.Printf("resolved inline obj: %s: %v", name, resolvedItem.Properties)
			var inlineFields []*Property
			for k, prop := range resolvedItem.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
			fld.SecondTypes = []schema.PrimitiveType{schema.ObjectType}
		}
		fld.PropType = PropTypeArray
	case fieldSchema.Type.Contains(schema.ObjectType):
		// if this field is a object
		switch {
		case fieldSchema.Reference == "" && fieldSchema.Properties != nil:
			// inline object without definitions
			var inlineFields []*Property
			for k, prop := range fieldSchema.Properties {
				f, err := NewProperty(k, prop, df, root)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to perse inline object: %s", k)
				}
				inlineFields = append(inlineFields, f)
			}
			fld.InlineProperties = inlineFields
		}
		fld.PropType = PropTypeObject
	default:
		// if this field is a scalar
		fld.PropType = PropTypeScalar
	}
	return fld, nil
}

// ParseValidators parse validator
func (p *Parser) ParseValidators() (Validators, error) {
	vals := make(Validators)
	for _, df := range p.schema.Definitions {
		for name, tp := range df.Properties {
			fs, err := resolveSchema(tp, p.schema)
			if err != nil {
				return nil, err
			}
			if fs.Pattern != nil &&
				!fs.Type.Contains(schema.ObjectType) && !fs.Type.Contains(schema.ArrayType) {
				v := Validator{
					Name:         name,
					RegexpString: fs.Pattern.String(),
				}
				vals[name] = v
			}
		}
	}
	return vals, nil
}

// ParseResources parse plain resource
func (p *Parser) ParseResources() (map[string]Resource, error) {
	res := make(map[string]Resource)
	// parse resource itself
	for id, df := range p.schema.Definitions {
		rs := Resource{
			Name:      id,
			Title:     df.Title,
			Schema:    df,
			IsPrimary: true,
		}
		// parse resource field
		var flds []*Property
		for name, tp := range df.Properties {
			fld, err := NewProperty(name, tp, df, p.schema)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to parse %s", id)
			}
			fld.InlineProperties = sortProperties(fld.InlineProperties)
			flds = append(flds, fld)
		}
		rs.Properties = sortProperties(flds)
		res[id] = rs
	}
	return res, nil
}

// ParseActions parse endpoints
func (p *Parser) ParseActions(res map[string]Resource) (map[string][]Action, error) {
	eptsMap := make(map[string][]Action)
	for id, df := range p.schema.Definitions {
		// use json hyper schema to parse links
		hsc := hschema.New()
		if err := hsc.Extract(df.Extras); err != nil {
			return nil, errors.Wrapf(err, "failed to extract links for (%s)", id)
		}
		// parse endpoints
		var eps []Action
		for _, e := range hsc.Links {
			href, err := url.QueryUnescape(e.Href)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to unescape %s", e.Href)
			}
			var encoding string
			if e.EncType == "" {
				encoding = "application/json"
			} else {
				encoding = e.EncType
			}
			ep := Action{
				Encoding: encoding,
				Href:     href,
				Method:   e.Method,
				Title:    e.Title,
				Rel:      e.Rel,
			}
			// parse request if exists
			if e.Schema != nil {
				var flds []*Property
				for name, tp := range e.Schema.Properties {
					fld, err := NewProperty(name, tp, e.Schema, p.schema)
					if err != nil {
						return nil, errors.Wrapf(err, "failed to parse %s", id)
					}
					flds = append(flds, fld)
				}
				ep.Request = &Resource{
					Name:       id,
					Properties: sortProperties(flds),
					Title:      e.Schema.Title,
					IsPrimary:  false,
				}
			}
			// parse response if exists
			if e.TargetSchema != nil {
				// http://json-schema.org/latest/json-schema-hypermedia.html#rfc.section.5.4
				switch {
				case e.TargetSchema.Reference == "":
					var flds []*Property
					for name, tp := range e.TargetSchema.Properties {
						fld, err := NewProperty(name, tp, df, p.schema)
						if err != nil {
							return nil, errors.Wrapf(err, "failed to parse %s", id)
						}
						flds = append(flds, fld)
					}
					ep.Response = &Resource{
						Name:       id,
						Properties: sortProperties(flds),
						Title:      e.TargetSchema.Title,
						Schema:     e.TargetSchema,
						IsPrimary:  false,
					}
				case e.TargetSchema.Reference != "" && IsRefToMainResource(e.TargetSchema.Reference):
					ep.Response = &Resource{
						Name:      id,
						Title:     e.TargetSchema.Title,
						Schema:    e.TargetSchema,
						IsPrimary: false,
					}
				case e.TargetSchema.Reference != "" && !IsRefToMainResource(e.TargetSchema.Reference):
					fld, err := NewProperty(e.TargetSchema.ID, e.TargetSchema, df, p.schema)
					if err != nil {
						return nil, errors.Wrapf(err, "failed to parse %s", id)
					}
					ep.Response = &Resource{
						Name:       id,
						Properties: sortProperties(fld.InlineProperties),
						Title:      e.TargetSchema.Title,
						Schema:     e.TargetSchema,
						IsPrimary:  false,
					}
				}
			} else {
				// if targetSchema is not set, use default resource for this link
				resp, ok := res[id]
				if !ok {
					return nil, errors.Errorf("resource not found: %s", id)
				}
				ep.Response = &resp
			}
			eps = append(eps, ep)
		}
		eptsMap[id] = sortActions(eps)
	}
	return eptsMap, nil
}

// ParseJsValValidators parse validator
func (p *Parser) ParseJsValValidators() ([]*jsval.JSVal, error) {
	var validators []*jsval.JSVal
	for id, df := range p.schema.Definitions {
		// use json hyper schema to parse links
		hsc := hschema.New()
		if err := hsc.Extract(df.Extras); err != nil {
			return nil, errors.Wrapf(err, "failed to extract links for (%s)", id)
		}

		for _, e := range hsc.Links {
			var v *jsval.JSVal
			if e.Schema == nil {
				v = jsval.New()
				v.SetRoot(jsval.Any())
			} else {
				sh, err := resolveSchema(e.Schema, p.schema)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to resolve, %s", id)
				}
				b := builder.New()
				v, err = b.BuildWithCtx(sh, p.schema)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to build validator: %s", id)
				}
			}
			v.Name = varfmt.PublicVarName(
				strings.Replace(id+strings.Title(e.Rel), "-", "_", -1) + "Validator")
			validators = append(validators, v)
		}
	}
	return sortValidator(validators), nil
}

func isMainResource(ref string) bool {
	if ref == "" {
		return false
	}
	tmp := strings.Replace(ref, "#/definitions/", "", 1)
	return !strings.Contains(tmp, "/")
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015 Leon Dang, Nahanni Systems Inc
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

package ucl

import (
	"fmt"
	"io"
)

// The order of the keys as they appear in the file; this allows the user to
// have their own order for items.
const KeyOrder = "--ucl-keyorder--"

var UclExportKeyOrder bool = true

var Ucldebug bool = true
func debug(a... interface{}) {
	if Ucldebug {
		fmt.Println(a)
	}
}

type Parser struct {
	scanner *scanner

	ucl     map[string] interface{}

	tags    []*tag
	tagsi   int

	done    bool
	err     error
}

func NewParser(r io.Reader) *Parser {
	p := &Parser{
		scanner: newScanner(r),
		ucl: make(map[string] interface{}),
	}

	return p
}

func (p *Parser) nexttag() (*tag, error) {
	var err error

	if p.done {
		return nil, io.EOF
	}

	for {
		if p.tagsi >= len(p.tags) {
			p.tags, err = p.scanner.nexttags()
			if err != nil {
				return nil, err
			}
			p.tagsi = 0
		}
		for ; p.tagsi < len(p.tags); p.tagsi++ {
			m := p.tags[p.tagsi]
			if m.state == WHITESPACE || m.state == LCOMMENT ||
			   m.state == HCOMMENT {
				continue
			}
			p.tagsi++

			return m, nil
		}
	}

	return nil, fmt.Errorf("end of input line %d", p.scanner.line)
}

func (p *Parser) parsevalue(t *tag, parent interface{}) (interface{}, error) {
	var err error

restart:
	if t == nil {
		t, err = p.nexttag()
		if err != nil {
			return nil, err
		}
	}

	switch t.state {
	case TAG, QUOTE, VQUOTE, SLASH:
		// this could be either a value or a new key
		// have to see if the followon tags exist
		nt, err := p.nexttag()
		if err != nil {
			return nil, err
		}

		if nt == nil || nt.state == SEMICOL || nt.state == COMMA {
			return string(t.val), nil;  // leaf value; done
		}
		if nt.state == BRACECLOSE || nt.state == BRACKETCLOSE {
			nt.val = t.val
			return nt, nil
		}

		// "t" is a new key tag
		themap := make(map[string] interface{})
		res, err := p.parsevalue(nt, parent)

		if err != nil {
			debug("Error:", err)
			return nil, err
		}

		korder := make([]string, 1, 16)
		korder[0] = string(t.val)
		themap[KeyOrder] = korder
		themap[string(t.val)] = res
		return themap, nil

	case SEMICOL:
		// no value, let parent handle it
		if parent == nil {
			return t, fmt.Errorf("unexpected ';' at line %d", p.scanner.line)
		}
		return parent, nil

	case COMMA:
		// no value, let parent handle it
		if parent == nil {
			return t, fmt.Errorf("unexpected ',' at line %d", p.scanner.line)
		}
		return parent, nil

	case MLSTRING:
		// this must only be a value
		return string(t.val), nil

	case BRACEOPEN:
		// {, new map
		res, err := p.parse(t, parent)
		if err != nil {
			debug("parse error:", err)
		}
		return res, err

	case BRACECLOSE:
		// return until we hit the stack that has BRACEOPEN
		return parent, nil

	case BRACKETOPEN:
		thelist := make([]interface{}, 0, 32)
		res, err := p.parselist(nil, thelist)
		return res, err

	case BRACKETCLOSE:
		// list finished
		return parent, nil

	case EQUAL, COLON:
		t = nil
		goto restart
	}

	return nil, nil
}

func (p *Parser) parselist(t *tag, parent []interface{}) (ret interface{}, err error) {
	// Parse until bracket close
restart:
	if t == nil {
		t, err = p.nexttag()
		if err != nil {
			return nil, err
		}
	}

	switch t.state {
	case BRACKETCLOSE:
		// list finished
		return parent, nil

	case SEMICOL, COLON, EQUAL:
		// no value, let parent handle it
		return nil, fmt.Errorf("Invalid tag %s line %d",
		                       string(t.val), p.scanner.line)
	case COMMA:
		t = nil
		goto restart

	default:
		// append child
		res, err := p.parsevalue(t, nil)
		if err != nil {
			debug("error parsing value:", err)
			return nil, err
		} else {
			if restag, ok := res.(*tag); ok {
				// result is a tag; parsevalue didn't handle it
				if restag.state == BRACKETCLOSE {
					parent = append(parent, string(restag.val))
					return parent, nil
				} else {
					return nil, fmt.Errorf("Unexpected tag %s, line %d\n",
					               string(restag.val), p.scanner.line)
				}
			}

			parent = append(parent, res)
		}
		t = nil
		goto restart
	}
	return parent, err
}

func (p *Parser) parse(t *tag, parent interface{}) (ret interface{}, err error) {
	defer func() {
		p.err = err
	}()

restart:
	if t == nil {
		t, err = p.nexttag()
		if err != nil {
			return nil, err
		}
	}

	switch t.state {
	case TAG, QUOTE, VQUOTE, SLASH:
		// new key
		k := string(t.val)

		themap, ok := parent.(map[string] interface{})
		if !ok {
			debug("not a map at tag:", k)
			panic("...")
		}

		korder_intf, ok := themap[KeyOrder]
		var korder []string
		if !ok {
			korder = make([]string, 0, 16)
		} else {
			korder, ok = korder_intf.([]string)
			if !ok {
				debug("key order is not slice")
				return nil, fmt.Errorf("map[--keyorder--] is not slice")
			}
		}

		res, err := p.parsevalue(nil, nil)
		if err != nil {
			if restag, ok := res.(*tag); ok {
				if restag.state == SEMICOL {
					// no value for key, make it == null
					res = nil
				}
			} else {
				debug("parsevalue error:", err)
				return nil, err
			}
		} else if restag, ok := res.(*tag); ok {
			// result is a tag; parsevalue didn't handle it
			if restag.state != BRACECLOSE {
				t = restag
				goto restart
			}
			res = string(restag.val)
			t = restag
		}

		if mapitems, ok := themap[k]; ok {
			if childarray, ok := mapitems.([]interface{}); ok {
				// already an array, so append
				childarray = append(childarray, res)
				themap[k] = childarray
			} else {
				childarray := make([]interface{}, 1, 2)
				childarray[0] = themap[k]
				childarray = append(childarray, res)
				themap[k] = childarray
			}
		} else {
			// doesn't exist
			korder = append(korder, k)
			if UclExportKeyOrder {
				themap[KeyOrder] = korder
			}
			themap[k] = res
		}
		if t.state == BRACECLOSE {
			// map completed
			return parent, nil
		}

		// done for this tag; go to next
		t = nil
		goto restart

	case SEMICOL:
		t = nil
		goto restart

	case MLSTRING:
		// shouldn't happen
		return nil, fmt.Errorf("Unexpected multi-line string")

	case BRACEOPEN:
		// {
		// If parent is not a map and not nil, then error

		var theparent interface{}
		var ok bool
		if parent == nil {
			theparent = make(map[string] interface{})
		} else if theparent, ok = parent.(map[string] interface{}); !ok {
			if theparent, ok = parent.([]interface{}); !ok {
				debug("Error braceopen - parent is not a map/list/nil")
				return nil, fmt.Errorf("Invalid {, parent not nil|map|list")
			}
		}
		res, err := p.parse(nil, theparent)
		if err != nil {
			debug("Error parsing brace", err)
		}
		return res, err

	case BRACECLOSE:
		// map finished
		return parent, nil

	case BRACKETOPEN:
		thelist := make([]interface{}, 0, 32)
		res, err := p.parselist(nil, thelist)
		return res, err

	case BRACKETCLOSE:
		// list finished
		return parent, nil
	}

	return nil, nil
}

func (p *Parser) Ucl() (map[string] interface{}, error) {
	p.parse(nil, p.ucl)

	if p.err == io.EOF {
		p.err = nil
	}
	return p.ucl, p.err
}
<commit_msg>Only build KeyOrder array if it is to be exported<commit_after>/*
 * Copyright (c) 2015 Leon Dang, Nahanni Systems Inc
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

package ucl

import (
	"fmt"
	"io"
)

// The order of the keys as they appear in the file; this allows the user to
// have their own order for items.
const KeyOrder = "--ucl-keyorder--"

// Allow to disable constructing the KeyOrder arrays
var UclExportKeyOrder bool = true

var Ucldebug bool = true
func debug(a... interface{}) {
	if Ucldebug {
		fmt.Println(a)
	}
}

type Parser struct {
	scanner *scanner

	ucl     map[string] interface{}

	tags    []*tag
	tagsi   int

	done    bool
	err     error
}

func NewParser(r io.Reader) *Parser {
	p := &Parser{
		scanner: newScanner(r),
		ucl: make(map[string] interface{}),
	}

	return p
}

func (p *Parser) nexttag() (*tag, error) {
	var err error

	if p.done {
		return nil, io.EOF
	}

	for {
		if p.tagsi >= len(p.tags) {
			p.tags, err = p.scanner.nexttags()
			if err != nil {
				return nil, err
			}
			p.tagsi = 0
		}
		for ; p.tagsi < len(p.tags); p.tagsi++ {
			m := p.tags[p.tagsi]
			if m.state == WHITESPACE || m.state == LCOMMENT ||
			   m.state == HCOMMENT {
				continue
			}
			p.tagsi++

			return m, nil
		}
	}

	return nil, fmt.Errorf("end of input line %d", p.scanner.line)
}

func (p *Parser) parsevalue(t *tag, parent interface{}) (interface{}, error) {
	var err error

restart:
	if t == nil {
		t, err = p.nexttag()
		if err != nil {
			return nil, err
		}
	}

	switch t.state {
	case TAG, QUOTE, VQUOTE, SLASH:
		// this could be either a value or a new key
		// have to see if the followon tags exist
		nt, err := p.nexttag()
		if err != nil {
			return nil, err
		}

		if nt == nil || nt.state == SEMICOL || nt.state == COMMA {
			return string(t.val), nil;  // leaf value; done
		}
		if nt.state == BRACECLOSE || nt.state == BRACKETCLOSE {
			nt.val = t.val
			return nt, nil
		}

		// "t" is a new key tag
		themap := make(map[string] interface{})
		res, err := p.parsevalue(nt, parent)

		if err != nil {
			debug("Error:", err)
			return nil, err
		}

		korder := make([]string, 1, 16)
		korder[0] = string(t.val)
		themap[KeyOrder] = korder
		themap[string(t.val)] = res
		return themap, nil

	case SEMICOL:
		// no value, let parent handle it
		if parent == nil {
			return t, fmt.Errorf("unexpected ';' at line %d", p.scanner.line)
		}
		return parent, nil

	case COMMA:
		// no value, let parent handle it
		if parent == nil {
			return t, fmt.Errorf("unexpected ',' at line %d", p.scanner.line)
		}
		return parent, nil

	case MLSTRING:
		// this must only be a value
		return string(t.val), nil

	case BRACEOPEN:
		// {, new map
		res, err := p.parse(t, parent)
		if err != nil {
			debug("parse error:", err)
		}
		return res, err

	case BRACECLOSE:
		// return until we hit the stack that has BRACEOPEN
		return parent, nil

	case BRACKETOPEN:
		thelist := make([]interface{}, 0, 32)
		res, err := p.parselist(nil, thelist)
		return res, err

	case BRACKETCLOSE:
		// list finished
		return parent, nil

	case EQUAL, COLON:
		t = nil
		goto restart
	}

	return nil, nil
}

func (p *Parser) parselist(t *tag, parent []interface{}) (ret interface{}, err error) {
	// Parse until bracket close
restart:
	if t == nil {
		t, err = p.nexttag()
		if err != nil {
			return nil, err
		}
	}

	switch t.state {
	case BRACKETCLOSE:
		// list finished
		return parent, nil

	case SEMICOL, COLON, EQUAL:
		// no value, let parent handle it
		return nil, fmt.Errorf("Invalid tag %s line %d",
		                       string(t.val), p.scanner.line)
	case COMMA:
		t = nil
		goto restart

	default:
		// append child
		res, err := p.parsevalue(t, nil)
		if err != nil {
			debug("error parsing value:", err)
			return nil, err
		} else {
			if restag, ok := res.(*tag); ok {
				// result is a tag; parsevalue didn't handle it
				if restag.state == BRACKETCLOSE {
					parent = append(parent, string(restag.val))
					return parent, nil
				} else {
					return nil, fmt.Errorf("Unexpected tag %s, line %d\n",
					               string(restag.val), p.scanner.line)
				}
			}

			parent = append(parent, res)
		}
		t = nil
		goto restart
	}
	return parent, err
}

func (p *Parser) parse(t *tag, parent interface{}) (ret interface{}, err error) {
	defer func() {
		p.err = err
	}()

restart:
	if t == nil {
		t, err = p.nexttag()
		if err != nil {
			return nil, err
		}
	}

	switch t.state {
	case TAG, QUOTE, VQUOTE, SLASH:
		// new key
		k := string(t.val)

		themap, ok := parent.(map[string] interface{})
		if !ok {
			debug("not a map at tag:", k)
			panic("...")
		}

		korder_intf, ok := themap[KeyOrder]
		var korder []string
		if !ok {
			if UclExportKeyOrder {
				// only initialize if requested
				korder = make([]string, 0, 16)
			}
		} else {
			korder, ok = korder_intf.([]string)
			if !ok {
				debug("key order is not slice")
				return nil, fmt.Errorf("map[--keyorder--] is not slice")
			}
		}

		res, err := p.parsevalue(nil, nil)
		if err != nil {
			if restag, ok := res.(*tag); ok {
				if restag.state == SEMICOL {
					// no value for key, make it == null
					res = nil
				}
			} else {
				debug("parsevalue error:", err)
				return nil, err
			}
		} else if restag, ok := res.(*tag); ok {
			// result is a tag; parsevalue didn't handle it
			if restag.state != BRACECLOSE {
				t = restag
				goto restart
			}
			res = string(restag.val)
			t = restag
		}

		if mapitems, ok := themap[k]; ok {
			if childarray, ok := mapitems.([]interface{}); ok {
				// already an array, so append
				childarray = append(childarray, res)
				themap[k] = childarray
			} else {
				childarray := make([]interface{}, 1, 2)
				childarray[0] = themap[k]
				childarray = append(childarray, res)
				themap[k] = childarray
			}
		} else {
			// doesn't exist
			if cap(korder) != 0 {
				// only update KeyOrder if it was initialized
				korder = append(korder, k)
				themap[KeyOrder] = korder
			}
			themap[k] = res
		}
		if t.state == BRACECLOSE {
			// map completed
			return parent, nil
		}

		// done for this tag; go to next
		t = nil
		goto restart

	case SEMICOL:
		t = nil
		goto restart

	case MLSTRING:
		// shouldn't happen
		return nil, fmt.Errorf("Unexpected multi-line string")

	case BRACEOPEN:
		// {
		// If parent is not a map and not nil, then error

		var theparent interface{}
		var ok bool
		if parent == nil {
			theparent = make(map[string] interface{})
		} else if theparent, ok = parent.(map[string] interface{}); !ok {
			if theparent, ok = parent.([]interface{}); !ok {
				debug("Error braceopen - parent is not a map/list/nil")
				return nil, fmt.Errorf("Invalid {, parent not nil|map|list")
			}
		}
		res, err := p.parse(nil, theparent)
		if err != nil {
			debug("Error parsing brace", err)
		}
		return res, err

	case BRACECLOSE:
		// map finished
		return parent, nil

	case BRACKETOPEN:
		thelist := make([]interface{}, 0, 32)
		res, err := p.parselist(nil, thelist)
		return res, err

	case BRACKETCLOSE:
		// list finished
		return parent, nil
	}

	return nil, nil
}

func (p *Parser) Ucl() (map[string] interface{}, error) {
	p.parse(nil, p.ucl)

	if p.err == io.EOF {
		p.err = nil
	}
	return p.ucl, p.err
}
<|endoftext|>