<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

var (
	c = make(chan string, 100) // Allocate a channel.
)

func main() {
	// Connect to Database
	db, err := sql.Open("mysql", "root:@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// get first url to crawl
	c <- popToCrawlURL(db)

	for url := range c {
		crawl(db, url)

		// get next url to crawl
		c <- popToCrawlURL(db)
		time.Sleep(1 * time.Second) // should be a more polite value
	}

}

func crawl(db *sql.DB, url string) {

	fmt.Println("Trying to crawl: ", url)

	var s, err = getBody(url)
	if err != nil {
		return
	}

	// find links
	urlsFound := findLinks(s)

	for _, urlFound := range urlsFound {
		// normalize url
		urlFound, err := normalize(url, urlFound)
		if err != nil {
			fmt.Println(err)
			return
		}

		// insert into "to_crawl" table of db
		insertToCrawlURL(db, urlFound)
		fmt.Println("Found new url: ", urlFound)
	}
}

func getBody(url string) (string, error) {
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	return string(body), nil
}

func findLinks(s string) []string {
	var urlsFound []string

	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlFound := s[:end]
		urlsFound = append(urlsFound, urlFound)
	}
	return urlsFound
}

func popToCrawlURL(db *sql.DB) string {
	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT id, url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var id int
	var url string // we "scan" the result in here

	// Query the first element found
	err = stmtOut.QueryRow().Scan(&id, &url) // WHERE number = 13
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	// Prepare statement for deleting data
	stmtDel, err := db.Prepare("DELETE FROM to_crawl WHERE id = ?") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtDel.Close() // Close the statement when we leave main() / the program terminates,

	// Delete the element
	_, err = stmtDel.Exec(id)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	return url
}

func insertToCrawlURL(db *sql.DB, url string) {
	// Connect to Database
	db, err := sql.Open("mysql", "root:@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO to_crawl (url) VALUES(?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(url) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Print(err)
	}

}

func normalize(urlStart, urlFound string) (string, error) {
	// Add http if protocol isn't set
	if len(urlFound) > 1 && urlFound[:2] == "//" {
		urlFound = "http:" + urlFound
	}
	// Set start url in front if it's not set
	if len(urlFound) > 1 && urlFound[:1] == "/" {
		urlFound = urlStart + urlFound
	}
	// only add http(s) links
	if len(urlFound) > 7 && urlFound[0:7] != "http://" {
		if len(urlFound) > 8 && urlFound[0:8] != "https://" {
			return "", errors.New("Protocol should be http(s)")
		}
	}
	return urlFound, nil
}
<commit_msg>add html body to db<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

var (
	c = make(chan string, 100) // Allocate a channel.
)

func main() {
	// Connect to Database
	db, err := sql.Open("mysql", "root:@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// get first url to crawl
	c <- popToCrawlURL(db)

	for url := range c {
		crawl(db, url)

		// get next url to crawl
		c <- popToCrawlURL(db)
		time.Sleep(1 * time.Second) // should be a more polite value
	}
}

func crawl(db *sql.DB, url string) {

	fmt.Println("Trying to crawl: ", url)

	var s, err = getBody(url)
	if err != nil {
		return
	}

	insertBodyToURL(db, url, s)

	// find links
	urlsFound := findLinks(s)

	for _, urlFound := range urlsFound {
		// normalize url
		urlFound, err := normalize(url, urlFound)
		if err != nil {
			fmt.Println(err)
			return
		}

		// insert into "to_crawl" table of db
		insertToCrawlURL(db, urlFound)
		fmt.Println("Found new url: ", urlFound)
	}
}

func getBody(url string) (string, error) {
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	return string(body), nil
}

func findLinks(s string) []string {
	var urlsFound []string

	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlFound := s[:end]
		urlsFound = append(urlsFound, urlFound)
	}
	return urlsFound
}

func popToCrawlURL(db *sql.DB) string {

	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT id, url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var id int
	var url string // we "scan" the result in here

	// Query the first element found
	err = stmtOut.QueryRow().Scan(&id, &url) // WHERE number = 13
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	// Prepare statement for deleting data
	stmtDel, err := db.Prepare("DELETE FROM to_crawl WHERE id = ?") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtDel.Close() // Close the statement when we leave main() / the program terminates,

	// Delete the element
	_, err = stmtDel.Exec(id)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	return url
}

func insertToCrawlURL(db *sql.DB, url string) {
	// Connect to Database
	//db, err := sql.Open("mysql", "root:@/search")
	// if err != nil {
	// 	panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	// }
	// defer db.Close()

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO to_crawl (url) VALUES(?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(url) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Print(err)
	}

}

func insertBodyToURL(db *sql.DB, url, body string) {
	// Connect to Database
	//db, err := sql.Open("mysql", "root:@/search")
	// if err != nil {
	// 	panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	// }
	// defer db.Close()

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO urls (url, text) VALUES(?, ?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(url, body) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Print(err)
	}

}

func normalize(urlStart, urlFound string) (string, error) {
	// Add http if protocol isn't set
	if len(urlFound) > 1 && urlFound[:2] == "//" {
		urlFound = "http:" + urlFound
	}
	// Set start url in front if it's not set
	if len(urlFound) > 1 && urlFound[:1] == "/" {
		urlFound = urlStart + urlFound
	}
	// only add http(s) links
	if len(urlFound) > 7 && urlFound[0:7] != "http://" {
		if len(urlFound) > 8 && urlFound[0:8] != "https://" {
			return "", errors.New("Protocol should be http(s)")
		}
	}
	return urlFound, nil
}
<|endoftext|>


<commit_before>package css
// package main

import (
  "fmt"
	"strings"
	"rubex"
)

type Lexeme int

const (
  SPACES = iota
  COMMA
  UNIVERSAL
  TYPE
  ELEMENT
  CLASS
  ID
  LBRACKET
  RBRACKET
  ATTR_NAME
  ATTR_VALUE
  EQUALS
  CONTAINS_CLASS
  DASH_PREFIXED
  STARTS_WITH
  ENDS_WITH
  CONTAINS
  MATCH_OP
  PSEUDO_CLASS
  FIRST_CHILD
  FIRST_OF_TYPE
  NTH_CHILD
  NTH_OF_TYPE
  ONLY_CHILD
  ONLY_OF_TYPE
  LAST_CHILD
  LAST_OF_TYPE
  NOT
  LPAREN
  RPAREN
  COEFFICIENT
  SIGNED
  UNSIGNED
  ODD
  EVEN
  N
  OPERATOR
  PLUS
  MINUS
  BINOMIAL
  ADJACENT_TO
  PRECEDES
  PARENT_OF
  ANCESTOR_OF
  // and a counter ... I can't believe I didn't think of this sooner
  NUM_LEXEMES
)

var pattern [NUM_LEXEMES]string
var matcher [NUM_LEXEMES]*rubex.Regexp

type Scope int

const (
  GLOBAL = iota
  LOCAL
)

func Convert(css string, scope Scope) string {
  matchers := allocate()
  defer deallocate(matchers)
  xpath, _ := selectors(matchers, []byte(css), scope)
  return xpath
}

func allocate() []*rubex.Regexp {
  // some overlap in here, but it'll make the parsing functions clearer
  pattern[SPACES] = `\s+`
  pattern[COMMA] = `\s*,`
  pattern[UNIVERSAL] = `\*`
  pattern[TYPE] = `[_a-zA-Z]\w*`
  pattern[ELEMENT] = `(\*|[_a-zA-Z]\w*)`
  pattern[CLASS] = `\.[-\w]+`
  pattern[ID] = `\#[-\w]+`
  pattern[LBRACKET] = `\[`
  pattern[RBRACKET] = `\]`
  pattern[ATTR_NAME] = `[-_:a-zA-Z][-\w:.]*`
  pattern[ATTR_VALUE] = `("(\\.|[^"\\])*"|'(\\.|[^'\\])*')`
  pattern[EQUALS] = `=`
  pattern[CONTAINS_CLASS] = `~=`
  pattern[DASH_PREFIXED] = `\|=`
  pattern[STARTS_WITH] = `\^=`
  pattern[ENDS_WITH] = `\$=`
  pattern[CONTAINS] = `\*=`
  pattern[MATCH_OP] = "(" + strings.Join([]string{pattern[EQUALS], pattern[CONTAINS_CLASS], pattern[DASH_PREFIXED], pattern[STARTS_WITH], pattern[ENDS_WITH], pattern[CONTAINS]}, "|") + ")"
  pattern[PSEUDO_CLASS] = `:[-a-z]+`
  pattern[FIRST_CHILD] = `:first-child`
  pattern[FIRST_OF_TYPE] = `:first-of-type`
  pattern[NTH_CHILD] = `:nth-child`
  pattern[NTH_OF_TYPE] = `:nth-of-type`
  pattern[ONLY_CHILD] = `:only-child`
  pattern[ONLY_OF_TYPE] = `:only-of-type`
  pattern[LAST_CHILD] = `:last-child`
  pattern[LAST_OF_TYPE] = `:last-of-type`
  pattern[NOT] = `:not`
  pattern[LPAREN] = `\s*\(`
  pattern[RPAREN] = `\s*\)`
  pattern[COEFFICIENT] = `[-+]?(\d+)?`
  pattern[SIGNED] = `[-+]?\d+`
  pattern[UNSIGNED] = `\d+`
  pattern[ODD] = `odd`
  pattern[EVEN] = `even`
  pattern[N] = `[nN]`
  pattern[OPERATOR] = `[-+]`
  pattern[PLUS] = `\+`
  pattern[MINUS] = `-`
  pattern[BINOMIAL] = strings.Join([]string{pattern[COEFFICIENT], pattern[N], `\s*`, pattern[OPERATOR], `\s*`, pattern[UNSIGNED]}, "")
  pattern[ADJACENT_TO] = `\s*\+`
  pattern[PRECEDES] = `\s*~`
  pattern[PARENT_OF] = `\s*>`
  pattern[ANCESTOR_OF] = `\s+`
  matchers := make([]*rubex.Regexp, 0, NUM_LEXEMES)
  for _, p := range pattern {
    matchers = append(matchers, rubex.MustCompile(`\A` + p))
  }
  return matchers
}

func deallocate(matchers []*rubex.Regexp) {
  for _, m := range matchers {
    m.Free()
  }
}

func selectors(matchers []*rubex.Regexp, input []byte, scope Scope) (string, []byte) {
  x, input := selector(matchers, input, scope)
  xs := []string{x}
  for peek(matchers, COMMA, input) {
    _, input = token(matchers, COMMA, input)
    x, input = selector(matchers, input, scope)
    xs = append(xs, x)
  }
  return strings.Join(xs, " | "), input
}

func selector(matchers []*rubex.Regexp, input []byte, scope Scope) (string, []byte) {
  var combinator Lexeme
  var xs []string
  if scope == LOCAL {
    xs = []string{"."}
  }
  if matched, remainder := token(matchers, PARENT_OF, input); matched != nil {
    combinator, input = PARENT_OF, remainder
  } else {
    combinator = ANCESTOR_OF
  }
  x, input := sequence(matchers, input, combinator)
  xs = append(xs, x)
  for {
    if matched, remainder := token(matchers, ADJACENT_TO, input); matched != nil {
      combinator, input = ADJACENT_TO, remainder
    } else if matched, remainder := token(matchers, PRECEDES, input); matched != nil {
      combinator, input = PRECEDES, remainder
    } else if matched, remainder := token(matchers, PARENT_OF, input); matched != nil {
      combinator, input = PARENT_OF, remainder
    } else if matched, remainder := token(matchers, ANCESTOR_OF, input); matched != nil {
      combinator, input = ANCESTOR_OF, remainder
    } else {
      break
    }
    x, input = sequence(matchers, input, combinator)
    xs = append(xs, x)
  }
  return strings.Join(xs, ""), input
}

func sequence(matchers []*rubex.Regexp, input []byte, combinator Lexeme) (string, []byte) {
  _, input = token(matchers, SPACES, input)
  x, ps := "", []string{}

  switch combinator {
  case ANCESTOR_OF:
    x = "/descendant-or-self::*/*"
  case PARENT_OF:
    x = "/child::*"
  case PRECEDES:
    x = "/following-sibling::*"
  case ADJACENT_TO:
    x = "/following-sibling::*"
    ps = append(ps, "position()=1")
  }

  if e, remainder := token(matchers, ELEMENT, input); e != nil {
    input = remainder
    if len(ps) > 0 {
      ps = append(ps, " and ")
    }
    ps = append(ps, "self::"+string(e))
    if !(peek(matchers, ID, input) || peek(matchers, CLASS, input) || peek(matchers, PSEUDO_CLASS, input) || peek(matchers, LBRACKET, input)) {
      pstr := strings.Join(ps, "")
      if pstr != "" {
        pstr = fmt.Sprintf("[%s]", pstr)
      }
      return x + pstr, input
    }
  }
  q, input, connective := qualifier(matchers, input)
  if q == "" {
    panic("Invalid CSS selector")
  }
  if len(ps) > 0 {
    ps = append(ps, connective)
  }
  ps = append(ps, q)
  for q, r, c := qualifier(matchers, input); q != ""; q, r, c = qualifier(matchers, input) {
    ps, input = append(ps, c, q), r
  }
  pstr := strings.Join(ps, "")
  if combinator != NOT {
    pstr = fmt.Sprintf("[%s]", pstr)
  }
  return x + pstr, input
}

func qualifier(matchers []*rubex.Regexp, input []byte) (string, []byte, string) {
  p, connective := "", ""
  if t, remainder := token(matchers, CLASS, input); t != nil {
    p = fmt.Sprintf(`contains(concat(" ", @class, " "), " %s ")`, string(t[1:]))
    input = remainder
    connective = " and "
  } else if t, remainder := token(matchers, ID, input); t != nil {
    p, input, connective = fmt.Sprintf(`@id="%s"`, string(t[1:])), remainder, " and "
  } else if peek(matchers, PSEUDO_CLASS, input) {
    p, input, connective = pseudoClass(matchers, input)
  } else if peek(matchers, LBRACKET, input) {
    p, input = attribute(matchers, input)
    connective = " and "
  }
  return p, input, connective
}

func pseudoClass(matchers []*rubex.Regexp, input []byte) (string, []byte, string) {
  class, input := token(matchers, PSEUDO_CLASS, input)
  var p, connective string
  switch string(class) {
  case ":first-child":
    p, connective = "position()=1", " and "
  case ":first-of-type":
    p, connective = "position()=1", "]["
  case ":last-child":
    p, connective = "position()=last()", " and "
  case ":last-of-type":
    p, connective = "position()=last()", "]["
  case ":only-child":
    p, connective = "position() = 1 and position() = last()", " and "
  case ":only-of-type":
    p, connective = "position() = 1 and position() = last()", "]["
  case ":nth-child":
    p, input = nth(matchers, input)
    connective = " and "
  case ":nth-of-type":
    p, input = nth(matchers, input)
    connective = "]["
  case ":not":
    p, input = negate(matchers, input)
    connective = " and "
  default:
    panic(`Cannot convert CSS pseudo-class "` + string(class) + `" to XPath.`)
  }
  return p, input, connective
}

func nth(matchers []*rubex.Regexp, input []byte) (string, []byte) {
  lparen, input := token(matchers, LPAREN, input)
  if lparen == nil {
    panic(":nth-child and :nth-of-type require an parenthesized argument")
  }
  _, input = token(matchers, SPACES, input)
  var expr string
  if e, rem := token(matchers, EVEN, input); e != nil {
    expr, input = "position() mod 2 = 0", rem
  } else if e, rem := token(matchers, ODD, input); e != nil {
    expr, input = "position() mod 2 = 1", rem
  } else if e, _ := token(matchers, BINOMIAL, input); e != nil {
    var coefficient, operator, constant []byte
    coefficient, input = token(matchers, COEFFICIENT, input)
    switch string(coefficient) {
    case "", "+":
      coefficient = []byte("1")
    case "-":
      coefficient = []byte("-1")
    }
    _, input = token(matchers, N, input)
    _, input = token(matchers, SPACES, input)
    operator, input = token(matchers, OPERATOR, input)
    _, input = token(matchers, SPACES, input)
    constant, input = token(matchers, UNSIGNED, input)
    expr = fmt.Sprintf("(position() %s %s) mod %s = 0", invert(string(operator)), string(constant), string(coefficient))
  } else if e, rem := token(matchers, SIGNED, input); e != nil {
    expr, input = "position() = "+string(e), rem
  } else {
    panic("Invalid argument to :nth-child or :nth-of-type.")
  }
  fmt.Println(string(input))
  _, input = token(matchers, SPACES, input)
  rparen, input := token(matchers, RPAREN, input)
  if rparen == nil {
    panic("Unterminated argument to :nth-child or :nth-of-type.")
  }
  return expr, input
}

func invert(op string) string {
  op = strings.TrimSpace(op)
  if op == "+" {
    op = "-"
  } else {
    op = "+"
  }
  return op
}

func negate(matchers []*rubex.Regexp, input []byte) (string, []byte) {
  _, input = token(matchers, SPACES, input)
  lparen, input := token(matchers, LPAREN, input)
  if lparen == nil {
    panic(":not requires a parenthesized argument.")
  }
  _, input = token(matchers, SPACES, input)
  p, input := sequence(matchers, input, NOT)
  _, input = token(matchers, SPACES, input)
  rparen, input := token(matchers, RPAREN, input)
  if rparen == nil {
    panic("Unterminated argument to :not.")
  }
  return fmt.Sprintf("not(%s)", p), input
}

func attribute(matchers []*rubex.Regexp, input []byte) (string, []byte) {
  _, input = token(matchers, LBRACKET, input)
  _, input = token(matchers, SPACES, input)
  name, input := token(matchers, ATTR_NAME, input)
  if name == nil {
    panic("Attribute selector requires an attribute name.")
  }
  _, input = token(matchers, SPACES, input)
  if rbracket, remainder := token(matchers, RBRACKET, input); rbracket != nil {
    return "@" + string(name), remainder
  }
  op, input := token(matchers, MATCH_OP, input)
  if op == nil {
    panic("Missing operator in attribute selector.")
  }
  _, input = token(matchers, SPACES, input)
  val, input := token(matchers, ATTR_VALUE, input)
  if val == nil {
    panic("Missing value in attribute selector.")
  }
  _, input = token(matchers, SPACES, input)
  rbracket, input := token(matchers, RBRACKET, input)
  if rbracket == nil {
    panic("Unterminated attribute selector.")
  }
  var expr string
  n, v := string(name), string(val)
  switch string(op) {
  case "=":
    expr = fmt.Sprintf("@%s=%s", n, v)
  case "~=":
    expr = fmt.Sprintf(`contains(concat(" ", @%s, " "), concat(" ", %s, " "))`, n, v)
  case "|=":
    expr = fmt.Sprintf(`(@%s=%s or starts-with(@%s, concat(%s, "-")))`, n, v, n, v)
  case "^=":
    expr = fmt.Sprintf("starts-with(@%s, %s)", n, v)
  case "$=":
    // oy, libxml doesn't support ends-with
    // generate something like: div[substring(@class, string-length(@class) - string-length('foo') + 1) = 'foo']
    expr = fmt.Sprintf("substring(@%s, string-length(@%s) - string-length(%s) + 1) = %s", n, n, v, v)
  case "*=":
    expr = fmt.Sprintf("contains(@%s, %s)", n, v)
  }
  return expr, input
}

func token(matchers []*rubex.Regexp, lexeme Lexeme, input []byte) ([]byte, []byte) {
  matched := matchers[lexeme].Find(input)
  length := len(matched)
  if length == 0 {
    matched = nil
  }
  return matched, input[length:]
}

func peek(matchers []*rubex.Regexp, lexeme Lexeme, input []byte) bool {
  matched, _ := token(matchers, lexeme, input)
  return matched != nil
}
<commit_msg>remove unused global var<commit_after>package css
// package main

import (
  "fmt"
	"strings"
	"rubex"
)

type Lexeme int

const (
  SPACES = iota
  COMMA
  UNIVERSAL
  TYPE
  ELEMENT
  CLASS
  ID
  LBRACKET
  RBRACKET
  ATTR_NAME
  ATTR_VALUE
  EQUALS
  CONTAINS_CLASS
  DASH_PREFIXED
  STARTS_WITH
  ENDS_WITH
  CONTAINS
  MATCH_OP
  PSEUDO_CLASS
  FIRST_CHILD
  FIRST_OF_TYPE
  NTH_CHILD
  NTH_OF_TYPE
  ONLY_CHILD
  ONLY_OF_TYPE
  LAST_CHILD
  LAST_OF_TYPE
  NOT
  LPAREN
  RPAREN
  COEFFICIENT
  SIGNED
  UNSIGNED
  ODD
  EVEN
  N
  OPERATOR
  PLUS
  MINUS
  BINOMIAL
  ADJACENT_TO
  PRECEDES
  PARENT_OF
  ANCESTOR_OF
  // and a counter ... I can't believe I didn't think of this sooner
  NUM_LEXEMES
)

var pattern [NUM_LEXEMES]string

type Scope int

const (
  GLOBAL = iota
  LOCAL
)

func Convert(css string, scope Scope) string {
  matchers := allocate()
  defer deallocate(matchers)
  xpath, _ := selectors(matchers, []byte(css), scope)
  return xpath
}

func allocate() []*rubex.Regexp {
  // some overlap in here, but it'll make the parsing functions clearer
  pattern[SPACES] = `\s+`
  pattern[COMMA] = `\s*,`
  pattern[UNIVERSAL] = `\*`
  pattern[TYPE] = `[_a-zA-Z]\w*`
  pattern[ELEMENT] = `(\*|[_a-zA-Z]\w*)`
  pattern[CLASS] = `\.[-\w]+`
  pattern[ID] = `\#[-\w]+`
  pattern[LBRACKET] = `\[`
  pattern[RBRACKET] = `\]`
  pattern[ATTR_NAME] = `[-_:a-zA-Z][-\w:.]*`
  pattern[ATTR_VALUE] = `("(\\.|[^"\\])*"|'(\\.|[^'\\])*')`
  pattern[EQUALS] = `=`
  pattern[CONTAINS_CLASS] = `~=`
  pattern[DASH_PREFIXED] = `\|=`
  pattern[STARTS_WITH] = `\^=`
  pattern[ENDS_WITH] = `\$=`
  pattern[CONTAINS] = `\*=`
  pattern[MATCH_OP] = "(" + strings.Join([]string{pattern[EQUALS], pattern[CONTAINS_CLASS], pattern[DASH_PREFIXED], pattern[STARTS_WITH], pattern[ENDS_WITH], pattern[CONTAINS]}, "|") + ")"
  pattern[PSEUDO_CLASS] = `:[-a-z]+`
  pattern[FIRST_CHILD] = `:first-child`
  pattern[FIRST_OF_TYPE] = `:first-of-type`
  pattern[NTH_CHILD] = `:nth-child`
  pattern[NTH_OF_TYPE] = `:nth-of-type`
  pattern[ONLY_CHILD] = `:only-child`
  pattern[ONLY_OF_TYPE] = `:only-of-type`
  pattern[LAST_CHILD] = `:last-child`
  pattern[LAST_OF_TYPE] = `:last-of-type`
  pattern[NOT] = `:not`
  pattern[LPAREN] = `\s*\(`
  pattern[RPAREN] = `\s*\)`
  pattern[COEFFICIENT] = `[-+]?(\d+)?`
  pattern[SIGNED] = `[-+]?\d+`
  pattern[UNSIGNED] = `\d+`
  pattern[ODD] = `odd`
  pattern[EVEN] = `even`
  pattern[N] = `[nN]`
  pattern[OPERATOR] = `[-+]`
  pattern[PLUS] = `\+`
  pattern[MINUS] = `-`
  pattern[BINOMIAL] = strings.Join([]string{pattern[COEFFICIENT], pattern[N], `\s*`, pattern[OPERATOR], `\s*`, pattern[UNSIGNED]}, "")
  pattern[ADJACENT_TO] = `\s*\+`
  pattern[PRECEDES] = `\s*~`
  pattern[PARENT_OF] = `\s*>`
  pattern[ANCESTOR_OF] = `\s+`
  matchers := make([]*rubex.Regexp, 0, NUM_LEXEMES)
  for _, p := range pattern {
    matchers = append(matchers, rubex.MustCompile(`\A` + p))
  }
  return matchers
}

func deallocate(matchers []*rubex.Regexp) {
  for _, m := range matchers {
    m.Free()
  }
}

func selectors(matchers []*rubex.Regexp, input []byte, scope Scope) (string, []byte) {
  x, input := selector(matchers, input, scope)
  xs := []string{x}
  for peek(matchers, COMMA, input) {
    _, input = token(matchers, COMMA, input)
    x, input = selector(matchers, input, scope)
    xs = append(xs, x)
  }
  return strings.Join(xs, " | "), input
}

func selector(matchers []*rubex.Regexp, input []byte, scope Scope) (string, []byte) {
  var combinator Lexeme
  var xs []string
  if scope == LOCAL {
    xs = []string{"."}
  }
  if matched, remainder := token(matchers, PARENT_OF, input); matched != nil {
    combinator, input = PARENT_OF, remainder
  } else {
    combinator = ANCESTOR_OF
  }
  x, input := sequence(matchers, input, combinator)
  xs = append(xs, x)
  for {
    if matched, remainder := token(matchers, ADJACENT_TO, input); matched != nil {
      combinator, input = ADJACENT_TO, remainder
    } else if matched, remainder := token(matchers, PRECEDES, input); matched != nil {
      combinator, input = PRECEDES, remainder
    } else if matched, remainder := token(matchers, PARENT_OF, input); matched != nil {
      combinator, input = PARENT_OF, remainder
    } else if matched, remainder := token(matchers, ANCESTOR_OF, input); matched != nil {
      combinator, input = ANCESTOR_OF, remainder
    } else {
      break
    }
    x, input = sequence(matchers, input, combinator)
    xs = append(xs, x)
  }
  return strings.Join(xs, ""), input
}

func sequence(matchers []*rubex.Regexp, input []byte, combinator Lexeme) (string, []byte) {
  _, input = token(matchers, SPACES, input)
  x, ps := "", []string{}

  switch combinator {
  case ANCESTOR_OF:
    x = "/descendant-or-self::*/*"
  case PARENT_OF:
    x = "/child::*"
  case PRECEDES:
    x = "/following-sibling::*"
  case ADJACENT_TO:
    x = "/following-sibling::*"
    ps = append(ps, "position()=1")
  }

  if e, remainder := token(matchers, ELEMENT, input); e != nil {
    input = remainder
    if len(ps) > 0 {
      ps = append(ps, " and ")
    }
    ps = append(ps, "self::"+string(e))
    if !(peek(matchers, ID, input) || peek(matchers, CLASS, input) || peek(matchers, PSEUDO_CLASS, input) || peek(matchers, LBRACKET, input)) {
      pstr := strings.Join(ps, "")
      if pstr != "" {
        pstr = fmt.Sprintf("[%s]", pstr)
      }
      return x + pstr, input
    }
  }
  q, input, connective := qualifier(matchers, input)
  if q == "" {
    panic("Invalid CSS selector")
  }
  if len(ps) > 0 {
    ps = append(ps, connective)
  }
  ps = append(ps, q)
  for q, r, c := qualifier(matchers, input); q != ""; q, r, c = qualifier(matchers, input) {
    ps, input = append(ps, c, q), r
  }
  pstr := strings.Join(ps, "")
  if combinator != NOT {
    pstr = fmt.Sprintf("[%s]", pstr)
  }
  return x + pstr, input
}

func qualifier(matchers []*rubex.Regexp, input []byte) (string, []byte, string) {
  p, connective := "", ""
  if t, remainder := token(matchers, CLASS, input); t != nil {
    p = fmt.Sprintf(`contains(concat(" ", @class, " "), " %s ")`, string(t[1:]))
    input = remainder
    connective = " and "
  } else if t, remainder := token(matchers, ID, input); t != nil {
    p, input, connective = fmt.Sprintf(`@id="%s"`, string(t[1:])), remainder, " and "
  } else if peek(matchers, PSEUDO_CLASS, input) {
    p, input, connective = pseudoClass(matchers, input)
  } else if peek(matchers, LBRACKET, input) {
    p, input = attribute(matchers, input)
    connective = " and "
  }
  return p, input, connective
}

func pseudoClass(matchers []*rubex.Regexp, input []byte) (string, []byte, string) {
  class, input := token(matchers, PSEUDO_CLASS, input)
  var p, connective string
  switch string(class) {
  case ":first-child":
    p, connective = "position()=1", " and "
  case ":first-of-type":
    p, connective = "position()=1", "]["
  case ":last-child":
    p, connective = "position()=last()", " and "
  case ":last-of-type":
    p, connective = "position()=last()", "]["
  case ":only-child":
    p, connective = "position() = 1 and position() = last()", " and "
  case ":only-of-type":
    p, connective = "position() = 1 and position() = last()", "]["
  case ":nth-child":
    p, input = nth(matchers, input)
    connective = " and "
  case ":nth-of-type":
    p, input = nth(matchers, input)
    connective = "]["
  case ":not":
    p, input = negate(matchers, input)
    connective = " and "
  default:
    panic(`Cannot convert CSS pseudo-class "` + string(class) + `" to XPath.`)
  }
  return p, input, connective
}

func nth(matchers []*rubex.Regexp, input []byte) (string, []byte) {
  lparen, input := token(matchers, LPAREN, input)
  if lparen == nil {
    panic(":nth-child and :nth-of-type require an parenthesized argument")
  }
  _, input = token(matchers, SPACES, input)
  var expr string
  if e, rem := token(matchers, EVEN, input); e != nil {
    expr, input = "position() mod 2 = 0", rem
  } else if e, rem := token(matchers, ODD, input); e != nil {
    expr, input = "position() mod 2 = 1", rem
  } else if e, _ := token(matchers, BINOMIAL, input); e != nil {
    var coefficient, operator, constant []byte
    coefficient, input = token(matchers, COEFFICIENT, input)
    switch string(coefficient) {
    case "", "+":
      coefficient = []byte("1")
    case "-":
      coefficient = []byte("-1")
    }
    _, input = token(matchers, N, input)
    _, input = token(matchers, SPACES, input)
    operator, input = token(matchers, OPERATOR, input)
    _, input = token(matchers, SPACES, input)
    constant, input = token(matchers, UNSIGNED, input)
    expr = fmt.Sprintf("(position() %s %s) mod %s = 0", invert(string(operator)), string(constant), string(coefficient))
  } else if e, rem := token(matchers, SIGNED, input); e != nil {
    expr, input = "position() = "+string(e), rem
  } else {
    panic("Invalid argument to :nth-child or :nth-of-type.")
  }
  fmt.Println(string(input))
  _, input = token(matchers, SPACES, input)
  rparen, input := token(matchers, RPAREN, input)
  if rparen == nil {
    panic("Unterminated argument to :nth-child or :nth-of-type.")
  }
  return expr, input
}

func invert(op string) string {
  op = strings.TrimSpace(op)
  if op == "+" {
    op = "-"
  } else {
    op = "+"
  }
  return op
}

func negate(matchers []*rubex.Regexp, input []byte) (string, []byte) {
  _, input = token(matchers, SPACES, input)
  lparen, input := token(matchers, LPAREN, input)
  if lparen == nil {
    panic(":not requires a parenthesized argument.")
  }
  _, input = token(matchers, SPACES, input)
  p, input := sequence(matchers, input, NOT)
  _, input = token(matchers, SPACES, input)
  rparen, input := token(matchers, RPAREN, input)
  if rparen == nil {
    panic("Unterminated argument to :not.")
  }
  return fmt.Sprintf("not(%s)", p), input
}

func attribute(matchers []*rubex.Regexp, input []byte) (string, []byte) {
  _, input = token(matchers, LBRACKET, input)
  _, input = token(matchers, SPACES, input)
  name, input := token(matchers, ATTR_NAME, input)
  if name == nil {
    panic("Attribute selector requires an attribute name.")
  }
  _, input = token(matchers, SPACES, input)
  if rbracket, remainder := token(matchers, RBRACKET, input); rbracket != nil {
    return "@" + string(name), remainder
  }
  op, input := token(matchers, MATCH_OP, input)
  if op == nil {
    panic("Missing operator in attribute selector.")
  }
  _, input = token(matchers, SPACES, input)
  val, input := token(matchers, ATTR_VALUE, input)
  if val == nil {
    panic("Missing value in attribute selector.")
  }
  _, input = token(matchers, SPACES, input)
  rbracket, input := token(matchers, RBRACKET, input)
  if rbracket == nil {
    panic("Unterminated attribute selector.")
  }
  var expr string
  n, v := string(name), string(val)
  switch string(op) {
  case "=":
    expr = fmt.Sprintf("@%s=%s", n, v)
  case "~=":
    expr = fmt.Sprintf(`contains(concat(" ", @%s, " "), concat(" ", %s, " "))`, n, v)
  case "|=":
    expr = fmt.Sprintf(`(@%s=%s or starts-with(@%s, concat(%s, "-")))`, n, v, n, v)
  case "^=":
    expr = fmt.Sprintf("starts-with(@%s, %s)", n, v)
  case "$=":
    // oy, libxml doesn't support ends-with
    // generate something like: div[substring(@class, string-length(@class) - string-length('foo') + 1) = 'foo']
    expr = fmt.Sprintf("substring(@%s, string-length(@%s) - string-length(%s) + 1) = %s", n, n, v, v)
  case "*=":
    expr = fmt.Sprintf("contains(@%s, %s)", n, v)
  }
  return expr, input
}

func token(matchers []*rubex.Regexp, lexeme Lexeme, input []byte) ([]byte, []byte) {
  matched := matchers[lexeme].Find(input)
  length := len(matched)
  if length == 0 {
    matched = nil
  }
  return matched, input[length:]
}

func peek(matchers []*rubex.Regexp, lexeme Lexeme, input []byte) bool {
  matched, _ := token(matchers, lexeme, input)
  return matched != nil
}
<|endoftext|>


<commit_before>package ctx

import (
	"fmt"
	"net"
	"os"
	"os/user"
)

func Hostname() string {
	ensureLogLoaded()
	return conf.hostname
}

func LogLevel() string {
	ensureLogLoaded()
	return conf.logLevel
}

func Zones() map[string]string {
	ensureLogLoaded()
	return conf.zones
}

func Tunnels() map[string]string {
	ensureLogLoaded()
	return conf.tunnels
}

func KafkaHome() string {
	ensureLogLoaded()
	return conf.kafkaHome
}

func InfluxdbHost() string {
	ensureLogLoaded()
	return conf.influxdbHost
}

func SortedZones() []string {
	ensureLogLoaded()
	return conf.sortedZones()
}

func ZoneZkAddrs(zone string) (zkAddrs string) {
	ensureLogLoaded()
	var present bool
	if zkAddrs, present = conf.zones[zone]; present {
		return
	}

	// should never happen
	fmt.Printf("zone[%s] undefined", zone)
	os.Exit(1)
	return ""
}

// LocalIP tries to determine a non-loopback address for the local machine
func LocalIP() (net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, addr := range addrs {
		if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
			if ipnet.IP.To4() != nil || ipnet.IP.To16() != nil {
				return ipnet.IP, nil
			}
		}
	}
	return nil, nil
}

func CurrentUserIsRoot() bool {
	user, err := user.Current()
	if err != nil {
		return false
	}

	return user.Uid == "0"
}
<commit_msg>append LF on the prompt<commit_after>package ctx

import (
	"fmt"
	"net"
	"os"
	"os/user"
)

func Hostname() string {
	ensureLogLoaded()
	return conf.hostname
}

func LogLevel() string {
	ensureLogLoaded()
	return conf.logLevel
}

func Zones() map[string]string {
	ensureLogLoaded()
	return conf.zones
}

func Tunnels() map[string]string {
	ensureLogLoaded()
	return conf.tunnels
}

func KafkaHome() string {
	ensureLogLoaded()
	return conf.kafkaHome
}

func InfluxdbHost() string {
	ensureLogLoaded()
	return conf.influxdbHost
}

func SortedZones() []string {
	ensureLogLoaded()
	return conf.sortedZones()
}

func ZoneZkAddrs(zone string) (zkAddrs string) {
	ensureLogLoaded()
	var present bool
	if zkAddrs, present = conf.zones[zone]; present {
		return
	}

	// should never happen
	fmt.Printf("zone[%s] undefined\n", zone)
	os.Exit(1)
	return ""
}

// LocalIP tries to determine a non-loopback address for the local machine
func LocalIP() (net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, addr := range addrs {
		if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
			if ipnet.IP.To4() != nil || ipnet.IP.To16() != nil {
				return ipnet.IP, nil
			}
		}
	}
	return nil, nil
}

func CurrentUserIsRoot() bool {
	user, err := user.Current()
	if err != nil {
		return false
	}

	return user.Uid == "0"
}
<|endoftext|>


<commit_before>package dag

import (
	"fmt"
	"sort"
	"strings"
	"sync"

	"github.com/hashicorp/go-multierror"
)

// AcyclicGraph is a specialization of Graph that cannot have cycles. With
// this property, we get the property of sane graph traversal.
type AcyclicGraph struct {
	Graph
}

// WalkFunc is the callback used for walking the graph.
type WalkFunc func(Vertex) error

// DepthWalkFunc is a walk function that also receives the current depth of the
// walk as an argument
type DepthWalkFunc func(Vertex, int) error

// Returns a Set that includes every Vertex yielded by walking down from the
// provided starting Vertex v.
func (g *AcyclicGraph) Ancestors(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.DownEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.DepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Returns a Set that includes every Vertex yielded by walking up from the
// provided starting Vertex v.
func (g *AcyclicGraph) Descendents(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.UpEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.ReverseDepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Root returns the root of the DAG, or an error.
//
// Complexity: O(V)
func (g *AcyclicGraph) Root() (Vertex, error) {
	roots := make([]Vertex, 0, 1)
	for _, v := range g.Vertices() {
		if g.UpEdges(v).Len() == 0 {
			roots = append(roots, v)
		}
	}

	if len(roots) > 1 {
		// TODO(mitchellh): make this error message a lot better
		return nil, fmt.Errorf("multiple roots: %#v", roots)
	}

	if len(roots) == 0 {
		return nil, fmt.Errorf("no roots found")
	}

	return roots[0], nil
}

// TransitiveReduction performs the transitive reduction of graph g in place.
// The transitive reduction of a graph is a graph with as few edges as
// possible with the same reachability as the original graph. This means
// that if there are three nodes A => B => C, and A connects to both
// B and C, and B connects to C, then the transitive reduction is the
// same graph with only a single edge between A and B, and a single edge
// between B and C.
//
// The graph must be valid for this operation to behave properly. If
// Validate() returns an error, the behavior is undefined and the results
// will likely be unexpected.
//
// Complexity: O(V(V+E)), or asymptotically O(VE)
func (g *AcyclicGraph) TransitiveReduction() {
	// For each vertex u in graph g, do a DFS starting from each vertex
	// v such that the edge (u,v) exists (v is a direct descendant of u).
	//
	// For each v-prime reachable from v, remove the edge (u, v-prime).
	for _, u := range g.Vertices() {
		uTargets := g.DownEdges(u)
		vs := AsVertexList(g.DownEdges(u))

		g.DepthFirstWalk(vs, func(v Vertex, d int) error {
			shared := uTargets.Intersection(g.DownEdges(v))
			for _, vPrime := range AsVertexList(shared) {
				g.RemoveEdge(BasicEdge(u, vPrime))
			}

			return nil
		})
	}
}

// Validate validates the DAG. A DAG is valid if it has a single root
// with no cycles.
func (g *AcyclicGraph) Validate() error {
	if _, err := g.Root(); err != nil {
		return err
	}

	// Look for cycles of more than 1 component
	var err error
	cycles := g.Cycles()
	if len(cycles) > 0 {
		for _, cycle := range cycles {
			cycleStr := make([]string, len(cycle))
			for j, vertex := range cycle {
				cycleStr[j] = VertexName(vertex)
			}

			err = multierror.Append(err, fmt.Errorf(
				"Cycle: %s", strings.Join(cycleStr, ", ")))
		}
	}

	// Look for cycles to self
	for _, e := range g.Edges() {
		if e.Source() == e.Target() {
			err = multierror.Append(err, fmt.Errorf(
				"Self reference: %s", VertexName(e.Source())))
		}
	}

	return err
}

func (g *AcyclicGraph) Cycles() [][]Vertex {
	var cycles [][]Vertex
	for _, cycle := range StronglyConnected(&g.Graph) {
		if len(cycle) > 1 {
			cycles = append(cycles, cycle)
		}
	}
	return cycles
}

// Walk walks the graph, calling your callback as each node is visited.
// This will walk nodes in parallel if it can. Because the walk is done
// in parallel, the error returned will be a multierror.
func (g *AcyclicGraph) Walk(cb WalkFunc) error {
	// Cache the vertices since we use it multiple times
	vertices := g.Vertices()

	// Build the waitgroup that signals when we're done
	var wg sync.WaitGroup
	wg.Add(len(vertices))
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)
		wg.Wait()
	}()

	// The map of channels to watch to wait for vertices to finish
	vertMap := make(map[Vertex]chan struct{})
	for _, v := range vertices {
		vertMap[v] = make(chan struct{})
	}

	// The map of whether a vertex errored or not during the walk
	var errLock sync.Mutex
	var errs error
	errMap := make(map[Vertex]bool)
	for _, v := range vertices {
		// Build our list of dependencies and the list of channels to
		// wait on until we start executing for this vertex.
		deps := AsVertexList(g.DownEdges(v))
		depChs := make([]<-chan struct{}, len(deps))
		for i, dep := range deps {
			depChs[i] = vertMap[dep]
		}

		// Get our channel so that we can close it when we're done
		ourCh := vertMap[v]

		// Start the goroutine to wait for our dependencies
		readyCh := make(chan bool)
		go func(v Vertex, deps []Vertex, chs []<-chan struct{}, readyCh chan<- bool) {
			// First wait for all the dependencies
			for _, ch := range chs {
				<-ch
			}

			// Then, check the map to see if any of our dependencies failed
			errLock.Lock()
			defer errLock.Unlock()
			for _, dep := range deps {
				if errMap[dep] {
					errMap[v] = true
					readyCh <- false
					return
				}
			}

			readyCh <- true
		}(v, deps, depChs, readyCh)

		// Start the goroutine that executes
		go func(v Vertex, doneCh chan<- struct{}, readyCh <-chan bool) {
			defer close(doneCh)
			defer wg.Done()

			var err error
			if ready := <-readyCh; ready {
				err = cb(v)
			}

			errLock.Lock()
			defer errLock.Unlock()
			if err != nil {
				errMap[v] = true
				errs = multierror.Append(errs, err)
			}
		}(v, ourCh, readyCh)
	}

	<-doneCh
	return errs
}

// simple convenience helper for converting a dag.Set to a []Vertex
func AsVertexList(s *Set) []Vertex {
	rawList := s.List()
	vertexList := make([]Vertex, len(rawList))
	for i, raw := range rawList {
		vertexList[i] = raw.(Vertex)
	}
	return vertexList
}

type vertexAtDepth struct {
	Vertex Vertex
	Depth  int
}

// depthFirstWalk does a depth-first walk of the graph starting from
// the vertices in start. This is not exported now but it would make sense
// to export this publicly at some point.
func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.DownEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
// the vertices in start.
func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.UpEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// byVertexName implements sort.Interface so a list of Vertices can be sorted
// consistently by their VertexName
type byVertexName []Vertex

func (b byVertexName) Len() int      { return len(b) }
func (b byVertexName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byVertexName) Less(i, j int) bool {
	return VertexName(b[i]) < VertexName(b[j])
}
<commit_msg>core: log every 5s while waiting for dependencies<commit_after>package dag

import (
	"fmt"
	"log"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/go-multierror"
)

// AcyclicGraph is a specialization of Graph that cannot have cycles. With
// this property, we get the property of sane graph traversal.
type AcyclicGraph struct {
	Graph
}

// WalkFunc is the callback used for walking the graph.
type WalkFunc func(Vertex) error

// DepthWalkFunc is a walk function that also receives the current depth of the
// walk as an argument
type DepthWalkFunc func(Vertex, int) error

// Returns a Set that includes every Vertex yielded by walking down from the
// provided starting Vertex v.
func (g *AcyclicGraph) Ancestors(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.DownEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.DepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Returns a Set that includes every Vertex yielded by walking up from the
// provided starting Vertex v.
func (g *AcyclicGraph) Descendents(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.UpEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.ReverseDepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Root returns the root of the DAG, or an error.
//
// Complexity: O(V)
func (g *AcyclicGraph) Root() (Vertex, error) {
	roots := make([]Vertex, 0, 1)
	for _, v := range g.Vertices() {
		if g.UpEdges(v).Len() == 0 {
			roots = append(roots, v)
		}
	}

	if len(roots) > 1 {
		// TODO(mitchellh): make this error message a lot better
		return nil, fmt.Errorf("multiple roots: %#v", roots)
	}

	if len(roots) == 0 {
		return nil, fmt.Errorf("no roots found")
	}

	return roots[0], nil
}

// TransitiveReduction performs the transitive reduction of graph g in place.
// The transitive reduction of a graph is a graph with as few edges as
// possible with the same reachability as the original graph. This means
// that if there are three nodes A => B => C, and A connects to both
// B and C, and B connects to C, then the transitive reduction is the
// same graph with only a single edge between A and B, and a single edge
// between B and C.
//
// The graph must be valid for this operation to behave properly. If
// Validate() returns an error, the behavior is undefined and the results
// will likely be unexpected.
//
// Complexity: O(V(V+E)), or asymptotically O(VE)
func (g *AcyclicGraph) TransitiveReduction() {
	// For each vertex u in graph g, do a DFS starting from each vertex
	// v such that the edge (u,v) exists (v is a direct descendant of u).
	//
	// For each v-prime reachable from v, remove the edge (u, v-prime).
	for _, u := range g.Vertices() {
		uTargets := g.DownEdges(u)
		vs := AsVertexList(g.DownEdges(u))

		g.DepthFirstWalk(vs, func(v Vertex, d int) error {
			shared := uTargets.Intersection(g.DownEdges(v))
			for _, vPrime := range AsVertexList(shared) {
				g.RemoveEdge(BasicEdge(u, vPrime))
			}

			return nil
		})
	}
}

// Validate validates the DAG. A DAG is valid if it has a single root
// with no cycles.
func (g *AcyclicGraph) Validate() error {
	if _, err := g.Root(); err != nil {
		return err
	}

	// Look for cycles of more than 1 component
	var err error
	cycles := g.Cycles()
	if len(cycles) > 0 {
		for _, cycle := range cycles {
			cycleStr := make([]string, len(cycle))
			for j, vertex := range cycle {
				cycleStr[j] = VertexName(vertex)
			}

			err = multierror.Append(err, fmt.Errorf(
				"Cycle: %s", strings.Join(cycleStr, ", ")))
		}
	}

	// Look for cycles to self
	for _, e := range g.Edges() {
		if e.Source() == e.Target() {
			err = multierror.Append(err, fmt.Errorf(
				"Self reference: %s", VertexName(e.Source())))
		}
	}

	return err
}

func (g *AcyclicGraph) Cycles() [][]Vertex {
	var cycles [][]Vertex
	for _, cycle := range StronglyConnected(&g.Graph) {
		if len(cycle) > 1 {
			cycles = append(cycles, cycle)
		}
	}
	return cycles
}

// Walk walks the graph, calling your callback as each node is visited.
// This will walk nodes in parallel if it can. Because the walk is done
// in parallel, the error returned will be a multierror.
func (g *AcyclicGraph) Walk(cb WalkFunc) error {
	// Cache the vertices since we use it multiple times
	vertices := g.Vertices()

	// Build the waitgroup that signals when we're done
	var wg sync.WaitGroup
	wg.Add(len(vertices))
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)
		wg.Wait()
	}()

	// The map of channels to watch to wait for vertices to finish
	vertMap := make(map[Vertex]chan struct{})
	for _, v := range vertices {
		vertMap[v] = make(chan struct{})
	}

	// The map of whether a vertex errored or not during the walk
	var errLock sync.Mutex
	var errs error
	errMap := make(map[Vertex]bool)
	for _, v := range vertices {
		// Build our list of dependencies and the list of channels to
		// wait on until we start executing for this vertex.
		deps := AsVertexList(g.DownEdges(v))
		depChs := make([]<-chan struct{}, len(deps))
		for i, dep := range deps {
			depChs[i] = vertMap[dep]
		}

		// Get our channel so that we can close it when we're done
		ourCh := vertMap[v]

		// Start the goroutine to wait for our dependencies
		readyCh := make(chan bool)
		go func(v Vertex, deps []Vertex, chs []<-chan struct{}, readyCh chan<- bool) {
			// First wait for all the dependencies
			for i, ch := range chs {
			DepSatisfied:
				for {
					select {
					case <-ch:
						break DepSatisfied
					case <-time.After(time.Second * 5):
						log.Printf("[DEBUG] vertex %s, waiting for: %s",
							VertexName(v), VertexName(deps[i]))
					}
				}
				log.Printf("[DEBUG] vertex %s, got dep: %s",
					VertexName(v), VertexName(deps[i]))
			}

			// Then, check the map to see if any of our dependencies failed
			errLock.Lock()
			defer errLock.Unlock()
			for _, dep := range deps {
				if errMap[dep] {
					errMap[v] = true
					readyCh <- false
					return
				}
			}

			readyCh <- true
		}(v, deps, depChs, readyCh)

		// Start the goroutine that executes
		go func(v Vertex, doneCh chan<- struct{}, readyCh <-chan bool) {
			defer close(doneCh)
			defer wg.Done()

			var err error
			if ready := <-readyCh; ready {
				err = cb(v)
			}

			errLock.Lock()
			defer errLock.Unlock()
			if err != nil {
				errMap[v] = true
				errs = multierror.Append(errs, err)
			}
		}(v, ourCh, readyCh)
	}

	<-doneCh
	return errs
}

// simple convenience helper for converting a dag.Set to a []Vertex
func AsVertexList(s *Set) []Vertex {
	rawList := s.List()
	vertexList := make([]Vertex, len(rawList))
	for i, raw := range rawList {
		vertexList[i] = raw.(Vertex)
	}
	return vertexList
}

type vertexAtDepth struct {
	Vertex Vertex
	Depth  int
}

// depthFirstWalk does a depth-first walk of the graph starting from
// the vertices in start. This is not exported now but it would make sense
// to export this publicly at some point.
func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.DownEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
// the vertices in start.
func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.UpEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// byVertexName implements sort.Interface so a list of Vertices can be sorted
// consistently by their VertexName
type byVertexName []Vertex

func (b byVertexName) Len() int      { return len(b) }
func (b byVertexName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byVertexName) Less(i, j int) bool {
	return VertexName(b[i]) < VertexName(b[j])
}
<|endoftext|>


<commit_before>package db

import (
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	"github.com/jmoiron/sqlx/reflectx"
)

func init() {
	var err error

	db, err = sqlx.Open("mysql", getDSN())
	if err != nil {
		panic(err)
	}
	if err = db.Ping(); err != nil {
		panic(err)
	}

	db.SetMaxIdleConns(20)
	db.Mapper = reflectx.NewMapper("json")
}

func getDSN() string {
	return "chanxuehong:chanxuehong@tcp(gaowenbin.mysql.rds.aliyuncs.com:3306)/cxhtest?clientFoundRows=false&parseTime=true&loc=Asia%2FShanghai&timeout=5s&charset=utf8&collation=utf8_general_ci"
}
<commit_msg>no message<commit_after>package db

import (
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	"github.com/jmoiron/sqlx/reflectx"
)

func init() {
	var err error

	db, err = sqlx.Open("mysql", getDSN())
	if err != nil {
		panic(err)
	}
	if err = db.Ping(); err != nil {
		panic(err)
	}

	db.SetMaxIdleConns(20)
	db.Mapper = reflectx.NewMapper("json")
}

func getDSN() string {
	return "chanxuehong:chanxuehong@tcp(xxxxx:3306)/cxhtest?clientFoundRows=false&parseTime=true&loc=Asia%2FShanghai&timeout=5s&charset=utf8&collation=utf8_general_ci"
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"fmt"
	"time"
)

//
// SQL queries
//
var createPostTable string = `
CREATE TABLE IF NOT EXISTS Posts(
   id %s,
   authorId INTEGER,
   title VARCHAR(255),
   content TEXT,
   imageURL VARCHAR(255),
   date %s,
   FOREIGN KEY (authorId) REFERENCES Authors(id) ON DELETE SET NULL
)`

var dropPostTable string = `
DROP TABLE Posts;
`

var insertOrReplacePostForId string = `
INSERT OR REPLACE INTO Posts( authorId, title, content, imageURL, date)
VALUES( ?, ?, ?, ?, ?)`

var findPostById string = `
SELECT P.authorId, P.title, P.content, P.imageURL, P.date
FROM Posts AS P
WHERE P.id = ?`

var deletePostById string = `
DELETE FROM Posts
WHERE Posts.id = ?`

var queryForAllPost string = `
SELECT P.id, P.authorId, P.title, P.content, P.imageURL, P.date
FROM Posts AS P`

// Relations
var queryForAllCommentsOfPostId string = `
SELECT C.userId, C.postId, C.content, C.date, C.upVote, C.downVote
FROM Comments as C
WHERE C.postId = ?`

// Represents a post in the blog
type Post struct {
	id       int64
	authorId int64
	title    string
	content  string
	imageURL string
	date     time.Time
	db       Databaser
}

func (p *Post) Id() int64 {
	return p.id
}

func (p *Post) AuthorId() int64 {
	return p.authorId
}

func (p *Post) SetAuthorId(id int64) {
	p.authorId = id
}

func (p *Post) Title() string {
	return p.title
}

func (p *Post) SetTitle(title string) {
	p.title = title
}

func (p *Post) Content() string {
	return p.content
}

func (p *Post) SetContent(content string) {
	p.content = content
}

func (p *Post) ImageURL() string {
	return p.imageURL
}

func (p *Post) SetImageURL(imageURL string) {
	p.imageURL = imageURL
}

func (p *Post) Date() time.Time {
	return p.date
}

func (p *Post) SetDate(time time.Time) {
	p.date = time
}

func (p *Post) Comments() ([]Comment, error) {
	db, err := sql.Open(p.db.Driver(), p.db.Name())
	if err != nil {
		fmt.Println("Couldn't open DB:", err)
		return nil, err
	}
	defer db.Close()

	stmt, err := db.Prepare(queryForAllCommentsOfPostId)
	if err != nil {
		fmt.Printf("Couldn't prepare statement: %s", queryForAllPostsOfAuthorId)
		fmt.Println(err)
		return nil, err
	}
	defer stmt.Close()

	var comments []Comment

	rows, err := stmt.Query(p.id)
	if err != nil {
		fmt.Println("Couldn't read rows from statement", err)
		return comments, err
	}
	defer rows.Close()

	for rows.Next() {
		var id int64
		var userId int64
		var postId int64
		var content string
		var date time.Time
		var upVote int64
		var downVote int64
		rows.Scan(&id, &userId, &postId, &content, &date, &upVote, &downVote)
		c := Comment{
			id:       id,
			userId:   userId,
			postId:   postId,
			content:  content,
			date:     date,
			upVote:   upVote,
			downVote: downVote,
			db:       p.db,
		}
		comments = append(comments, c)
	}

	return comments, nil
}

//
// SQL stuff
//

//
// Post-specific operations on Persister
//

// Create the table Post in the database interface
func (persist *Persister) createPostTable() {

	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("Error on open of database", err)
		return
	}
	defer db.Close()

	var query = fmt.Sprintf(
		createPostTable,
		dbaser.IncrementPrimaryKey(),
		dbaser.DateField())

	_, err = db.Exec(query)
	if err != nil {
		fmt.Printf("Error creating Posts table, driver \"%s\", dbname \"%s\", query = \"%s\"\n",
			dbaser.Driver(), dbaser.Name(), query)
		fmt.Println(err)
		return
	}
}

func (persist *Persister) dropPostTable() {
	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("Error on open of database", err)
		return
	}
	defer db.Close()

	_, err = db.Exec(dropPostTable)
	if err != nil {
		fmt.Println("Error droping table:", err)
	}

}

// Creates a new Post attached to the Database (but not saved)
func (persist *Persister) NewPost(authorId int64, title string, content string, imageURL string, date time.Time) *Post {

	return &Post{
		id:       -1,
		authorId: authorId,
		title:    title,
		content:  content,
		imageURL: imageURL,
		date:     date,
		db:       persist.databaser,
	}
}

// Finds all the posts in the database
func (persist *Persister) FindAllPosts() ([]Post, error) {

	var posts []Post
	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("FindAllPosts 1:", err)
		return posts, err
	}
	defer db.Close()

	rows, err := db.Query(queryForAllPost)
	if err != nil {
		fmt.Println("FindAllPosts 2:", err)
		return posts, err
	}
	defer rows.Close()

	for rows.Next() {
		var id int64
		var authorId int64
		var title string
		var content string
		var imageURL string
		var date time.Time
		rows.Scan(&id, &authorId, &title, &content, &imageURL, &date)
		p := Post{
			id:       id,
			authorId: authorId,
			title:    title,
			content:  content,
			imageURL: imageURL,
			date:     date,
			db:       dbaser,
		}
		posts = append(posts, p)
	}

	return posts, nil
}

// Finds a post that matches the given id
func (persist *Persister) FindPostById(id int64) (*Post, error) {

	var p *Post
	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("FindPostById 1:", err)
		return p, err
	}
	defer db.Close()

	stmt, err := db.Prepare(findPostById)
	if err != nil {
		fmt.Println("FindPostById 2:", err)
		return p, err
	}
	defer stmt.Close()

	var authorId int64
	var title string
	var content string
	var imageURL string
	var date time.Time
	err = stmt.QueryRow(id).Scan(&authorId, &title, &content, &imageURL, &date)
	if err != nil {
		// normal if the post doesnt exist
		return p, err
	}

	p = &Post{
		id:       id,
		authorId: authorId,
		title:    title,
		content:  content,
		imageURL: imageURL,
		date:     date,
		db:       dbaser,
	}

	return p, nil
}

//
// Operations on Post
//

// Saves the post (or update it if it already exists)
// to the database
func (p *Post) Save() error {
	db, err := sql.Open(p.db.Driver(), p.db.Name())
	if err != nil {
		fmt.Println("Save 1:", err)
		return err
	}
	defer db.Close()

	stmt, err := db.Prepare(insertOrReplacePostForId)
	if err != nil {
		fmt.Println("Save 2:", err)
		return err
	}
	defer stmt.Close()

	res, err := stmt.Exec(p.authorId, p.title, p.content, p.imageURL, p.date)
	if err != nil {
		fmt.Println("Save 3:", err)
		return err
	}

	p.id, _ = res.LastInsertId()
	return nil
}

// Deletes the post from the database
func (p *Post) Destroy() error {

	db, err := sql.Open(p.db.Driver(), p.db.Name())
	if err != nil {
		fmt.Println("Destroy:", err)
		return err
	}
	defer db.Close()

	stmt, err := db.Prepare(deletePostById)
	if err != nil {
		fmt.Println("Destroy:", err)
		return err
	}
	defer stmt.Close()

	_, err = stmt.Exec(p.id)
	if err != nil {
		fmt.Println("Destroy:", err)
		return err
	}

	return nil
}
<commit_msg>Fix wrong error message<commit_after>package db

import (
	"database/sql"
	"fmt"
	"time"
)

//
// SQL queries
//
var createPostTable string = `
CREATE TABLE IF NOT EXISTS Posts(
   id %s,
   authorId INTEGER,
   title VARCHAR(255),
   content TEXT,
   imageURL VARCHAR(255),
   date %s,
   FOREIGN KEY (authorId) REFERENCES Authors(id) ON DELETE SET NULL
)`

var dropPostTable string = `
DROP TABLE Posts;
`

var insertOrReplacePostForId string = `
INSERT OR REPLACE INTO Posts( authorId, title, content, imageURL, date)
VALUES( ?, ?, ?, ?, ?)`

var findPostById string = `
SELECT P.authorId, P.title, P.content, P.imageURL, P.date
FROM Posts AS P
WHERE P.id = ?`

var deletePostById string = `
DELETE FROM Posts
WHERE Posts.id = ?`

var queryForAllPost string = `
SELECT P.id, P.authorId, P.title, P.content, P.imageURL, P.date
FROM Posts AS P`

// Relations
var queryForAllCommentsOfPostId string = `
SELECT C.userId, C.postId, C.content, C.date, C.upVote, C.downVote
FROM Comments as C
WHERE C.postId = ?`

// Represents a post in the blog
type Post struct {
	id       int64
	authorId int64
	title    string
	content  string
	imageURL string
	date     time.Time
	db       Databaser
}

func (p *Post) Id() int64 {
	return p.id
}

func (p *Post) AuthorId() int64 {
	return p.authorId
}

func (p *Post) SetAuthorId(id int64) {
	p.authorId = id
}

func (p *Post) Title() string {
	return p.title
}

func (p *Post) SetTitle(title string) {
	p.title = title
}

func (p *Post) Content() string {
	return p.content
}

func (p *Post) SetContent(content string) {
	p.content = content
}

func (p *Post) ImageURL() string {
	return p.imageURL
}

func (p *Post) SetImageURL(imageURL string) {
	p.imageURL = imageURL
}

func (p *Post) Date() time.Time {
	return p.date
}

func (p *Post) SetDate(time time.Time) {
	p.date = time
}

func (p *Post) Comments() ([]Comment, error) {
	db, err := sql.Open(p.db.Driver(), p.db.Name())
	if err != nil {
		fmt.Println("Couldn't open DB:", err)
		return nil, err
	}
	defer db.Close()

	stmt, err := db.Prepare(queryForAllCommentsOfPostId)
	if err != nil {
		fmt.Printf("Couldn't prepare statement: %s", queryForAllCommentsOfPostId)
		fmt.Println(err)
		return nil, err
	}
	defer stmt.Close()

	var comments []Comment

	rows, err := stmt.Query(p.id)
	if err != nil {
		fmt.Println("Couldn't read rows from statement", err)
		return comments, err
	}
	defer rows.Close()

	for rows.Next() {
		var id int64
		var userId int64
		var postId int64
		var content string
		var date time.Time
		var upVote int64
		var downVote int64
		rows.Scan(&id, &userId, &postId, &content, &date, &upVote, &downVote)
		c := Comment{
			id:       id,
			userId:   userId,
			postId:   postId,
			content:  content,
			date:     date,
			upVote:   upVote,
			downVote: downVote,
			db:       p.db,
		}
		comments = append(comments, c)
	}

	return comments, nil
}

//
// SQL stuff
//

//
// Post-specific operations on Persister
//

// Create the table Post in the database interface
func (persist *Persister) createPostTable() {

	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("Error on open of database", err)
		return
	}
	defer db.Close()

	var query = fmt.Sprintf(
		createPostTable,
		dbaser.IncrementPrimaryKey(),
		dbaser.DateField())

	_, err = db.Exec(query)
	if err != nil {
		fmt.Printf("Error creating Posts table, driver \"%s\", dbname \"%s\", query = \"%s\"\n",
			dbaser.Driver(), dbaser.Name(), query)
		fmt.Println(err)
		return
	}
}

func (persist *Persister) dropPostTable() {
	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("Error on open of database", err)
		return
	}
	defer db.Close()

	_, err = db.Exec(dropPostTable)
	if err != nil {
		fmt.Println("Error droping table:", err)
	}

}

// Creates a new Post attached to the Database (but not saved)
func (persist *Persister) NewPost(authorId int64, title string, content string, imageURL string, date time.Time) *Post {

	return &Post{
		id:       -1,
		authorId: authorId,
		title:    title,
		content:  content,
		imageURL: imageURL,
		date:     date,
		db:       persist.databaser,
	}
}

// Finds all the posts in the database
func (persist *Persister) FindAllPosts() ([]Post, error) {

	var posts []Post
	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("FindAllPosts 1:", err)
		return posts, err
	}
	defer db.Close()

	rows, err := db.Query(queryForAllPost)
	if err != nil {
		fmt.Println("FindAllPosts 2:", err)
		return posts, err
	}
	defer rows.Close()

	for rows.Next() {
		var id int64
		var authorId int64
		var title string
		var content string
		var imageURL string
		var date time.Time
		rows.Scan(&id, &authorId, &title, &content, &imageURL, &date)
		p := Post{
			id:       id,
			authorId: authorId,
			title:    title,
			content:  content,
			imageURL: imageURL,
			date:     date,
			db:       dbaser,
		}
		posts = append(posts, p)
	}

	return posts, nil
}

// Finds a post that matches the given id
func (persist *Persister) FindPostById(id int64) (*Post, error) {

	var p *Post
	var dbaser = persist.databaser

	db, err := sql.Open(dbaser.Driver(), dbaser.Name())
	if err != nil {
		fmt.Println("FindPostById 1:", err)
		return p, err
	}
	defer db.Close()

	stmt, err := db.Prepare(findPostById)
	if err != nil {
		fmt.Println("FindPostById 2:", err)
		return p, err
	}
	defer stmt.Close()

	var authorId int64
	var title string
	var content string
	var imageURL string
	var date time.Time
	err = stmt.QueryRow(id).Scan(&authorId, &title, &content, &imageURL, &date)
	if err != nil {
		// normal if the post doesnt exist
		return p, err
	}

	p = &Post{
		id:       id,
		authorId: authorId,
		title:    title,
		content:  content,
		imageURL: imageURL,
		date:     date,
		db:       dbaser,
	}

	return p, nil
}

//
// Operations on Post
//

// Saves the post (or update it if it already exists)
// to the database
func (p *Post) Save() error {
	db, err := sql.Open(p.db.Driver(), p.db.Name())
	if err != nil {
		fmt.Println("Save 1:", err)
		return err
	}
	defer db.Close()

	stmt, err := db.Prepare(insertOrReplacePostForId)
	if err != nil {
		fmt.Println("Save 2:", err)
		return err
	}
	defer stmt.Close()

	res, err := stmt.Exec(p.authorId, p.title, p.content, p.imageURL, p.date)
	if err != nil {
		fmt.Println("Save 3:", err)
		return err
	}

	p.id, _ = res.LastInsertId()
	return nil
}

// Deletes the post from the database
func (p *Post) Destroy() error {

	db, err := sql.Open(p.db.Driver(), p.db.Name())
	if err != nil {
		fmt.Println("Destroy:", err)
		return err
	}
	defer db.Close()

	stmt, err := db.Prepare(deletePostById)
	if err != nil {
		fmt.Println("Destroy:", err)
		return err
	}
	defer stmt.Close()

	_, err = stmt.Exec(p.id)
	if err != nil {
		fmt.Println("Destroy:", err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"v.io/lib/cmdline"
	"v.io/tools/lib/collect"
)

// cmdIntegration represents the "v23 integration" command
var cmdIntegration = &cmdline.Command{
	Name:     "integration",
	Short:    "Manage vanadium integration test support",
	Long:     "Manage vanadium integration test support",
	Children: []*cmdline.Command{cmdIntegrationGenerate},
}

var cmdIntegrationGenerate = &cmdline.Command{
	Run:   runIntegrationGenerate,
	Name:  "generate",
	Short: "Generates supporting code for vanadium tests.",
	Long: `
The v23 integration subcommand supports the vanadium integration test
framework and unit tests by generating go files that contain supporting
code. v23 integration generate is intended to be invoked via the
'go generate' mechanism and the resulting files are to be checked in.

Integration tests are functions of the form shown below that are defined
in 'external' tests (i.e. those occurring in _test packages, rather than
being part of the package being tested). This ensures that integration
tests are isolated from the packages being tested and can be moved to their
own package if need be. Integration tests have the following form:

    func V23Test<x> (i integration.T)

    'v23 integration generate' operates as follows:

In addition, some commonly used functionality in vanadium unit tests
is streamlined. Arguably this should be in a separate command/file but
for now they are lumped together. The additional functionality is as
follows:

1. v.io/veyron/lib/modules requires the use of an explicit
   registration mechanism. 'v23 integration generate' automatically
   generates these registration functions for any test function matches
   the modules.Main signature.

   For:
   // SubProc does the following...
   // Usage: <a> <b>...
   func SubProc(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error

   It will generate:

   modules.RegisterChild("SubProc",` + "`" + `SubProc does the following...
Usage: <a> <b>...` + "`" + `, SubProc)

2. 'TestMain' is used as the entry point for all vanadium tests, integration
   and otherwise. v23 will generate an appropriate version of this if one is
   not already defined. TestMain is 'special' in that only one definiton can
   occur across both the internal and external test packages. This is a
   consequence of how the go testing system is implemented.
`,

	// TODO(cnicolaou): once the initial deployment is done, revisit the
	// this functionality and possibly dissallow the 'if this doesn't exist
	// generate it' behaviour and instead always generate the required helper
	// functions.

	ArgsName: "[packages]",
	ArgsLong: "list of go packages"}

var (
	outputFileName string
)

func init() {
	cmdIntegrationGenerate.Flags.StringVar(&outputFileName, "output", "v23_test.go", "name of output files; two files are generated, <file_name> and internal_<file_name>.")
}

func runIntegrationGenerate(command *cmdline.Command, args []string) error {
	// TODO(cnicolaou): use http://godoc.org/golang.org/x/tools/go/loader
	// to replace accessing the AST directly. In the meantime make sure
	// the command line API is consistent with that change.

	if len(args) > 1 || (len(args) == 1 && args[0] != ".") {
		return command.UsageErrorf("unexpected or wrong arguments, currently only . is supported as a package name.")
	}
	fi, err := ioutil.ReadDir(".")
	if err != nil {
		return err
	}
	candidates := []string{}
	re := regexp.MustCompile(".*_test.go")
	for _, f := range fi {
		if f.IsDir() {
			continue
		}
		if re.MatchString(f.Name()) {
			candidates = append(candidates, f.Name())
		}
	}

	integrationTests := []string{}

	internalModules := []moduleCommand{}
	externalModules := []moduleCommand{}

	hasTestMain := false
	packageName := ""

	re = regexp.MustCompile(`V23Test(.*)`)
	fset := token.NewFileSet() // positions are relative to fset
	for _, file := range candidates {
		// Ignore the files we are generating.
		if file == outputFileName || file == "internal_"+outputFileName {
			continue
		}
		f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
		if err != nil {
			return err
		}

		// An external test package is one named <pkg>_test.
		isExternal := strings.HasSuffix(f.Name.Name, "_test")
		if !isExternal && len(packageName) == 0 {
			packageName = f.Name.Name
		}
		for _, d := range f.Decls {
			fn, ok := d.(*ast.FuncDecl)
			if !ok {
				continue
			}

			// If this function matches the declaration for modules.Main,
			// keep track of the names and comments associated with
			// such functions so that we can generate calls to
			// modules.RegisterChild for them.
			if n, c := isModulesMain(fn); len(n) > 0 {

				if isExternal {
					externalModules = append(externalModules, moduleCommand{n, c})
				} else {
					internalModules = append(internalModules, moduleCommand{n, c})

				}
			}

			// If this function is the testing TestMain then
			// keep track of the fact that we've seen it.
			if isTestMain(fn) {
				hasTestMain = true
			}
			name := fn.Name.String()
			if parts := re.FindStringSubmatch(name); isExternal && len(parts) == 2 {
				integrationTests = append(integrationTests, parts[1])
			}
		}
	}

	needInternalFile := len(internalModules) > 0
	needExternalFile := len(externalModules) > 0 || len(integrationTests) > 0

	// TestMain is special in that it can only occur once even across
	// internal and external test packages. If if it doesn't occur
	// in either, we want to make sure we write it out in the internal
	// package.
	if !hasTestMain && !needInternalFile && !needExternalFile {
		needInternalFile = true
	}

	if needInternalFile {
		if err := writeInternalFile("internal_"+outputFileName, packageName, !hasTestMain, internalModules); err != nil {
			return err
		}
		hasTestMain = true
	}

	if needExternalFile {
		return writeExternalFile(outputFileName, packageName, !hasTestMain, externalModules, integrationTests)
	}
	return nil
}

func isModulesMain(d ast.Decl) (string, string) {
	fn, ok := d.(*ast.FuncDecl)
	if !ok {
		return "", ""
	}

	if fn.Type == nil || fn.Type.Params == nil || fn.Type.Results == nil {
		return "", ""
	}
	name := fn.Name.Name

	typeNames := func(fl *ast.FieldList) []string {
		names := []string{}
		for _, f := range fl.List {
			switch v := f.Type.(type) {
			case *ast.Ident:
				names = append(names, v.Name)
			case *ast.SelectorExpr:
				// Deal with 'a, b type' parameters.
				for _, _ = range f.Names {
					if pkg, ok := v.X.(*ast.Ident); ok {
						names = append(names, pkg.Name+"."+v.Sel.Name)
					}
				}
			case *ast.MapType:
				if t, ok := v.Key.(*ast.Ident); !ok || t.Name != "string" {
					break
				}
				if t, ok := v.Value.(*ast.Ident); !ok || t.Name != "string" {
					break
				}
				names = append(names, "map[string]string")
			case *ast.Ellipsis:
				if t, ok := v.Elt.(*ast.Ident); !ok || t.Name != "string" {
					break
				}
				names = append(names, "...string")
			}
		}
		return names
	}

	cmp := func(a, b []string) bool {
		if len(a) != len(b) {
			return false
		}
		for i, av := range a {
			if av != b[i] {
				return false
			}
		}
		return true
	}

	comments := func(cg *ast.CommentGroup) string {
		if cg == nil {
			return ""
		}
		c := ""
		for _, l := range cg.List {
			t := strings.TrimPrefix(l.Text, "//")
			c += strings.TrimSpace(t) + "\n"
		}
		return strings.TrimSuffix(c, "\n")
	}

	// the Modules.Main signature is as follows:
	// type Main func(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error
	results := []string{"error"}
	parameters := []string{"io.Reader", "io.Writer", "io.Writer", "map[string]string", "...string"}
	_, _ = results, parameters

	p := typeNames(fn.Type.Params)
	r := typeNames(fn.Type.Results)

	if !cmp(results, r) || !cmp(parameters, p) {
		return "", ""
	}
	return name, comments(fn.Doc)
}

func isTestMain(fn *ast.FuncDecl) bool {
	// TODO(cnicolaou): check the signature as well as the name
	if fn.Name.Name != "TestMain" {
		return false
	}
	return true
}

type moduleCommand struct {
	name, comment string
}

// writeInternalFile writes a generated test file that is inside the package.
// It cannot contain integration tests.
func writeInternalFile(fileName string, packageName string, needsTestMain bool, modules []moduleCommand) (e error) {

	hasModules := len(modules) > 0

	if !needsTestMain && !hasModules {
		return nil
	}

	out, err := os.Create(fileName)
	if err != nil {
		return err
	}
	defer collect.Error(func() error { return out.Close() }, &e)

	fmt.Fprintln(out, "// This file was auto-generated via go generate.")
	fmt.Fprintln(out, "// DO NOT UPDATE MANUALLY")
	fmt.Fprintf(out, "package %s\n\n", packageName)

	if needsTestMain {
		fmt.Fprintln(out, `import "testing"`)
		if needsTestMain {
			fmt.Fprintln(out, `import "os"`)
		}
		fmt.Fprintln(out, "")
	}

	if hasModules {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/modules"`)
	}

	if needsTestMain {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/testutil"`)
	}

	if hasModules {
		fmt.Fprintln(out, "")
		fmt.Fprintln(out, "func init() {")
		writeModuleRegistration(out, modules)
		fmt.Fprintln(out, "}")
	}

	if needsTestMain {
		writeTestMain(out)
	}
	return nil
}

// writeExternalFile write a generated test file that is outside the package.
// It can contain intgreation tests.
func writeExternalFile(fileName string, packageName string, needsTestMain bool, modules []moduleCommand, tests []string) (e error) {

	hasTests := len(tests) > 0
	hasModules := len(modules) > 0
	if !needsTestMain && !hasModules && !hasTests {
		return nil
	}

	out, err := os.Create(fileName)
	if err != nil {
		return err
	}
	defer collect.Error(func() error { return out.Close() }, &e)

	fmt.Fprintln(out, "// This file was auto-generated via go generate.")
	fmt.Fprintln(out, "// DO NOT UPDATE MANUALLY")
	fmt.Fprintf(out, "package %s_test\n\n", packageName)

	if needsTestMain {
		fmt.Fprintln(out, `import "testing"`)
		if needsTestMain {
			fmt.Fprintln(out, `import "os"`)
		}
		fmt.Fprintln(out, "")
	}

	if hasModules {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/modules"`)
	}

	if needsTestMain {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/testutil"`)
	}

	if hasTests {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/testutil/integration"`)
	}

	if hasModules {
		fmt.Fprintln(out, "")
		fmt.Fprintln(out, "func init() {")
		writeModuleRegistration(out, modules)
		fmt.Fprintln(out, "}")
	}

	if needsTestMain {
		writeTestMain(out)
	}

	// integration test wrappers.
	for _, t := range tests {
		fmt.Fprintf(out, "\nfunc TestV23%s(t *testing.T) {\n", t)
		fmt.Fprintf(out, "\tintegration.RunTest(t, V23Test%s)\n}\n", t)
	}
	return nil
}

func writeTestMain(out io.Writer) {
	fmt.Fprintf(out, `
func TestMain(m *testing.M) {
	testutil.Init()
	// TODO(cnicolaou): call modules.Dispatch and remove the need for TestHelperProcess
	os.Exit(m.Run())
}
`)
}

func writeModuleRegistration(out io.Writer, modules []moduleCommand) {
	for _, m := range modules {
		fmt.Fprintf(out, "\tmodules.RegisterChild(%q, `%s`, %s)\n", m.name, m.comment, m.name)
	}
}
<commit_msg>tools/lib/testutil: add support for a new test 'vanadium-v23-tests'.<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"v.io/lib/cmdline"
	"v.io/tools/lib/collect"
)

// cmdIntegration represents the "v23 integration" command
var cmdIntegration = &cmdline.Command{
	Name:     "integration",
	Short:    "Manage vanadium integration test support",
	Long:     "Manage vanadium integration test support",
	Children: []*cmdline.Command{cmdIntegrationGenerate},
}

var cmdIntegrationGenerate = &cmdline.Command{
	Run:   runIntegrationGenerate,
	Name:  "generate",
	Short: "Generates supporting code for vanadium tests.",
	Long: `
The v23 integration subcommand supports the vanadium integration test
framework and unit tests by generating go files that contain supporting
code. v23 integration generate is intended to be invoked via the
'go generate' mechanism and the resulting files are to be checked in.

Integration tests are functions of the form shown below that are defined
in 'external' tests (i.e. those occurring in _test packages, rather than
being part of the package being tested). This ensures that integration
tests are isolated from the packages being tested and can be moved to their
own package if need be. Integration tests have the following form:

    func V23Test<x> (i integration.T)

    'v23 integration generate' operates as follows:

In addition, some commonly used functionality in vanadium unit tests
is streamlined. Arguably this should be in a separate command/file but
for now they are lumped together. The additional functionality is as
follows:

1. v.io/veyron/lib/modules requires the use of an explicit
   registration mechanism. 'v23 integration generate' automatically
   generates these registration functions for any test function matches
   the modules.Main signature.

   For:
   // SubProc does the following...
   // Usage: <a> <b>...
   func SubProc(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error

   It will generate:

   modules.RegisterChild("SubProc",` + "`" + `SubProc does the following...
Usage: <a> <b>...` + "`" + `, SubProc)

2. 'TestMain' is used as the entry point for all vanadium tests, integration
   and otherwise. v23 will generate an appropriate version of this if one is
   not already defined. TestMain is 'special' in that only one definiton can
   occur across both the internal and external test packages. This is a
   consequence of how the go testing system is implemented.
`,

	// TODO(cnicolaou): once the initial deployment is done, revisit the
	// this functionality and possibly dissallow the 'if this doesn't exist
	// generate it' behaviour and instead always generate the required helper
	// functions.

	ArgsName: "[packages]",
	ArgsLong: "list of go packages"}

var (
	outputFileName string
)

func init() {
	cmdIntegrationGenerate.Flags.StringVar(&outputFileName, "output", "v23_test.go", "name of output files; two files are generated, <file_name> and internal_<file_name>.")
}

func runIntegrationGenerate(command *cmdline.Command, args []string) error {
	// TODO(cnicolaou): use http://godoc.org/golang.org/x/tools/go/loader
	// to replace accessing the AST directly. In the meantime make sure
	// the command line API is consistent with that change.

	if len(args) > 1 || (len(args) == 1 && args[0] != ".") {
		return command.UsageErrorf("unexpected or wrong arguments, currently only . is supported as a package name.")
	}
	fi, err := ioutil.ReadDir(".")
	if err != nil {
		return err
	}
	candidates := []string{}
	re := regexp.MustCompile(".*_test.go")
	for _, f := range fi {
		if f.IsDir() {
			continue
		}
		if re.MatchString(f.Name()) {
			candidates = append(candidates, f.Name())
		}
	}

	integrationTests := []string{}

	internalModules := []moduleCommand{}
	externalModules := []moduleCommand{}

	hasTestMain := false
	packageName := ""

	re = regexp.MustCompile(`V23Test(.*)`)
	fset := token.NewFileSet() // positions are relative to fset
	for _, file := range candidates {
		// Ignore the files we are generating.
		if file == outputFileName || file == "internal_"+outputFileName {
			continue
		}
		f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
		if err != nil {
			return err
		}

		// An external test package is one named <pkg>_test.
		isExternal := strings.HasSuffix(f.Name.Name, "_test")
		if len(packageName) == 0 {
			packageName = strings.TrimSuffix(f.Name.Name, "_test")
		}
		for _, d := range f.Decls {
			fn, ok := d.(*ast.FuncDecl)
			if !ok {
				continue
			}

			// If this function matches the declaration for modules.Main,
			// keep track of the names and comments associated with
			// such functions so that we can generate calls to
			// modules.RegisterChild for them.
			if n, c := isModulesMain(fn); len(n) > 0 {

				if isExternal {
					externalModules = append(externalModules, moduleCommand{n, c})
				} else {
					internalModules = append(internalModules, moduleCommand{n, c})

				}
			}

			// If this function is the testing TestMain then
			// keep track of the fact that we've seen it.
			if isTestMain(fn) {
				hasTestMain = true
			}
			name := fn.Name.String()
			if parts := re.FindStringSubmatch(name); isExternal && len(parts) == 2 {
				integrationTests = append(integrationTests, parts[1])
			}
		}
	}

	needInternalFile := len(internalModules) > 0
	needExternalFile := len(externalModules) > 0 || len(integrationTests) > 0

	// TestMain is special in that it can only occur once even across
	// internal and external test packages. If if it doesn't occur
	// in either, we want to make sure we write it out in the internal
	// package.
	if !hasTestMain && !needInternalFile && !needExternalFile {
		needInternalFile = true
	}

	if needInternalFile {
		if err := writeInternalFile("internal_"+outputFileName, packageName, !hasTestMain, internalModules); err != nil {
			return err
		}
		hasTestMain = true
	}

	if needExternalFile {
		return writeExternalFile(outputFileName, packageName, !hasTestMain, externalModules, integrationTests)
	}
	return nil
}

func isModulesMain(d ast.Decl) (string, string) {
	fn, ok := d.(*ast.FuncDecl)
	if !ok {
		return "", ""
	}

	if fn.Type == nil || fn.Type.Params == nil || fn.Type.Results == nil {
		return "", ""
	}
	name := fn.Name.Name

	typeNames := func(fl *ast.FieldList) []string {
		names := []string{}
		for _, f := range fl.List {
			switch v := f.Type.(type) {
			case *ast.Ident:
				names = append(names, v.Name)
			case *ast.SelectorExpr:
				// Deal with 'a, b type' parameters.
				for _, _ = range f.Names {
					if pkg, ok := v.X.(*ast.Ident); ok {
						names = append(names, pkg.Name+"."+v.Sel.Name)
					}
				}
			case *ast.MapType:
				if t, ok := v.Key.(*ast.Ident); !ok || t.Name != "string" {
					break
				}
				if t, ok := v.Value.(*ast.Ident); !ok || t.Name != "string" {
					break
				}
				names = append(names, "map[string]string")
			case *ast.Ellipsis:
				if t, ok := v.Elt.(*ast.Ident); !ok || t.Name != "string" {
					break
				}
				names = append(names, "...string")
			}
		}
		return names
	}

	cmp := func(a, b []string) bool {
		if len(a) != len(b) {
			return false
		}
		for i, av := range a {
			if av != b[i] {
				return false
			}
		}
		return true
	}

	comments := func(cg *ast.CommentGroup) string {
		if cg == nil {
			return ""
		}
		c := ""
		for _, l := range cg.List {
			t := strings.TrimPrefix(l.Text, "//")
			c += strings.TrimSpace(t) + "\n"
		}
		return strings.TrimSuffix(c, "\n")
	}

	// the Modules.Main signature is as follows:
	// type Main func(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error
	results := []string{"error"}
	parameters := []string{"io.Reader", "io.Writer", "io.Writer", "map[string]string", "...string"}
	_, _ = results, parameters

	p := typeNames(fn.Type.Params)
	r := typeNames(fn.Type.Results)

	if !cmp(results, r) || !cmp(parameters, p) {
		return "", ""
	}
	return name, comments(fn.Doc)
}

func isTestMain(fn *ast.FuncDecl) bool {
	// TODO(cnicolaou): check the signature as well as the name
	if fn.Name.Name != "TestMain" {
		return false
	}
	return true
}

type moduleCommand struct {
	name, comment string
}

// writeInternalFile writes a generated test file that is inside the package.
// It cannot contain integration tests.
func writeInternalFile(fileName string, packageName string, needsTestMain bool, modules []moduleCommand) (e error) {

	hasModules := len(modules) > 0

	if !needsTestMain && !hasModules {
		return nil
	}

	out, err := os.Create(fileName)
	if err != nil {
		return err
	}
	defer collect.Error(func() error { return out.Close() }, &e)

	fmt.Fprintln(out, "// This file was auto-generated via go generate.")
	fmt.Fprintln(out, "// DO NOT UPDATE MANUALLY")
	fmt.Fprintf(out, "package %s\n\n", packageName)

	if needsTestMain {
		fmt.Fprintln(out, `import "testing"`)
		if needsTestMain {
			fmt.Fprintln(out, `import "os"`)
		}
		fmt.Fprintln(out, "")
	}

	if hasModules {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/modules"`)
	}

	if needsTestMain {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/testutil"`)
	}

	if hasModules {
		fmt.Fprintln(out, "")
		fmt.Fprintln(out, "func init() {")
		writeModuleRegistration(out, modules)
		fmt.Fprintln(out, "}")
	}

	if needsTestMain {
		writeTestMain(out)
	}
	return nil
}

// writeExternalFile write a generated test file that is outside the package.
// It can contain intgreation tests.
func writeExternalFile(fileName string, packageName string, needsTestMain bool, modules []moduleCommand, tests []string) (e error) {

	hasTests := len(tests) > 0
	hasModules := len(modules) > 0
	if !needsTestMain && !hasModules && !hasTests {
		return nil
	}

	out, err := os.Create(fileName)
	if err != nil {
		return err
	}
	defer collect.Error(func() error { return out.Close() }, &e)

	fmt.Fprintln(out, "// This file was auto-generated via go generate.")
	fmt.Fprintln(out, "// DO NOT UPDATE MANUALLY")
	fmt.Fprintf(out, "package %s_test\n\n", packageName)

	if needsTestMain {
		fmt.Fprintln(out, `import "testing"`)
		if needsTestMain {
			fmt.Fprintln(out, `import "os"`)
		}
		fmt.Fprintln(out, "")
	}

	if hasModules {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/modules"`)
	}

	if needsTestMain {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/testutil"`)
	}

	if hasTests {
		fmt.Fprintln(out, `import "v.io/core/veyron/lib/testutil/integration"`)
	}

	if hasModules {
		fmt.Fprintln(out, "")
		fmt.Fprintln(out, "func init() {")
		writeModuleRegistration(out, modules)
		fmt.Fprintln(out, "}")
	}

	if needsTestMain {
		writeTestMain(out)
	}

	// integration test wrappers.
	for _, t := range tests {
		fmt.Fprintf(out, "\nfunc TestV23%s(t *testing.T) {\n", t)
		fmt.Fprintf(out, "\tintegration.RunTest(t, V23Test%s)\n}\n", t)
	}
	return nil
}

func writeTestMain(out io.Writer) {
	fmt.Fprintf(out, `
func TestMain(m *testing.M) {
	testutil.Init()
	// TODO(cnicolaou): call modules.Dispatch and remove the need for TestHelperProcess
	os.Exit(m.Run())
}
`)
}

func writeModuleRegistration(out io.Writer, modules []moduleCommand) {
	for _, m := range modules {
		fmt.Fprintf(out, "\tmodules.RegisterChild(%q, `%s`, %s)\n", m.name, m.comment, m.name)
	}
}
<|endoftext|>