remove hardcoded mappings now correctly in curie_map.yaml

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their @prefix (or literal object class)
#  Reduce predicates to the specific term identifier
#  and the ontology they are from. (curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
	print "usage: ntriple2dot.awk curie_map.yaml rdf.nt > rdf.dot"
}

##########################################################
# remove first and last chars from input string <>
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed local identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:|^https:\/\/monarchinitiative.org\/.well-known\/genid\//))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_*|_*$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	u = uri
	# shorten till longest uri in curi map is found (or not)
	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	############################################################
	# re-visit whether these exceptions are still necessary often
	# they may have been moved into the curie prefix mapping file
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
    	# in IMPC
	prefix["http://www.mousephenotype.org"]="IMPC"

    # http://www.mousephenotype.org/data/genes
	# in panther
	# prefix["http://identifiers.org/wormbase/"]="WormBase"

	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"

}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# optional whitespace, a colon, optional whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" spo[2] " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their @prefix (or literal object class)
#  Reduce predicates to the specific term identifier
#  and the ontology they are from. (curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology as opposed to
#  a data uri.

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
	print "usage: ntriple2dot.awk curie_map.yaml rdf.nt > rdf.dot"
}

##########################################################
# remove first and last chars from input string <>
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed local identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:|^https:\/\/monarchinitiative.org\/.well-known\/genid\//))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_*|_*$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	u = uri
	# shorten till longest uri in curi map is found (or not)
	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	# prefix["https://monarchinitiative.org/_"]="BNODE"
	# just until skolemized bnodes get in the curie map?
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	############################################################
	# revisit whether these exceptions are still necessary often
	# they may have been moved into the  prefix mapping file
	prefix["http://www.w3.org/1999/02/22-rdf-syntax-ns#"]="rdf"
	prefix["http://www.w3.org/2000/01/rdf-schema#"]="rdfs"
	prefix["http://www.w3.org/2002/07/owl#"]="owl"
	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
    # http://www.mousephenotype.org/data/genes
	# in panther
	prefix["http://identifiers.org/wormbase"]="WormBase"
	# in IMPC
	prefix["http://www.mousephenotype.org"]="IMPC"
	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"
	# till all non httpS: are purged
	prefix["http://monarchinitiative.org/"]="BASE"
	prefix["http://www.monarchinitiative.org/"]="BASE"
	prefix["http://monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://www.monarchinitiative.org/MONARCH"]=""
	prefix["http://www.monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://data.monarchinitiative.org/ttl"]="MonarchData"
	prefix["http://archive.monarchinitiative.org/ttl"]="MonarchArchive"
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# optional whitespace, a colon, optional whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" spo[2] " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}


un optimization

un optimization

# sameasFromNT
#

$2 == "<http://www.w3.org/2002/07/owl#sameAs>" {
   print
}
$2 == "<http://purl.org/dc/terms/isReferencedBy>" {
   print
}

# sameasFromNT
#

$2 == "<http://purl.org/dc/terms/isReferencedBy>" || $2 == "<http://www.w3.org/2002/07/owl#sameAs>" {
   print
}


PLTokenizer: header comment

PLTokenizer: header comment

####################################################################
# PLTokenizer.awk                                                  #
####################################################################

# Tokenize a string.
function plTokenize(returnTokens, string,
                    delimiters,
                    newlines,
                    quotes,
                    escapeChars,
                    leftBlockComments,
                    rightBlockComments,
                    lineComments,
                    reservedOperators,
                    reservedPatterns,
                    ####
                    blockCommenting,
                    c,
                    currentToken,
                    escaping,
                    i,
                    lineCommenting,
                    p,
                    quoting,
                    r,
                    s,
                    tempGroup,
                    tempPattern,
                    tempString) {
    # Default parameters
    if (!delimiters[0]) {
        delimiters[0] = " "  # whitespace
        delimiters[1] = "\t" # horizontal tab
        delimiters[2] = "\v" # vertical tab
    }
    if (!newlines[0]) {
        newlines[0] = "\n" # line feed
        newlines[1] = "\r" # carriage return
    }
    if (!quotes[0]) {
        quotes[0] = "\"" # double quote
    }
    if (!escapeChars[0]) {
        escapeChars[0] = "\\" # backslash
    }
    if (!leftBlockComments[0]) {
        leftBlockComments[0] = "#|" # Lisp-style extended comment (open)
        leftBlockComments[1] = "/*" # C-style comment (open)
        leftBlockComments[2] = "(*" # ML-style comment (open)
    }
    if (!rightBlockComments[0]) {
        rightBlockComments[0] = "|#" # Lisp-style extended comment (close)
        rightBlockComments[1] = "*/" # C-style comment (close)
        rightBlockComments[2] = "*)" # ML-style comment (close)
    }
    if (!lineComments[0]) {
        lineComments[0] = ";"  # Lisp-style line comment
        lineComments[1] = "//" # C++-style line comment
        lineComments[2] = "#"  # hash comment
    }
    if (!reservedOperators[0]) {
        reservedOperators[0] = "(" #  left parenthesis
        reservedOperators[1] = ")" # right parenthesis
        reservedOperators[2] = "[" #  left bracket
        reservedOperators[3] = "]" # right bracket
        reservedOperators[4] = "{" #  left brace
        reservedOperators[5] = "}" # right brace
        reservedOperators[6] = "," # comma
    }
    if (!reservedPatterns[0]) {
        reservedPatterns[0] = "[+-]?((0|[1-9][0-9]*)|[.][0-9]*|(0|[1-9][0-9]*)[.][0-9]*)([Ee][+-]?[0-9]+)?" # numeric literal (scientific notation possible)
        reservedPatterns[1] = "[+-]?0[0-7]+([.][0-7]*)?" # numeric literal (octal)
        reservedPatterns[2] = "[+-]?0[Xx][0-9A-Fa-f]+([.][0-9A-Fa-f]*)?" # numeric literal (hexadecimal)
    }

    split(string, s, "")
    currentToken = ""
    quoting = escaping = blockCommenting = lineCommenting = 0
    p = 0
    i = 1
    while (i <= length(s)) {
        c = s[i]
        r = substr(string, i)

        if (blockCommenting) {
            if (tempString = startsWithAny(r, rightBlockComments))
                blockCommenting = 0 # block comment ends

            i++

        } else if (lineCommenting) {
            if (belongsTo(c, newlines))
                lineCommenting = 0 # line comment ends

            i++

        } else if (quoting) {
            currentToken = currentToken c

            if (escaping) {
                escaping = 0 # escape ends

            } else {
                if (belongsTo(c, quotes)) {
                    # Finish the current token
                    if (currentToken) {
                        returnTokens[p++] = currentToken
                        currentToken = ""
                    }

                    quoting = 0 # quotation ends

                } else if (belongsTo(c, escapeChars)) {
                    escaping = 1 # escape begins

                } else {
                    # Continue
                }
            }

            i++

        } else {
            if (belongsTo(c, delimiters) || belongsTo(c, newlines)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                i++

            } else if (belongsTo(c, quotes)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                }

                currentToken = c

                quoting = 1 # quotation begins

                i++

            } else if (tempString = startsWithAny(r, leftBlockComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                blockCommenting = 1 # block comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, lineComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                lineCommenting = 1 # line comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, reservedOperators)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                returnTokens[p++] = tempString

                i += length(tempString)

            } else if (tempPattern = matchesAny(r, reservedPatterns)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                match(r, "^" tempPattern, tempGroup)
                returnTokens[p++] = tempGroup[0]

                i += length(tempGroup[0])

            } else {
                # Continue with the current token
                currentToken = currentToken c

                i++
            }
        }
    }

    # Finish the last token
    if (currentToken)
        returnTokens[p++] = currentToken

}

#!/usr/bin/gawk -f

# Tokenize a string.
function plTokenize(returnTokens, string,
                    delimiters,
                    newlines,
                    quotes,
                    escapeChars,
                    leftBlockComments,
                    rightBlockComments,
                    lineComments,
                    reservedOperators,
                    reservedPatterns,
                    ####
                    blockCommenting,
                    c,
                    currentToken,
                    escaping,
                    i,
                    lineCommenting,
                    p,
                    quoting,
                    r,
                    s,
                    tempGroup,
                    tempPattern,
                    tempString) {
    # Default parameters
    if (!delimiters[0]) {
        delimiters[0] = " "  # whitespace
        delimiters[1] = "\t" # horizontal tab
        delimiters[2] = "\v" # vertical tab
    }
    if (!newlines[0]) {
        newlines[0] = "\n" # line feed
        newlines[1] = "\r" # carriage return
    }
    if (!quotes[0]) {
        quotes[0] = "\"" # double quote
    }
    if (!escapeChars[0]) {
        escapeChars[0] = "\\" # backslash
    }
    if (!leftBlockComments[0]) {
        leftBlockComments[0] = "#|" # Lisp-style extended comment (open)
        leftBlockComments[1] = "/*" # C-style comment (open)
        leftBlockComments[2] = "(*" # ML-style comment (open)
    }
    if (!rightBlockComments[0]) {
        rightBlockComments[0] = "|#" # Lisp-style extended comment (close)
        rightBlockComments[1] = "*/" # C-style comment (close)
        rightBlockComments[2] = "*)" # ML-style comment (close)
    }
    if (!lineComments[0]) {
        lineComments[0] = ";"  # Lisp-style line comment
        lineComments[1] = "//" # C++-style line comment
        lineComments[2] = "#"  # hash comment
    }
    if (!reservedOperators[0]) {
        reservedOperators[0] = "(" #  left parenthesis
        reservedOperators[1] = ")" # right parenthesis
        reservedOperators[2] = "[" #  left bracket
        reservedOperators[3] = "]" # right bracket
        reservedOperators[4] = "{" #  left brace
        reservedOperators[5] = "}" # right brace
        reservedOperators[6] = "," # comma
    }
    if (!reservedPatterns[0]) {
        reservedPatterns[0] = "[+-]?((0|[1-9][0-9]*)|[.][0-9]*|(0|[1-9][0-9]*)[.][0-9]*)([Ee][+-]?[0-9]+)?" # numeric literal (scientific notation possible)
        reservedPatterns[1] = "[+-]?0[0-7]+([.][0-7]*)?" # numeric literal (octal)
        reservedPatterns[2] = "[+-]?0[Xx][0-9A-Fa-f]+([.][0-9A-Fa-f]*)?" # numeric literal (hexadecimal)
    }

    split(string, s, "")
    currentToken = ""
    quoting = escaping = blockCommenting = lineCommenting = 0
    p = 0
    i = 1
    while (i <= length(s)) {
        c = s[i]
        r = substr(string, i)

        if (blockCommenting) {
            if (tempString = startsWithAny(r, rightBlockComments))
                blockCommenting = 0 # block comment ends

            i++

        } else if (lineCommenting) {
            if (belongsTo(c, newlines))
                lineCommenting = 0 # line comment ends

            i++

        } else if (quoting) {
            currentToken = currentToken c

            if (escaping) {
                escaping = 0 # escape ends

            } else {
                if (belongsTo(c, quotes)) {
                    # Finish the current token
                    if (currentToken) {
                        returnTokens[p++] = currentToken
                        currentToken = ""
                    }

                    quoting = 0 # quotation ends

                } else if (belongsTo(c, escapeChars)) {
                    escaping = 1 # escape begins

                } else {
                    # Continue
                }
            }

            i++

        } else {
            if (belongsTo(c, delimiters) || belongsTo(c, newlines)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                i++

            } else if (belongsTo(c, quotes)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                }

                currentToken = c

                quoting = 1 # quotation begins

                i++

            } else if (tempString = startsWithAny(r, leftBlockComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                blockCommenting = 1 # block comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, lineComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                lineCommenting = 1 # line comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, reservedOperators)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                returnTokens[p++] = tempString

                i += length(tempString)

            } else if (tempPattern = matchesAny(r, reservedPatterns)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                match(r, "^" tempPattern, tempGroup)
                returnTokens[p++] = tempGroup[0]

                i += length(tempGroup[0])

            } else {
                # Continue with the current token
                currentToken = currentToken c

                i++
            }
        }
    }

    # Finish the last token
    if (currentToken)
        returnTokens[p++] = currentToken

}


Fix an awk script that doesn't seem to be very portable

BEGIN {
}

function getExtension(str){
  n = split(str, parts, ".")
  if (n == 1){
    return "<NO EXTENSION>"
  } else {
    return parts[n]
  }
}

$2 != "total" {
  by_type[getExtension($2)] += $1
}

$2 ~ "CMakeLists.txt" {
  #cmake should not count as text
  by_type["txt"] -= $1
  by_type["cmake"] += $1
}

$2 ~ "test" {
  by_type[getExtension($2)] -= $1
  by_type["TEST"] += $1
}

END {

  # join up cpp types
  by_type["cpp"] += by_type["hpp"]
  delete by_type["hpp"]

  #get rid of some filetypes I dont care about
  delete by_type["gitignore"]
  delete by_type["jpg"]
  delete by_type["png"]
  delete by_type["ttf"]
  delete by_type["pdf"]

  #print each getExtension, collect the total
  total=0
  for (x in by_type) {
    if (by_type[x] > 0) {
      total += by_type[x]
      print (x, by_type[x])
    }
  }

  print("Total:", total)
}

BEGIN {
}

function extension(str){
  n = split(str, parts, ".")
  if (n == 1){
    return "<NO EXTENSION>"
  } else {
    return parts[n]
  }
}

$2 != "total" {
  by_type[extension($2)] += $1
}

$2 ~ "CMakeLists.txt" {
  #cmake should not count as text
  by_type["txt"] -= $1
  by_type["cmake"] += $1
}

$2 ~ "test" {
  by_type[extension($2)] -= $1
  by_type["TEST"] += $1
}

END {

  # join up cpp types
  by_type["cpp"] += by_type["hpp"]
  delete by_type["hpp"]

  #get rid of some filetypes I dont care about
  delete by_type["gitignore"]
  delete by_type["jpg"]
  delete by_type["png"]
  delete by_type["ttf"]
  delete by_type["pdf"]

  #print each extension, collect the total
  total=0
  for (x in by_type) {
    if (by_type[x] > 0) {
      total += by_type[x]
      print (x, by_type[x])
    }
  }

  print("Total:", total)
}


Translators/YandexTranslate: fix #111

Translators/YandexTranslate: fix #111

####################################################################
# YandexTranslate.awk                                              #
####################################################################
#
# Last Updated: 11 Mar 2016
BEGIN { provides("yandex") }

function genSID(    content, group, temp) {
    content = curl("http://translate.yandex.com")

    match(content, /SID:[[:space:]]*'([^']+)'/, group)
    if (group[1]) {
        split(group[1], temp, ".")
        SID = reverse(temp[1]) "." reverse(temp[2]) "." reverse(temp[3])
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function yandexInit() {
    genSID() # generate a one-time key
    YandexWebTranslate = "z5h64q92x9.net" # host for web translation

    HttpProtocol = "http://"
    HttpHost = "translate.yandex.net"
    HttpPort = 80
}

function yandexRequestUrl(text, sl, tl, hl,    group) {
    # Quick hack: Yandex doesn't support digraphia code (yet)
    split(sl, group, "-"); sl = group[1]
    split(tl, group, "-"); tl = group[1]

    return HttpPathPrefix "/api/v1/tr.json/translate?"                  \
        "id=" SID "&srv=tr-text"                                        \
        "&text=" preprocess(text) "&lang=" (sl == "auto" ? tl : sl "-" tl)
}

function yandexTTSUrl(text, tl) {
    switch (tl) { # List of available TTS language codes
    case "ar": tl = "ar_AE"; break
    case "cs": tl = "cs_CZ"; break
    case "da": tl = "da_DK"; break
    case "de": tl = "de_DE"; break
    case "el": tl = "el_GR"; break
    case "en": tl = "en_GB"; break
    case "es": tl = "es_ES"; break
    case "fi": tl = "fi_FI"; break
    case "fr": tl = "fr_FR"; break
    case "it": tl = "it_IT"; break
    case "nl": tl = "nl_NL"; break
    case "no": tl = "no_NO"; break
    case "pl": tl = "pl_PL"; break
    case "pt": tl = "pt_PT"; break
    case "ru": tl = "ru_RU"; break
    case "sv": tl = "sv_SE"; break
    case "tr": tl = "tr_TR"; break
    default: tl = NULLSTR
    }
    return HttpProtocol "tts.voicetech.yandex.net" "/tts?"              \
        "text=" preprocess(text) (tl ? "&lang=" tl : tl)                \
        "&format=mp3" "&quality=hi"
}

function yandexWebTranslateUrl(uri, sl, tl, hl) {
    gsub(/:\/\//, "/", uri)
    return HttpProtocol YandexWebTranslate "/proxy_u/"  \
        (sl == "auto" ? tl : sl "-" tl)"/" uri
}

# Get the translation of a string.
function yandexTranslate(text, sl, tl, hl,
                         isVerbose, toSpeech, returnPlaylist, returnIl,
                         ####
                         r,
                         content, tokens, ast,
                         _sl, _tl, _hl, il,
                         translation,
                         wShowOriginal, wShowTranslation, wShowLanguages,
                         group, temp) {
    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    content = getResponse(text, _sl, _tl, _hl)
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    if (ast[0 SUBSEP "code"] != "200") {
        e("[ERROR] " unparameterize(ast[0 SUBSEP "message"]))
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "text" SUBSEP 0])

    split(unparameterize(ast[0 SUBSEP "lang"]), group, "-")
    returnIl[0] = il = group[1]
    if (Option["verbose"] < 0)
        return getList(il)

    # Generate output
    if (!isVerbose) {
        # Brief mode
        r = translation

    } else {
        # Verbose mode
        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]

        if (wShowOriginal) {
            # Display: original text
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
        }

        if (wShowTranslation) {
            # Display: major translation
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}

####################################################################
# YandexTranslate.awk                                              #
####################################################################
#
# Last Updated: 1 Jan 2016
BEGIN { provides("yandex") }

function genSID(    content, group) {
    content = curl("http://translate.yandex.com")

    match(content, /SID:[[:space:]]*'([^']+)'/, group)
    if (group[1]) {
        SID = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function yandexInit() {
    genSID() # generate a one-time key
    YandexWebTranslate = "z5h64q92x9.net" # host for web translation

    HttpProtocol = "http://"
    HttpHost = "translate.yandex.net"
    HttpPort = 80
}

function yandexRequestUrl(text, sl, tl, hl,    group) {
    # Quick hack: Yandex doesn't support digraphia code (yet)
    split(sl, group, "-"); sl = group[1]
    split(tl, group, "-"); tl = group[1]

    return HttpPathPrefix "/api/v1/tr.json/translate?"                  \
        "id=" SID "&srv=tr-text"                                        \
        "&text=" preprocess(text) "&lang=" (sl == "auto" ? tl : sl "-" tl)
}

function yandexTTSUrl(text, tl) {
    switch (tl) { # List of available TTS language codes
    case "ar": tl = "ar_AE"; break
    case "cs": tl = "cs_CZ"; break
    case "da": tl = "da_DK"; break
    case "de": tl = "de_DE"; break
    case "el": tl = "el_GR"; break
    case "en": tl = "en_GB"; break
    case "es": tl = "es_ES"; break
    case "fi": tl = "fi_FI"; break
    case "fr": tl = "fr_FR"; break
    case "it": tl = "it_IT"; break
    case "nl": tl = "nl_NL"; break
    case "no": tl = "no_NO"; break
    case "pl": tl = "pl_PL"; break
    case "pt": tl = "pt_PT"; break
    case "ru": tl = "ru_RU"; break
    case "sv": tl = "sv_SE"; break
    case "tr": tl = "tr_TR"; break
    default: tl = NULLSTR
    }
    return HttpProtocol "tts.voicetech.yandex.net" "/tts?"              \
        "text=" preprocess(text) (tl ? "&lang=" tl : tl)                \
        "&format=mp3" "&quality=hi"
}

function yandexWebTranslateUrl(uri, sl, tl, hl) {
    gsub(/:\/\//, "/", uri)
    return HttpProtocol YandexWebTranslate "/proxy_u/"  \
        (sl == "auto" ? tl : sl "-" tl)"/" uri
}

# Get the translation of a string.
function yandexTranslate(text, sl, tl, hl,
                         isVerbose, toSpeech, returnPlaylist, returnIl,
                         ####
                         r,
                         content, tokens, ast,
                         _sl, _tl, _hl, il,
                         translation,
                         wShowOriginal, wShowTranslation, wShowLanguages,
                         group, temp) {
    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    content = getResponse(text, _sl, _tl, _hl)
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    if (ast[0 SUBSEP "code"] != "200") {
        e("[ERROR] " unparameterize(ast[0 SUBSEP "message"]))
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "text" SUBSEP 0])

    split(unparameterize(ast[0 SUBSEP "lang"]), group, "-")
    returnIl[0] = il = group[1]
    if (Option["verbose"] < 0)
        return getList(il)

    # Generate output
    if (!isVerbose) {
        # Brief mode
        r = translation

    } else {
        # Verbose mode
        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]

        if (wShowOriginal) {
            # Display: original text
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
        }

        if (wShowTranslation) {
            # Display: major translation
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}


added basic filtering by user

added basic filtering by user

function obsForm()
{

  htmlHeader("Make an observation", 400);

  print "
Make an observation
";
  print "
for tree with tag " f["tag"] "
 (Please double check the tag code)
";
  print "<form enctype=\"multipart/form-data\" method=\"post\" action=\"map?method=obsD\">" ;
  print "
<input type=\"hidden\" name=\"tag\" value=\"" f["tag"] "\" />Type of observation:
<select name=\"type\"><option value=\"bud\">Spring buds just breaking open</option><option value=\"flower\">In flower (any stage)</option><option value=\"fruit\">In fruit (any stage)</option><option value=\"fall\">Outstanding fall color</option><option value=\"fobs\">Feeding obs. (note animal and plant part)</option></select>
";
  # <option value=\"damage\">Report plant damage</option></select>
";
  print "
Upload a photo (optional)
(small please: <2 MB; NB: iPhone does not support file uploads)
<input type=\"file\" name=\"photo\" />
";
  print "
Upload a photo of the plant tag (optional)
<input type=\"file\" name=\"tagphoto\" />
";
  print "
Your notes (optional):
<textarea name=\"notes\" cols=\"30\" rows=\"2\"></textarea>
";
  print "
Your email or username (optional)
<input type=\"text\" name=\"email\" size=\"30\"/>
";

  print "
<input type=\"submit\" value=\"Send observation\" />
";
  print "</form>";

  htmlFooter();

}

function obsCapture()
{

# Script to parse return from observation form

# Form template:

# -----------------------------1928954284948244595549659756
# Content-Disposition: form-data; name="type"
#
# fall
# -----------------------------1928954284948244595549659756
# Content-Disposition: form-data; name="notes"
#
# Notes
# -----------------------------1928954284948244595549659756
# Content-Disposition: form-data; name="photo"; filename="img.png"
# Content-Type: image/png
#
# <image>
# -----------------------------1928954284948244595549659756--

  htmlHeader("Data stored",400);

  # View form data during testing with:
  #  RS = "\x04"; getline; print "
" $0;
  #  print $0 > "tmp";
  #  print "
"; exit;

  RS = "\r\n"; getline;
  # Use the first line as the multipart form delimiter
  # escape regexp chars (prob just `-'):
  gsub(/\-/,"\\-",$0); gsub(/\*/,"\\*",$0);
  gsub(/\./,"\\.",$0); gsub(/\?/,"\\?",$0);
  RS = $0 "[^\r]*" "\r?\n" ;
  recdate = strftime("%Y-%m-%d");
  frecdate = gensub(/\-/,"","G",recdate);
  rectime= strftime("%H%M%S") ;
  # had to add the extra regex return because webkit and mozilla use a
  #   different last delimiter ( ----XYZ-- )
  while (getline > 0)
  	{
	  # print "
" $0 "
";
	  # split the dataparts
	  lines = split($0, line, "\r\n");
	  # if the media:
	  if ((line[2] ~ /Content\-Type/) && \
	      ("png|jpeg|jpg" ~ gensub(/[^/]*\/([a-z]+)/,"\\1","G",line[2])))
		{
		  ORS = "";
		  split (line[1], lineTmp2, ";");
		  # now need to differentiate between main photo, and tag photo
		  pht = gensub(/ name="([^"]*)"$/,"\\1","G",lineTmp2[2]);
		  filename[pht] = gensub(/ filename="([^"]*)"$/,"\\1","G",lineTmp2[3]);
		  mediatype = gensub(/[^/]*\/([a-z]+)/,"\\1","G",line[2]);
		  tagmark = ""; if (pht == "tagphoto") tagmark = "_tag" ;
		  outfile[pht] = frecdate "T" rectime "_" tag tagmark "." mediatype ;
		  # likely all to be in line 4, but there may be a \r\n in the
		  #  file, so better to concatenate lines
		  for (x = 4 ; x < lines ; x++)
			{
			  # could add a rand component: srand(); substr(rand(),3,5)
			  # but chance of two uploads in same second very tiny
			  print line[x] "\r\n" >> "obsdata/" outfile[pht];
			}
		  ORS = "\n";
		}
	  else if (line[1] ~ /Content\-Disposition/)
		{
		  split (line[1], lineTmp, ";");
		  name = gensub(/\ name="([^"]*)"$/,"\\1","G",lineTmp[2]);
		  if (name == "tag")  { tag = line[3] }
		  if (name == "type") { obstype = line[3] }
		  if (name == "email") { email = line[3] }
		  if (name == "notes")
			{
			  for (x = 3 ; x < lines ; x++)
				{
				  notes = notes line[x] "\\n" ;
				}
			}
		}
	}

  gsub(/|/,"",notes) ; gsub(/|/,"",filename["photo"]); # just in case

  # WRITE TO FILE
  print "<date>" recdate \
	"</date><time>" rectime \
	"</time><tag>" tag \
	"</tag><obstype>" obstype \
	"</obstype><notes>" notes \
	"</notes><email>" email \
	"</email><oriname>"	filename["photo"] \
	"</oriname><outfile>" outfile["photo"] \
	"</outfile><tagoriname>"  filename["tagphoto"] \
    "</tagoriname><tagoutfile>" outfile["tagphoto"] \
	"</tagoutfile>" >> "obsdata/data.csv" ;

  # OUTPUT

  print "
Data stored
" ;
  print "
Thanks for your observation on plant " tag "
";
  print "
[ home ]
";

  # make thumb (of main photo only)
  if (outfile["photo"] != "")
  {
      system("convert 'obsdata/" outfile["photo"] "' -resize 50x50 'obsdata/sm_" outfile["photo"] "'");
  }

}

function obsReport()
{

  htmlHeader("Observations report", 420);

  stages["bud"] = "budding out";
  stages["flower"] = "in flower";
  stages["fruit"] = "in fruit";
  stages["fall"] = "Fall color";
  stages["fobs"] = "Feeding obs";

  FS="|";

  while ((getline < "data/aa-tree.csv") > 0)
  {
	sp[$1] = $4 ;
  }

  while ((getline < "obsdata/data.csv") > 0)
	{
	  parseXML(++n,$0);
	  if (XML[n,"tag"] == f["tag"])
		{
		  XML[n, "match"] = 1; tagDataFound = 1;
		}
	}

  # if a tag argument is given
  if ((f["tag"] != "") && (tagDataFound != 1))
	{
	  print "
Sorry, no citizen science records for this plant.
  Why not make one?
" ;
	}
  else if (tagDataFound)
	{
	  print "
Observations for " f["tag"] "
" ;
	  print "
";
	  print "
Tag	Taxon	Info	Date	Notes	Photo
";
	  for (i = 1; i <= n ; i++)
	  {
	      if (XML[i,"tag"] == f["tag"])
	      {
		  print "
" XML[i,"tag"] "	" ;
		  print "" sp[XML[i,"tag"]] "	" ;

			print "" stages[XML[i,"obstype"]] "	" ;
			print "" XML[i,"date"] "	" ;
			print "" gensub(/\\n/,"\\ ", "G", XML[i,"notes"]) "	" ;
			if (XML[i,"outfile"] != "")
			{
			    print "
" ;
			}
			else print "
" ;
		    }
		}
	  print "
";
	}
  else
	{
	  if (f["email"] != "")
		{
		  print "
Your observations
" ;
		  print "
" f["email"] "
" ;
		  start = 1;
		}
	  else
		{
		  print "
Recent observations
" ;
		  if (n > 10) start = n-10 ;
		  else start = 1;
		}
	  # print the table header
	  print "
";
	  print "
Tag	Taxon	Info	Date	Notes	Photo
";

	  for (i = start; i <= n ; i++)
		{
		  if (((f["email"] != "") && (f["email"] == XML[i,"email"])) \
			  || (f["email"] == ""))
			{
			  print "
" XML[i,"tag"] "	" ;
			  print "" sp[XML[i,"tag"]] "	" ;

			  print "" stages[XML[i,"obstype"]] "	" ;
			  print "" XML[i,"date"] "	" ;
			  print "" gensub(/\\n/,"\\ ", "G", XML[i,"notes"]) "	" ;
			  if (XML[i,"outfile"] != "")
				{
				  print "
" ;
				}
			  else print "
" ;
			}
		}
	  print "
";

	  # user form
	  print "

To see your own images (if tagged with your email address), enter your email address here:
<form method=\"get\" action=\"map\"><input name=\"method\" type=\"hidden\" value=\"obsS\" /><input type=\"text\" name=\"email\" length=\"20\"/><input type=\"submit\" value=\"Go\" /></form>" ;
	}

}

function parseXML(indx, string,    i)
{
  # parse simple XML
  i = 1;
  while (i <= length(string))
	{
	  # skip leading blanks and check in case there are no `<' in line
	  while ((substr(string,i,1) != "<") && (i <= length(string))) i++ ;
	  tag = ""; i++;
	  # read tag name
	  while ((substr(string,i,1) != ">") && (i <= length(string)))
		{
		  tag = tag substr(string,i,1) ; i++;
		}
	  # clean tag name of blanks
	  gsub(/\ /,"",tag);  data = ""; i++;
	  # read data, terminated by `</'
	  while ((substr(string,i,2) != "</") && (i <= length(string)))
		{
		  data = data substr(string,i,1) ; i++;
		}
	  # record
	  XML[indx, tag] = data ; i++;
	  # print tag, data;
	  # move to next tag, or finish
	  while ((substr(string,i+1,1) != "<") && (i <= length(string))) i++;
	}
}

function obsForm()
{

  htmlHeader("Make an observation", 400);

  print "
Make an observation
";
  print "
for tree with tag " f["tag"] "
 (Please double check the tag code)
";
  print "<form enctype=\"multipart/form-data\" method=\"post\" action=\"map?method=obsD\">" ;
  print "
<input type=\"hidden\" name=\"tag\" value=\"" f["tag"] "\" />Type of observation:
<select name=\"type\"><option value=\"bud\">Spring buds just breaking open</option><option value=\"flower\">In flower (any stage)</option><option value=\"fruit\">In fruit (any stage)</option><option value=\"fall\">Outstanding fall color</option><option value=\"fobs\">Feeding obs. (note animal and plant part)</option></select>
";
  # <option value=\"damage\">Report plant damage</option></select>
";
  print "
Upload a photo (optional)
(small please: <2 MB; NB: iPhone does not support file uploads)
<input type=\"file\" name=\"photo\" />
";
  print "
Upload a photo of the plant tag (optional)
<input type=\"file\" name=\"tagphoto\" />
";
  print "
Your notes (optional):
<textarea name=\"notes\" cols=\"30\" rows=\"2\"></textarea>
";
  print "
Your email or username (optional)
<input type=\"text\" name=\"email\" size=\"30\"/>
";

  print "
<input type=\"submit\" value=\"Send observation\" />
";
  print "</form>";

  htmlFooter();

}

function obsCapture()
{

# Script to parse return from observation form

# Form template:

# -----------------------------1928954284948244595549659756
# Content-Disposition: form-data; name="type"
#
# fall
# -----------------------------1928954284948244595549659756
# Content-Disposition: form-data; name="notes"
#
# Notes
# -----------------------------1928954284948244595549659756
# Content-Disposition: form-data; name="photo"; filename="img.png"
# Content-Type: image/png
#
# <image>
# -----------------------------1928954284948244595549659756--

  htmlHeader("Data stored",400);

  # View form data during testing with:
  #  RS = "\x04"; getline; print "
" $0;
  #  print $0 > "tmp";
  #  print "
"; exit;

  RS = "\r\n"; getline;
  # Use the first line as the multipart form delimiter
  # escape regexp chars (prob just `-'):
  gsub(/\-/,"\\-",$0); gsub(/\*/,"\\*",$0);
  gsub(/\./,"\\.",$0); gsub(/\?/,"\\?",$0);
  RS = $0 "[^\r]*" "\r?\n" ;
  recdate = strftime("%Y-%m-%d");
  frecdate = gensub(/\-/,"","G",recdate);
  rectime= strftime("%H%M%S") ;
  # had to add the extra regex return because webkit and mozilla use a
  #   different last delimiter ( ----XYZ-- )
  while (getline > 0)
  	{
	  # print "
" $0 "
";
	  # split the dataparts
	  lines = split($0, line, "\r\n");
	  # if the media:
	  if ((line[2] ~ /Content\-Type/) && \
	      ("png|jpeg|jpg" ~ gensub(/[^/]*\/([a-z]+)/,"\\1","G",line[2])))
		{
		  ORS = "";
		  split (line[1], lineTmp2, ";");
		  # now need to differentiate between main photo, and tag photo
		  pht = gensub(/ name="([^"]*)"$/,"\\1","G",lineTmp2[2]);
		  filename[pht] = gensub(/ filename="([^"]*)"$/,"\\1","G",lineTmp2[3]);
		  mediatype = gensub(/[^/]*\/([a-z]+)/,"\\1","G",line[2]);
		  tagmark = ""; if (pht == "tagphoto") tagmark = "_tag" ;
		  outfile[pht] = frecdate "T" rectime "_" tag tagmark "." mediatype ;
		  # likely all to be in line 4, but there may be a \r\n in the
		  #  file, so better to concatenate lines
		  for (x = 4 ; x < lines ; x++)
			{
			  # could add a rand component: srand(); substr(rand(),3,5)
			  # but chance of two uploads in same second very tiny
			  print line[x] "\r\n" >> "obsdata/" outfile[pht];
			}
		  ORS = "\n";
		}
	  else if (line[1] ~ /Content\-Disposition/)
		{
		  split (line[1], lineTmp, ";");
		  name = gensub(/\ name="([^"]*)"$/,"\\1","G",lineTmp[2]);
		  if (name == "tag")  { tag = line[3] }
		  if (name == "type") { obstype = line[3] }
		  if (name == "email") { email = line[3] }
		  if (name == "notes")
			{
			  for (x = 3 ; x < lines ; x++)
				{
				  notes = notes line[x] "\\n" ;
				}
			}
		}
	}

  gsub(/|/,"",notes) ; gsub(/|/,"",filename["photo"]); # just in case

  # WRITE TO FILE
  print "<date>" recdate \
	"</date><time>" rectime \
	"</time><tag>" tag \
	"</tag><obstype>" obstype \
	"</obstype><notes>" notes \
	"</notes><email>" email \
	"</email><oriname>"	filename["photo"] \
	"</oriname><outfile>" outfile["photo"] \
	"</outfile><tagoriname>"  filename["tagphoto"] \
    "</tagoriname><tagoutfile>" outfile["tagphoto"] \
	"</tagoutfile>" >> "obsdata/data.csv" ;

  # OUTPUT

  print "
Data stored
" ;
  print "
Thanks for your observation on plant " tag "
";
  print "
[ home ]
";

  # make thumb
  if (outfile["photo"] != "")
  {
      system("convert 'obsdata/" outfile["photo"] "' -resize 50x50 'obsdata/sm_" outfile["photo"] "'");
  }
  if (outfile["tagphoto"] != "")
  {
      system("convert 'obsdata/" outfile["tagphoto"] "' -resize 50x50 'obsdata/sm_" outfile["tagphoto"] "'");
  }

}

function obsReport()
{

  htmlHeader("Observations report", 420);

  stages["bud"] = "budding out";
  stages["flower"] = "in flower";
  stages["fruit"] = "in fruit";
  stages["fall"] = "Fall color";
  stages["fobs"] = "Feeding obs";

  FS="|";

  while ((getline < "data/aa-tree.csv") > 0)
  {
	sp[$1] = $4 ;
  }

  while ((getline < "obsdata/data.csv") > 0)
	{
	  parseXML(++n,$0);
	  if (XML[n,"tag"] == f["tag"])
		{
		  XML[n, "match"] = 1; tagDataFound = 1;
		}
	}

  if ((f["tag"] != "") && (tagDataFound != 1))
	{
	  print "
Sorry, no citizen science records for this plant.
  Why not make one?
" ;
	}
  else if (tagDataFound)
	{
	  print "
Observations for " f["tag"] "
" ;
	  print "
";
	  print "
Tag	Taxon	Info	Date	Notes	Photo
";
	  for (i = 1; i <= n ; i++)
	  {
	      if (XML[i,"tag"] == f["tag"])
	      {
		  print "
" XML[i,"tag"] "	" ;
		  print "" sp[XML[i,"tag"]] "	" ;

			print "" stages[XML[i,"obstype"]] "	" ;
			print "" XML[i,"date"] "	" ;
			print "" gensub(/\\n/,"\\ ", "G", XML[i,"notes"]) "	" ;
			if (XML[i,"outfile"] != "")
			{
			    print "
" ;
			}
			else print "
" ;
		    }
		}
	  print "
";
	}
  else
	{
	    print "
Recent observations
" ;
	  print "
";
	  print "
Tag	Taxon	Info	Date	Notes	Photo
";
	  if (n > 10) start = n-10 ;
	  else start = 1;
	  for (i = start; i <= n ; i++)
		{
		  print "
" XML[i,"tag"] "	" ;
		  print "" sp[XML[i,"tag"]] "	" ;

		  print "" stages[XML[i,"obstype"]] "	" ;
		  print "" XML[i,"date"] "	" ;
		  print "" gensub(/\\n/,"\\ ", "G", XML[i,"notes"]) "	" ;
		  if (XML[i,"outfile"] != "")
			{
			  print "
" ;
			}
		  else print "
" ;
		}
	  print "
";
	}

}

function parseXML(indx, string,    i)
{
  # parse simple XML
  i = 1;
  while (i <= length(string))
	{
	  # skip leading blanks and check in case there are no `<' in line
	  while ((substr(string,i,1) != "<") && (i <= length(string))) i++ ;
	  tag = ""; i++;
	  # read tag name
	  while ((substr(string,i,1) != ">") && (i <= length(string)))
		{
		  tag = tag substr(string,i,1) ; i++;
		}
	  # clean tag name of blanks
	  gsub(/\ /,"",tag);  data = ""; i++;
	  # read data, terminated by `</'
	  while ((substr(string,i,2) != "</") && (i <= length(string)))
		{
		  data = data substr(string,i,1) ; i++;
		}
	  # record
	  XML[indx, tag] = data ; i++;
	  # print tag, data;
	  # move to next tag, or finish
	  while ((substr(string,i+1,1) != "<") && (i <= length(string))) i++;
	}
}


This should have been JAVA_EXTRA_ARGUMENTS.

This reverts commit b71ab57ccb7ec5435e4d6ed3cfbe8e5be2bcb673.

##**************************************************************
##
## Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
## University of Wisconsin-Madison, WI.
##
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at
##
##    http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
##**************************************************************

##
## convert_config_to_win32.awk
##
## Todd Tannenbaum <tannenba@cs.wisc.edu>, 10/06
##

# This is an awk script to modify condor_config.generic into a form that
# works with the InstallShield Windows installer.  The packager on Win32
# will pipe condor_config.generic through this awk script and save it as
# condor_config.orig, which is what the InstallShield script expects to
# modify after unbundling.

BEGIN {
	FS = "="
}

# Make it so we can fill in the VM_GAHP_SERVER option if we choose to in the
# installer. This is done before the .exe append bellow, because that match
# seems to block any other matches for that line.
/^#VM_GAHP_SERVER/ {
	print "#VM_GAHP_SERVER = $(BIN)/condor_vm-gahp.exe"
	print "VM_GAHP_SERVER = "
	next
}

# Add a .exe to the end of all condor daemon names
/BIN)\/condor_/ {
	printf "%s.exe\n",$0
	next
}

# Email Settings
# Use condor_mail.exe for MAIL, and put in a stub for SMTP_SERVER
/^MAIL/ {
	printf "MAIL = $(BIN)/condor_mail.exe\n\n"
	printf "## For Condor on Win32 we need to specify an SMTP server so\n"
	printf "## that Condor is able to send email.\n"
	printf "SMTP_SERVER =\n\n"
	next
}
/^SMTP_SERVER/ {
	next
}

# Specify $(FULL_HOSTNAME) as the sample commented-out entry for
# FILESYSTEM_DOMAIN and UID_DOMAIN, as this is what the installer expects.
/^UID_DOMAIN/ || /^FILESYSTEM_DOMAIN/ {
	printf "#%s= $(FULL_HOSTNAME)\n", $1
	next
}

# Have $(SBIN) point to $(BIN)
/^SBIN/ {
	printf "%s= $(BIN)\n", $1
	next
}

# QUEUE_SUPER_USERS should be condor and SYSTEM, not root! :)
/^QUEUE_SUPER_USERS/ {
	printf "%s= condor, SYSTEM\n", $1
	next
}

# JOB_RENICE_INCREMENT must be 10 or load average will be messed up
# Be careful, it may be commented out in condor_config.generic.
/^.*JOB_RENICE_INCREMENT/ {
	printf "JOB_RENICE_INCREMENT = 10\n"
	next
}

# Clean out the initial guess of /usr/bin/java for JAVA... ;)
# Note the space after JAVA below, so we do not change other JAVA settings.
/^JAVA / {
	printf "JAVA =\n"
	next
}

# Win32 has a path separator of ';', while on Unix it is ':'
/^JAVA_CLASSPATH_SEPARATOR/ {
	printf "%s= ;\n", $1
	next
}

# There's no reasonable default place to put log files for
# daemons that run as a user (on Unix we use /tmp). Use NUL.
/(^C_GAHP_LOG)|(^C_GAHP_LOCK)|(^C_GAHP_WORKER_THREAD_LOG)|(^C_GAHP_WORKER_THREAD_LOCK)|(^VM_GAHP_LOG)/ {
	printf "%s = NUL\n", $1
	next
}

# PROCD_ADDRESS needs to refer to a valid name for a Windows named
# pipe
/^PROCD_ADDRESS/ {
	print "PROCD_ADDRESS = \\\\.\\pipe\\condor_procd_pipe"
	next
}

# Make it so we can fill in the VM GAHP options if we choose
# to in the installer.
/^#VM_TYPE/ {
	print "VM_TYPE ="
	next
}
/^#VM_VERSION/ {
	print "#VM_VERSION = server1.0"
	print "VM_VERSION ="
	next
}
/^#VM_MAX_NUMBER/ {
	print "VM_MAX_NUMBER = $(NUM_CPUS)"
	next
}
/^#VM_MEMORY/ {
	print "VM_MEMORY = 128"
	next
}
/^#VMWARE_LOCAL_SETTINGS_FILE/ {
	print "VMWARE_LOCAL_SETTINGS_FILE = $(RELEASE_DIR)/condor_vmware_local_settings"
	next
}

# Set it up so we can change the VMWARE_SCRIPT option if we choose to
# do so in the installer
#/^VMWARE_SCRIPT/ {
#	printf "VMWARE_SCRIPT = $(BIN)/condor_vm_vmware.pl\n"
#	next
#}

# Change the configuration such that we can modify the VMware
# networking types
/^#VM_NETWORKING_TYPE/ {
	print "VM_NETWORKING_TYPE = nat"
	next
}
/^#VM_NETWORKING_DEFAULT_TYPE/ {
	print "VM_NETWORKING_DEFAULT_TYPE = nat"
	next
}

# We use the equals sign so that it will not match the names above
# and trash the resulting configuration file
/^#VM_NETWORKING =/ {
	print "VM_NETWORKING = FALSE"
	next
}
/^#VMWARE_NAT_NETWORKING_TYPE/ {
	printf "VMWARE_NAT_NETWORKING_TYPE = nat\n"
	next
}
/^#VMWARE_BRIDGE_NETWORKING_TYPE/ {
	printf "VMWARE_BRIDGE_NETWORKING_TYPE = bridged\n"
	next
}

# If we made it here, print out the line unchanged
{print $0}

##**************************************************************
##
## Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
## University of Wisconsin-Madison, WI.
##
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at
##
##    http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
##**************************************************************

##
## convert_config_to_win32.awk
##
## Todd Tannenbaum <tannenba@cs.wisc.edu>, 10/06
##

# This is an awk script to modify condor_config.generic into a form that
# works with the InstallShield Windows installer.  The packager on Win32
# will pipe condor_config.generic through this awk script and save it as
# condor_config.orig, which is what the InstallShield script expects to
# modify after unbundling.

BEGIN {
	FS = "="
}

# Make it so we can fill in the VM_GAHP_SERVER option if we choose to in the
# installer. This is done before the .exe append bellow, because that match
# seems to block any other matches for that line.
/^#VM_GAHP_SERVER/ {
	print "#VM_GAHP_SERVER = $(BIN)/condor_vm-gahp.exe"
	print "VM_GAHP_SERVER = "
	next
}

# Add a .exe to the end of all condor daemon names
/BIN)\/condor_/ {
	printf "%s.exe\n",$0
	next
}

# Email Settings
# Use condor_mail.exe for MAIL, and put in a stub for SMTP_SERVER
/^MAIL/ {
	printf "MAIL = $(BIN)/condor_mail.exe\n\n"
	printf "## For Condor on Win32 we need to specify an SMTP server so\n"
	printf "## that Condor is able to send email.\n"
	printf "SMTP_SERVER =\n\n"
	next
}
/^SMTP_SERVER/ {
	next
}

# Specify $(FULL_HOSTNAME) as the sample commented-out entry for
# FILESYSTEM_DOMAIN and UID_DOMAIN, as this is what the installer expects.
/^UID_DOMAIN/ || /^FILESYSTEM_DOMAIN/ {
	printf "#%s= $(FULL_HOSTNAME)\n", $1
	next
}

# Have $(SBIN) point to $(BIN)
/^SBIN/ {
	printf "%s= $(BIN)\n", $1
	next
}

# QUEUE_SUPER_USERS should be condor and SYSTEM, not root! :)
/^QUEUE_SUPER_USERS/ {
	printf "%s= condor, SYSTEM\n", $1
	next
}

# JOB_RENICE_INCREMENT must be 10 or load average will be messed up
# Be careful, it may be commented out in condor_config.generic.
/^.*JOB_RENICE_INCREMENT/ {
	printf "JOB_RENICE_INCREMENT = 10\n"
	next
}

# Clean out the initial guess of /usr/bin/java for JAVA... ;)
# Note the space after JAVA below, so we do not change other JAVA settings.
/^JAVA / {
	printf "JAVA =\n"
	next
}

# Default to -Xmx1024m
/^JAVA_MAXHEAP_ARGUMENT/ {
	printf "%s= -Xmx1024m\n", $1
	next
}

# Win32 has a path separator of ';', while on Unix it is ':'
/^JAVA_CLASSPATH_SEPARATOR/ {
	printf "%s= ;\n", $1
	next
}

# There's no reasonable default place to put log files for
# daemons that run as a user (on Unix we use /tmp). Use NUL.
/(^C_GAHP_LOG)|(^C_GAHP_LOCK)|(^C_GAHP_WORKER_THREAD_LOG)|(^C_GAHP_WORKER_THREAD_LOCK)|(^VM_GAHP_LOG)/ {
	printf "%s = NUL\n", $1
	next
}

# PROCD_ADDRESS needs to refer to a valid name for a Windows named
# pipe
/^PROCD_ADDRESS/ {
	print "PROCD_ADDRESS = \\\\.\\pipe\\condor_procd_pipe"
	next
}

# Make it so we can fill in the VM GAHP options if we choose
# to in the installer.
/^#VM_TYPE/ {
	print "VM_TYPE ="
	next
}
/^#VM_VERSION/ {
	print "#VM_VERSION = server1.0"
	print "VM_VERSION ="
	next
}
/^#VM_MAX_NUMBER/ {
	print "VM_MAX_NUMBER = $(NUM_CPUS)"
	next
}
/^#VM_MEMORY/ {
	print "VM_MEMORY = 128"
	next
}
/^#VMWARE_LOCAL_SETTINGS_FILE/ {
	print "VMWARE_LOCAL_SETTINGS_FILE = $(RELEASE_DIR)/condor_vmware_local_settings"
	next
}

# Set it up so we can change the VMWARE_SCRIPT option if we choose to
# do so in the installer
#/^VMWARE_SCRIPT/ {
#	printf "VMWARE_SCRIPT = $(BIN)/condor_vm_vmware.pl\n"
#	next
#}

# Change the configuration such that we can modify the VMware
# networking types
/^#VM_NETWORKING_TYPE/ {
	print "VM_NETWORKING_TYPE = nat"
	next
}
/^#VM_NETWORKING_DEFAULT_TYPE/ {
	print "VM_NETWORKING_DEFAULT_TYPE = nat"
	next
}

# We use the equals sign so that it will not match the names above
# and trash the resulting configuration file
/^#VM_NETWORKING =/ {
	print "VM_NETWORKING = FALSE"
	next
}
/^#VMWARE_NAT_NETWORKING_TYPE/ {
	printf "VMWARE_NAT_NETWORKING_TYPE = nat\n"
	next
}
/^#VMWARE_BRIDGE_NETWORKING_TYPE/ {
	printf "VMWARE_BRIDGE_NETWORKING_TYPE = bridged\n"
	next
}

# If we made it here, print out the line unchanged
{print $0}


Added more "Not available in Canada" strings

Acorn found two more ways to screw up my filtering. Fixed both.

# Given the output of an episode file (for example) curl -s https://acorn.tv/800words
# create lists of titles, descriptions, episodes and seasons
# to be pasted into a spreadsheet. Note that IN_CANADA affects processing.
#
# Invoked with file parameters as follows:
# awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#     -v EPISODES_FILE=$EPISODES_FILE -v SEASONS_FILE=$SEASONS_FILE \
#     -v IN_CANADA=$IN_CANADA -f fetchAcorn-episodes.awk

/span itemprop="name"/ {
    sub (/.*name">/,"")
    sub (/<.*/,"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    episodeStr = ""
    print >> TITLE_FILE
}

/id="franchise-description"/ {
    # get rid of boilerplate
    sub (/.*itemprop="description">/,"")
    sub (/<\/p>$/,"")
    # get rid of unnecessary characters and text
    gsub (/\\/,"")
    if (IN_CANADA != "yes") {
        sub (/Series 1 not available in Canada\./,"")
        sub (/Not [Aa]vailable in Canada\./,"")
        sub (/NOT AVAILABLE IN CANADA\./,"")
    }
    sub (/CC Available\. CC Available/,"CC Available")
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available")
    gsub (/ \./,".")
    gsub (/  */," ")
    sub (/^ */,"")
    sub (/ *$/,"")
    # fix funky HTML characters
    gsub (/'/,"'")
    print >> DESCRIPTION_FILE
}

/itemprop="numberOfSeasons"/ {
    sub (/.*content="/,"")
    sub (/" \/>.*/,"")
    print >> SEASONS_FILE
}

/itemprop="numberOfEpisodes"/ {
    sub (/.*content="/,"")
    sub (/" \/>.*/,"")
    episodeLinesFound += 1
    if (episodeLinesFound != 1)
        episodeStr = episodeStr "+" $0
}

END { print (episodeLinesFound == 1 ? "=0" : "=" episodeStr) >> EPISODES_FILE }

# Given the output of an episode file (for example) curl -s https://acorn.tv/800words
# create lists of titles, descriptions, episodes and seasons
# to be pasted into a spreadsheet. Note that IN_CANADA affects processing.
#
# Invoked with file parameters as follows:
# awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#     -v EPISODES_FILE=$EPISODES_FILE -v SEASONS_FILE=$SEASONS_FILE \
#     -v IN_CANADA=$IN_CANADA -f fetchAcorn-episodes.awk

/span itemprop="name"/ {
    sub (/.*name">/,"")
    sub (/<.*/,"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    episodeStr = ""
    print >> TITLE_FILE
}

/id="franchise-description"/ {
    # get rid of boilerplate
    sub (/.*itemprop="description">/,"")
    sub (/<\/p>$/,"")
    # get rid of unnecessary characters and text
    gsub (/\\/,"")
    if (IN_CANADA != "yes") {
        sub (/Not [Aa]vailable in Canada\./,"")
        sub (/NOT AVAILABLE IN CANADA\./,"")
    }
    sub (/CC Available\. CC Available/,"CC Available")
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available")
    gsub (/ \./,".")
    gsub (/  */," ")
    sub (/ *$/,"")
    # fix funky HTML characters
    gsub (/'/,"'")
    print >> DESCRIPTION_FILE
}

/itemprop="numberOfSeasons"/ {
    sub (/.*content="/,"")
    sub (/" \/>.*/,"")
    print >> SEASONS_FILE
}

/itemprop="numberOfEpisodes"/ {
    sub (/.*content="/,"")
    sub (/" \/>.*/,"")
    episodeLinesFound += 1
    if (episodeLinesFound != 1)
        episodeStr = episodeStr "+" $0
}

END { print (episodeLinesFound == 1 ? "=0" : "=" episodeStr) >> EPISODES_FILE }


Parser script displays number of compiler errors

#
# MIT License
#
# Copyright (c) 2017 Jean-Sebastien CONAN
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

#
# Part of the camelSCAD library.
#
# Parses and displays the results of the unit tests.
#
# @package util/test
# @author jsconan
#

BEGIN {
    # will parse a CSV file
    FS = ";"

    # init counters
    packageCount = 0
    moduleCount  = 0
    assertCount  = 0
    successCount = 0
    failedCount  = 0
    warningCount = 0
    errorCount   = 0

    # define ANSI colors
    BLACK    = "\033[0;30m"
    GRAY     = "\033[1;30m"
    RED      = "\033[0;31m"
    LRED     = "\033[1;31m"
    GREEN    = "\033[0;32m"
    LGREEN   = "\033[1;32m"
    BROWN    = "\033[0;33m"
    YELLOW   = "\033[1;33m"
    BLUE     = "\033[0;34m"
    LBLUE    = "\033[1;34m"
    MAGENTA  = "\033[0;35m"
    LMAGENTA = "\033[1;35m"
    CYAN     = "\033[0;36m"
    LCYAN    = "\033[1;36m"
    GREY     = "\033[0;37m"
    WHITE    = "\033[1;37m"
    RESET    = "\033[0m"
}

# detect compiler errors
$0~/^ERROR:/ {
    errorCount = errorCount + 1
    printf("\n%s%s\n", LRED, $0)
}
# detect compiler warnings
$0~/^WARNING:/ {
    warningCount = warningCount + 1
    printf("\n%s%s\n", LRED, $0)
}

# detect package declarations
$1~/package/ {
    packageCount = packageCount + 1
    package = $2
    lastType = "package"
    printf("\n%s[%s] ", LBLUE, $2)
}

# detect module declarations
$1~/module/ {
    moduleCount = moduleCount + 1
    module = $2
    lastType = "module"
    printf("\n%s    %s ", CYAN, $2)
}

# detect assertion results
$1~/assert/ {
    assertCount = assertCount + 1
    lastType = "assert"

    if ($2 == "#OK") {
        # successful assertion
        successCount = successCount + 1
        printf("%s.", LGREEN)
    } else {
        # failed assertion
        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $3
        failedAssertsMessage[failedCount] = $4
        failedAssertsDetails[failedCount] = $5
        failedCount = failedCount + 1

        # notify the failed assertion
        printf("%sF", LRED)
    }
}

# detect assertions counters
$1~/expect/ {
    # is failed count ?
    if ($5 != "true") {
        if ($3 == "") {
            # no expected number, try to take the actual number
            if (!$4) {
                # no actual number, assume it is at least 1
                expected = 1
            } else {
                # take the actual number as the expected number
                expected = $4
            }
        } else {
            # provided expected number
            expected = $3
        }

        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $6
        failedAssertsMessage[failedCount] = $7
        assertCount = assertCount + expected - $4
        failedCount = failedCount + 1

        # put a space after package name in case of some result will be displayed after
        if ($2 == "package") {
            printf(" ")
        }

        # if the last was a module, the result need to be displayed on a new line
        if ($2 == "module") {
            printf("\n    ")
        }

        # notify the missing assertions
        printf("%sM", LRED)
    }
}

END {
    separator = "%s-----------\n"
    printf("\n")

    if (failedCount > 0) {
        printf(separator, GREY)
        printf("%sFAILED\n", RED)
        package = ""
        context = ""

        for (i = 0; i < failedCount; i++) {
            if (package != failedAssertsPackage[i]) {
                package = failedAssertsPackage[i]
                printf("%s[%s]\n", GRAY, package)
            }
            if (failedAssertsContext[i] && context != failedAssertsContext[i]) {
                context = failedAssertsContext[i]
                printf("%s  %s\n", MAGENTA, context)
            }
            if (failedAssertsContext[i]) {
                printf("  ")
            }
            printf("%s- %s\n", RED, failedAssertsMessage[i])
            if (failedAssertsDetails[i]) {
                printf("    %s\n", failedAssertsDetails[i])
            }
            printf("\n")
        }
    }

    printf(separator, GREY)
    printf("%s%d packages\n", GREEN, packageCount)
    printf("%s%d modules\n", GREEN, moduleCount)
    printf("%s%d/%d asserts\n", GREEN, successCount, assertCount)
    printf(separator, GREY)

    if (errorCount > 0) {
        printf("%s%d errors!\n", RED, errorCount)
    }
    if (warningCount > 0) {
        printf("%s%d warnings!\n", RED, warningCount)
    }

    if (failedCount == 0) {
        printf("%sAll successful!\n", GREEN)
    } else {
        printf("%s%d failed asserts!\n", RED, failedCount)
    }
}

#
# MIT License
#
# Copyright (c) 2017 Jean-Sebastien CONAN
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

#
# Part of the camelSCAD library.
#
# Parses and displays the results of the unit tests.
#
# @package util/test
# @author jsconan
#

BEGIN {
    # will parse a CSV file
    FS = ";"

    # init counters
    packageCount = 0
    moduleCount  = 0
    assertCount  = 0
    successCount = 0
    failedCount  = 0
    warningCount = 0

    # define ANSI colors
    BLACK    = "\033[0;30m"
    GRAY     = "\033[1;30m"
    RED      = "\033[0;31m"
    LRED     = "\033[1;31m"
    GREEN    = "\033[0;32m"
    LGREEN   = "\033[1;32m"
    BROWN    = "\033[0;33m"
    YELLOW   = "\033[1;33m"
    BLUE     = "\033[0;34m"
    LBLUE    = "\033[1;34m"
    MAGENTA  = "\033[0;35m"
    LMAGENTA = "\033[1;35m"
    CYAN     = "\033[0;36m"
    LCYAN    = "\033[1;36m"
    GREY     = "\033[0;37m"
    WHITE    = "\033[1;37m"
    RESET    = "\033[0m"
}

# detect compiler warnings
$0~/^WARNING:/ {
    warningCount = warningCount + 1
    printf("\n%s%s\n", LRED, $0)
}

# detect package declarations
$1~/package/ {
    packageCount = packageCount + 1
    package = $2
    lastType = "package"
    printf("\n%s[%s] ", LBLUE, $2)
}

# detect module declarations
$1~/module/ {
    moduleCount = moduleCount + 1
    module = $2
    lastType = "module"
    printf("\n%s    %s ", CYAN, $2)
}

# detect assertion results
$1~/assert/ {
    assertCount = assertCount + 1
    lastType = "assert"

    if ($2 == "#OK") {
        # successful assertion
        successCount = successCount + 1
        printf("%s.", LGREEN)
    } else {
        # failed assertion
        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $3
        failedAssertsMessage[failedCount] = $4
        failedAssertsDetails[failedCount] = $5
        failedCount = failedCount + 1

        # notify the failed assertion
        printf("%sF", LRED)
    }
}

# detect assertions counters
$1~/expect/ {
    # is failed count ?
    if ($5 != "true") {
        if ($3 == "") {
            # no expected number, try to take the actual number
            if (!$4) {
                # no actual number, assume it is at least 1
                expected = 1
            } else {
                # take the actual number as the expected number
                expected = $4
            }
        } else {
            # provided expected number
            expected = $3
        }

        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $6
        failedAssertsMessage[failedCount] = $7
        assertCount = assertCount + expected - $4
        failedCount = failedCount + 1

        # put a space after package name in case of some result will be displayed after
        if ($2 == "package") {
            printf(" ")
        }

        # if the last was a module, the result need to be displayed on a new line
        if ($2 == "module") {
            printf("\n    ")
        }

        # notify the missing assertions
        printf("%sM", LRED)
    }
}

END {
    separator = "%s-----------\n"
    printf("\n")

    if (failedCount > 0) {
        printf(separator, GREY)
        printf("%sFAILED\n", RED)
        package = ""
        context = ""

        for (i = 0; i < failedCount; i++) {
            if (package != failedAssertsPackage[i]) {
                package = failedAssertsPackage[i]
                printf("%s[%s]\n", GRAY, package)
            }
            if (failedAssertsContext[i] && context != failedAssertsContext[i]) {
                context = failedAssertsContext[i]
                printf("%s  %s\n", MAGENTA, context)
            }
            if (failedAssertsContext[i]) {
                printf("  ")
            }
            printf("%s- %s\n", RED, failedAssertsMessage[i])
            if (failedAssertsDetails[i]) {
                printf("    %s\n", failedAssertsDetails[i])
            }
            printf("\n")
        }
    }

    printf(separator, GREY)
    printf("%s%d packages\n", GREEN, packageCount)
    printf("%s%d modules\n", GREEN, moduleCount)
    printf("%s%d/%d asserts\n", GREEN, successCount, assertCount)
    printf(separator, GREY)

    if (warningCount > 0) {
        printf("%s%d warnings!\n", RED, warningCount)
    }

    if (failedCount == 0) {
        printf("%sAll successful!\n", GREEN)
    } else {
        printf("%s%d failed asserts!\n", RED, failedCount)
    }
}