refactor URI path representation for efficency finding curie-prefix; & formating

--- scripts/ntriple2dot.awk
+++ scripts/ntriple2dot.awk
@@ -3,7 +3,7 @@
 #  Reduce the subject and object of RDF triples (ntriples format)
 #  down to their curie prefix (or literal object)
 #  Reduce predicates to the specific term identifier
-#  and the ontology they are from. (curi form)
+#  and the ontology namespace they are from. (curi prefix )
 #  Express the subject and object as nodes with a directed edge
 #  labeled with the predicate in the graphviz dot format
 #  Include a tally of each combination of nodes and edge
@@ -16,142 +16,151 @@
 #  Post processing to augment predicate identifiers
 #  with their labels seems to improve usefulness.

-
 function usage(){
-	print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
+    print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
 }

 ##########################################################
 # remove first and last chars from input string
 function trim(str){
-	return substr(str,2,length(str)-2)
-}
-
-########################################################
-# remove final(ish) element in paths with various delimiters
-# this leaves the namespace of a removed local identifier
-
-# this may need to be called more than once since
-# some identifiers may have embedded delimiters other uri use differently
-function stripid(uri,  delim, char, max, c, l){
-	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
-	if(0 < match(uri, /^_:|^https:\/\/monarchinitiative.org\/.well-known\/genid\//))
-		return "BNODE"
-	# perspective endpoints, choose the longest
-	delim["_"]=0; delim["/"]=0;
-	delim["="]=0; delim[":"]=0; delim["#"]=0;
-	char=""; max=-1;
-	for(c in delim){
-		l = match(uri, char)  # side effect is RLENGTH
-		if(l > max){
-			char = c;
-			delim[char] = RLENGTH
-		}
-	}
-	if(max<=0 || char=="")
-		# we don't know what it is, a literal perhaps or uri fragment
-		return uri
-	else
-		return substr(uri,1,delim[char])  # the probably truncated uri
-}
-
-# keep underscore, letters & numbers
+    return substr(str,2,length(str)-2)
+}
+
+# break URI into tokens to use a a path in
+# multi-dim array pointing to a curie-prefix
+function tokenize (str){
+    # b/c found in local IDs leave .:
+    gsub(/[?&:#=/_-]+/, SUBSEP, str)
+    # discard the first token it is only low info content  http|https|ftp
+    # discard any empty trailing tokens
+    tail = match(str, ".*" SUBSEP "$")  # side effects RSTART & RLENGTH
+    # print "tail = " tail " tail length = " RLENGTH
+    finish = 0
+    start = index(str, SUBSEP) + 1
+    if (tail>0){
+        finish = RLENGTH - start
+    }
+    else { # if(tail==0) {
+        # print str
+        finish = length(str) - start + 1
+    }
+
+    return substr(str, start, finish)
+}
+
+# remove final token from token path delimited with SUBSEP
+function detail(key) {
+    return substr(key, 1, match(key, SUBSEP "[[:alnum:].]+$")-1)
+}
+
+# for output, keep underscore, letters & numbers
 # change the rest to (a single) underscore sans leading & trailing
 # this passes valid node labels from dot's perspective (C-lang identifier rules)
 function simplify(str){
-	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
-	gsub(/^_*|_*$/, "",str)
-	gsub(/__*/,"_",str)
-	return str
-}
-
-# if possible, find a shorter form for the input
+    gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
+    gsub(/^_*|_*$/, "",str)
+    gsub(/__*/,"_",str)
+    return str
+}
+
+# Replace IRI with CuriePrefix when possible
+# Otherwise whinge and return a gv printable version of the original
 function contract(uri,  u){
-	u = uri
-	# shorten till longest uri in curi map is found (or not)
-
-	while(!(u in prefix) &&  length(u))
-		u = stripid(substr(u,1,length(u)-1))
-	if(u in prefix)
-		return prefix[u]
-	else{
-		# printf("WARN  %s\n", uri) > "/dev/stderr"
-		return simplify(uri)
-	}
+    u = tokenize(uri)
+    if(index(u, SUBSEP)>0)
+        u = detail(u)
+
+    if(u == "") {
+        printf("ERROR contracting %s\n", uri) > "/dev/stderr"
+        printf("ERROR tokenizeing %s\n", tokenize(uri)) > "/dev/stderr"
+        printf("ERROR detailing   %s\n", detail(tokenize(uri))) > "/dev/stderr"
+        # exit(1)  # while testing
+    }
+    # shorten till longest uri in curi map is found (or not)
+    while(!(u in prefix) &&  index(u, SUBSEP)>0)
+        u = detail(u)
+
+    if(u in prefix)
+        return prefix[u]
+    else{
+        printf("WARN  %s\n", uri) > "/dev/stderr"
+        printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
+        return simplify(uri)
+    }
 }

 # get the final (incl fragment identifier) portion of a slashed path
 function final(uri,  b, p, anchor){
-	split(uri, b, "/")
-	p = b[length(b)]
-	anchor = match(p, "#")
-	if(anchor > 0)
-		p = substr(p, anchor+1)
-	return p
+    split(uri, b, "/")
+    p = b[length(b)]
+    anchor = match(p, "#")
+    if(anchor > 0)
+        p = substr(p, anchor+1)
+    return p
 }

 # fix curie namespace when redundant due to OBO  i.e RO:RO_123
-
+# is this still needed circa 2020?
 function deoboify(curie,  a){
-	split(curie, a , ":")
-	if(match(a[2], a[1]"_") > 0){
-		sub("_", ":", a[2]);
-		curie=a[2]
-	}
-	return curie
+    split(curie, a , ":")
+    if(match(a[2], a[1]"_") > 0){
+        sub("_", ":", a[2]);
+        curie=a[2]
+    }
+    return curie
 }

 BEGIN{
-	# exceptions
-	prefix["BNODE"]="BNODE"  # is a fixed point
-	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
-	prefix["https://archive.monarchinitiative.org/"]="MonarchArchive"
-	############################################################
-	# not expected to be added to curie map
-	# in HPOA
-	prefix["http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part="]="NCBIBook"
-	prefix["http://www.ncbi.nlm.nih.gov/books/NBK"]="NCBINBK"
-
-	############################################################
-	# Often re-visit whether these exceptions are still necessary
-	# they may have been moved into the curie prefix mapping file
-	# or addressed in a local translation table
+    # exceptions
+    prefix["BNODE"]="BNODE"  # is a fixed point
+    prefix[tokenize("https://monarchinitiative.org/.well-known/genid")]="BNODE"
+    prefix[tokenize("https://archive.monarchinitiative.org/")]="MonarchArchive"
+    ############################################################
+    # not expected to be added to curie map
+    # in HPOA
+    prefix[tokenize("http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part=")]="NCBIBook"
+    prefix[tokenize("http://www.ncbi.nlm.nih.gov/books/NBK")]="NCBINBK"
+
+    ############################################################
+    # Often re-visit whether these exceptions are still necessary
+    # they may have been moved into the curie prefix mapping file
+    # or addressed in a local translation table
     # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

-	# in mgi
-	prefix["https://www.mousephenotype.org"]="IMPC"
-	# in IMPC (not httpS  --- hmm all three IRI exist)
-	prefix["http://www.mousephenotype.org"]="IMPC"
-	# note in curie_map IMPC is:
-	# http://www.mousephenotype.org/data/genes/
-
-	# in panther
-	# prefix["http://identifiers.org/wormbase/"]="WormBase"
-	# note in curie_map WormBase is
-	# 'https://www.wormbase.org/get?name='
-
-	# various, transient?
-    prefix["http://www.ncbi.nlm.nih.gov/gene/"]="NCBIGene"
-    prefix["http://www.ncbi.nlm.nih.gov/genome/"]="NCBIGenome"
-    prefix["http://www.ncbi.nlm.nih.gov/assembly?term="]="NCBIAssembly"
-	# kegg
-	prefix["http://www.genome.jp/kegg/pathway/map/map"]="KEGG-img"
-	# in EOM
-	prefix["https://elementsofmorphology.nih.gov/images/terms/"]="EOM_IMG"
-	prefix["http://elementsofmorphology.nih.gov/index.cgi?tid="]="EOM"  # w/o httpS
-
-	# playing with the idea of a LOCAL identifier
+    # in mgi
+    prefix[tokenize("https://www.mousephenotype.org")]="IMPC"
+    # in IMPC (not httpS  --- hmm all three IRI exist)
+    prefix[tokenize("http://www.mousephenotype.org")]="IMPC"
+    # note in curie_map IMPC is:
+    # http://www.mousephenotype.org/data/genes/
+
+    # in panther
+    # prefix["http://identifiers.org/wormbase/"]="WormBase"
+    # note in curie_map WormBase is
+    # 'https://www.wormbase.org/get?name='
+
+    # various, transient?
+    prefix[tokenize("http://www.ncbi.nlm.nih.gov/gene/")]="NCBIGene"
+    prefix[tokenize("http://www.ncbi.nlm.nih.gov/genome/")]="NCBIGenome"
+    prefix[tokenize("http://www.ncbi.nlm.nih.gov/assembly?term=")]="NCBIAssembly"
+    # kegg
+    prefix[tokenize("http://www.genome.jp/kegg/pathway/map/map")]="KEGG-img"
+    # in EOM
+    prefix[tokenize("https://elementsofmorphology.nih.gov/images/terms/")]="EOM_IMG"
+    prefix[tokenize("http://elementsofmorphology.nih.gov/index.cgi?tid=")]="EOM"  # w/o httpS
+
+    # playing with the idea of a LOCAL identifier
     # not the same as a bnode in that tools won't rewrite identifier
-	# and their IRI are unroutable, i.e in same machine (or local net)
-	# this is motivated in part by not wanting to publish
-	# urls to our own servers which go nowhere.
-	prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
-	prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
-	prefix["http://0.0.0.0/"]="LCL"	                       # ID halfway house
-
-	# LCL's obvious down side is zero options for global collisions
-	# best you can do is go the uuid which is ugly, or
-	# belive your obscure but readable string _is_ a special snowflake
+    # and their IRI are unroutable, i.e in same machine (or local net)
+    # this is motivated in part by not wanting to publish
+    # urls to our own servers which go nowhere.
+    # prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
+    # prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
+    # prefix["http://0.0.0.0/"]="LCL"                              # ID halfway house
+
+    # LCL's obvious down side is zero options for global collisions
+    # best you can do is go the uuid which is ugly, or
+    # belive your obscure but readable string _is_ a special snowflake
 }

 # main loop
@@ -159,93 +168,94 @@
 # YAML format is tic delimited word (the curi prefix)
 # a colon, whitespace
 # then a tic delimited url
-# (FNR == NR) && /^'[^']*': 'http[^']*'.*/ { # loosing some?
+# convert the base url to a path in a multi-dimensional array
+# (FNR == NR) && /^'[fht]\+tp[^']*'.*/ { # loosing some?
 (FNR == NR) && /^'.*/ {
-	split($0, arr, "'")
-	if(arr[2]=="")
-		arr[2]="BASE"
-	prefix[arr[4]]=arr[2]
+    split($0, arr, "'")
+    if(arr[2]=="")
+        arr[2]="BASE"
+    prefix[tokenize(arr[4])]=arr[2]
 }

 # process the ntriple file(s)  which are not the first file

 ### subject predicate and object are all uri
 (FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
-	s = contract(stripid(trim($1)))
-	p =  final(trim($2))
-	ns = contract(stripid(trim($2)))
-	o = contract(stripid(trim($3)))
-	edgelist[s,ns ":" p,o]++
-	done[NR]=1
+    s = contract(trim($1))
+    p =  final(trim($2))
+    ns = contract(trim($2))
+    o = contract(trim($3))
+    edgelist[s,ns ":" p,o]++
+    done[NR]=1
 }
 ### subject & predicate are uri but the object is a literal
 (FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
-	s = contract(stripid(trim($1)))
-	p = final(trim($2))
-	ns = contract(stripid(trim($2)))
-	edgelist[s, ns ":" p, "LITERAL"]++
-	done[NR]=1
+    s = contract(trim($1))
+    p = final(trim($2))
+    ns = contract(trim($2))
+    edgelist[s, ns ":" p, "LITERAL"]++
+    done[NR]=1
 }
 ### subject is a bare blank node, predicate and object are uri
 (FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
-	# once our bnode syntax is uniform we may want more info out of it
-	# s = contract(stripid($1))
-	p =  final(trim($2))
-	ns = contract(stripid(trim($2)))
-	### Object (like subject)
-	o = contract(stripid(trim($3)))
-	edgelist["BNODE",ns ":" p,o]++
-	done[NR]=1
+    # once our bnode syntax is uniform we may want more info out of it
+    # s = contract(($1)
+    p =  final(trim($2))
+    ns = contract(trim($2))
+    ### Object (like subject)
+    o = contract(trim($3))
+    edgelist["BNODE",ns ":" p,o]++
+    done[NR]=1
 }
 ### subject & predicate are uri & object is a bare blank node
 (FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
-	s = contract(stripid(trim($1)))
-	p =  final(trim($2))
-	ns = contract(stripid(trim($2)))
-	#o = contract(stripid($3))
-	edgelist[s,ns ":" p,"BNODE"]++
-	done[NR]=1
+    s = contract(trim($1))
+    p =  final(trim($2))
+    ns = contract(trim($2))
+    #o = contract($3))
+    edgelist[s,ns ":" p,"BNODE"]++
+    done[NR]=1
 }
 ### subject and object are a bare blank nodes, predicate is a uri
 (FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
-	# s = contract(stripid($1))
-	p =  final(trim($2))
-	ns = contract(stripid(trim($2)))
-	# o = contract(stripid($3))
-	edgelist["BNODE",ns ":" p,"BNODE"]++
-	done[NR]=1
+    # s = contract($1)
+    p =  final(trim($2))
+    ns = contract(trim($2))
+    # o = contract($3)
+    edgelist["BNODE",ns ":" p,"BNODE"]++
+    done[NR]=1
 }
 ### subject is a bare blank node, predicate is a uri & the object is a literal
 (FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
-	# s = contract(stripid($1))
-	p = final(trim($2))
-	ns = contract(stripid(trim($2)))
-	edgelist["BNODE", ns ":" p, "LITERAL"]++
-	done[NR]=1
+    # s = contract($1)
+    p = final(trim($2))
+    ns = contract(trim($2))
+    edgelist["BNODE", ns ":" p, "LITERAL"]++
+    done[NR]=1
 }

 # print anything else so it makes itself known
 (FNR != NR) && (done[NR]<1) && (NF>1){
-	printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
+    printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
 }

 # output graphviz dot file, include edge counts
 END{
-	print "digraph {"
-	print "rankdir=LR;"
-	print "charset=\"utf-8\";"
-	# sort the edges to facilitate future inter dot comparisons
-	n=asorti(edgelist, sortlist)
-	for (i=1; i<=n; i++) {
-		edge=sortlist[i]
-		split(edge ,spo, SUBSEP);
-		print simplify(spo[1]) " -> " simplify(spo[3]) \
-		" [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
-	}
-	print "LITERAL [shape=record];"
-	print "labelloc=\"t\";"
-	title = final(FILENAME)
-	datestamp = strftime("%Y%m%d", systime())
-	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
-	print "}"
-}
+    print "digraph {"
+    print "rankdir=LR;"
+    print "charset=\"utf-8\";"
+    # sort the edges to facilitate future inter dot comparisons
+    n=asorti(edgelist, sortlist)
+    for (i=1; i<=n; i++) {
+        edge=sortlist[i]
+        split(edge ,spo, SUBSEP);
+        print simplify(spo[1]) " -> " simplify(spo[3]) \
+        " [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
+    }
+    print "LITERAL [shape=record];"
+    print "labelloc=\"t\";"
+    title = final(FILENAME)
+    datestamp = strftime("%Y%m%d", systime())
+    print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
+    print "}"
+}


Deprecated functions shouldn't be in the command table. (dm)

git-svn-id: 30a5f035a20f1bc647618dbad7eea2a951b61b7c@6550 91a5dbb7-01b9-0310-9b5f-b28072856b6e


Improved bad SnEp detection

Improved bad SnEp detection

--- getMHzFromSitemap.awk
+++ getMHzFromSitemap.awk
@@ -246,7 +246,7 @@
     # Handle normal (Sn 1 Ep 1) with variations in spacing and capitalization
     # and ones missing the second letter (Sn 1 E 1), (S1 E1), or wrong second letter (Sm 1 Ep 1)
     if (match (episodeTitle,\
-        /[ ]*\(S[Nnm]*[ ]*[[:digit:]]+[ ]+[Ee][Pp]*[ ]*[[:digit:]]+[[:space:]]*\)/))  {
+        /[Ss][Nnm]*[ ]*[[:digit:]]+[ ]+[Ee][Pp]*[ ]*[[:digit:]]+[[:space:]]*\)/))  {
             if (episodeTitle !~ / \(Sn [[:digit:]]+ Ep [[:digit:]]+\)/)
                 printf ("==> Malformed Sn/Ep in \"%s: %s\"\n", showTitle, episodeTitle) >> ERRORS
             sub (/[ ]*\(S[Nnm]*[ ]*[[:digit:]]+[ ]+[Ee][Pp]*[ ]*[[:digit:]]+[[:space:]]*\)/,\


Fixed a bug where lines that didn't start with * weren't ignored.


bin/util/header2params2.awk now produces descriptions for all columns -even for raw (need metrics).

--- bin/util/header2params2.awk
+++ bin/util/header2params2.awk
@@ -25,11 +25,12 @@
 # -v person_uri

 BEGIN {
-   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
-   FS=","
+   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
+   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
+   #FS=","
    STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
    TYPE = length(conversionID) ? "Enhancement" : "Raw";
-   if(length(conversionID)) {
+   if(length(showConversionProcess)) {
       print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
       print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
    }
@@ -55,9 +56,7 @@
    print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
    print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
    print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
-   #if(showConversionProcess>0) {
-      print "@prefix ov:         <http://open.vocab.org/terms/> ."
-   #}
+   print "@prefix ov:         <http://open.vocab.org/terms/> ."
    print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

    # Converter produces URIs for the LayerDatasets:
@@ -72,8 +71,12 @@
    # Still not right; moved down to subject:
    #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

+
+   #
+   # Describe the creator.
+   #
+   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                      print
-   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
    if( length(machine_uri) && length(whoami) ) {
       if(length(person_uri)) {
                                     printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
@@ -87,35 +90,46 @@
    }else if(length(person_uri)&&length(whoami)) {
                                     printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
    }
+
+   #
+   # Describe the dataset.
+   #
                                      print
                                     printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
-                                     print "   a conversion:LayerDataset, void:Dataset;"
+                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                     printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                     printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                     printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                     printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                     #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
-   if(showConversionProcess > 0) {
+                                     print ""
                                      print "   conversion:conversion_process ["
                                     printf("      a conversion:%sConversionProcess;\n",TYPE);
-   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
+   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
+   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
    if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
+
+   #
+   # Authorship description.
+   #
                                      print
-   if(length(person_uri)&&length(machine_uri)&&length(whoami)) { # NOTE: implemented in bin/util/user-account.sh, too.
-                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami);
-   }else if(length(machine_uri) && length(whoami)) {
-                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # TODO: same as above.
-   }else if(length(person_uri)  && length(whoami)) {
+   if(length(machine_uri) && length(whoami)) {
+                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
+   }else if(length(person_uri) && length(whoami)) {
                                     printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                     printf("                        sioc:account_of <%s> ];\n",person_uri);
    }else if(length(whoami)) {
                                     printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
    }
    if(length(nowXSD)) {
-                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n",nowXSD);
+                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
    }
+
+   #
+   # Structural enhancement parameters.
+   #
    if(length(header)) {
-   # TODO: include header info even if just raw.
+                                    # Include header info even if just raw.
                                     printf("      conversion:enhance [      \n");
                                     printf("         ov:csvRow %s;\n",header);
                                     printf("         a conversion:HeaderRow;\n");
@@ -127,7 +141,7 @@
                                     printf("         a conversion:DataStartRow; \n");
                                     printf("      ];                            \n");
    }
-   if(length(interpretAsNull)) {
+   if(length(interpretAsNull) && length(conversionID)) {
                                     printf("      conversion:interpret [          \n");
                                     printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                     printf("         conversion:intepretation conversion:null; \n");
@@ -139,31 +153,36 @@
                                     printf("         a conversion:DataEndRow; \n");
                                     printf("      ];                          \n");
    }
-   } # END if(showConversionProcess > 0)
 }

-length(conversionID) { #NR == 1 && length(conversionID) {
-      label=$0;
-      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
-      print "      conversion:enhance ["
-      printf("         ov:csvCol         %s;\n",NR)
-      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
-      if(length(conversionID)) {
-         printf("         conversion:label \"%s\";\n",label);
-         printf("         conversion:comment \"\";\n");
-      }
-      printf("         conversion:range  %s:Literal;\n",RDFS); # this is either 'rdfs' or 'todo' (raw and e1, respectively)
+{ # length(conversionID) {
+   cellValue=$0;
+
+   # If we know there is no header, we can give an example value.
+   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";
+
+   print "      conversion:enhance ["
+   printf("         ov:csvCol          %s;\n",NR);
+   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
+   if(length(conversionID)) {
+      # NOTE: This MUST NOT be added to the raw interpretation parameters,
+      #       otherwise columns will collapse together prematurely (with human approval).
+      printf("         #conversion:label   \"%s\";\n",cellValue);
+   }
+   printf("         conversion:comment \"\";\n");
+   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
+   if(length(conversionID)) {
       if( length(onlyIfCol) && onlyIfCol == i ) {
          print "         a conversion:Only_if_column;"
       }
       if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
          print "         a conversion:Repeat_previous_if_empty_column;"
       }
-      print "      ];"
-   #}
+   }
+   print "      ];"
 }

 END {
-   if (showConversionProcess > 0) print "   ];"
+   print "   ];"
    printf(".");
 }


added commented-out enhancement for example resources in template creation.

--- bin/util/header2params2.awk
+++ bin/util/header2params2.awk
@@ -210,6 +210,11 @@
 }

 END {
+   print "      #conversion:enhance [            "
+   print "      #   ov:csvRow 3,4,5;             "
+   print "      #   a conversion:ExampleResource;"
+   print "      #];                              "
+
    print "   ];"
    printf(".");
 }


Revert "Fix bracket expression matches a character at gawk"

--- generateversionscript.awk
+++ generateversionscript.awk
@@ -8,7 +8,7 @@
 	gsub(/\r/,"", $0);

 	# Skip empty lines and comment lines starting with semicolon
-	if (NF && !match($0, /^[[:space:]]*;/))
+	if (NF && !match($0, /^[:space:]*;/))
 	{
 		print "        "  $0 ";";
 	}


Languages: add Lingala

Languages: add Lingala

--- include/Languages.awk
+++ include/Languages.awk
@@ -917,6 +917,19 @@
     Locale["lv"]["iso"]                = "lav"
     Locale["lv"]["glotto"]             = "latv1249"
     Locale["lv"]["script"]             = "Latn"
+
+    # Lingala
+    Locale["ln"]["name"]               = "Lingala"
+    Locale["ln"]["endonym"]            = "Lingála"
+    #Locale["ln"]["translations-of"]
+    #Locale["ln"]["definitions-of"]
+    #Locale["ln"]["synonyms"]
+    #Locale["ln"]["examples"]
+    #Locale["ln"]["see-also"]
+    Locale["ln"]["family"]             = "Atlantic-Congo"
+    Locale["ln"]["iso"]                = "lin"
+    Locale["ln"]["glotto"]             = "ling1269"
+    Locale["ln"]["script"]             = "Latn"

     #56 Lithuanian
     Locale["lt"]["name"]               = "Lithuanian"


implement wordMode and dictMode

implement wordMode and dictMode

--- include/Translate.awk
+++ include/Translate.awk
@@ -199,97 +199,114 @@
     } else {
         # Verbose mode

-        r = AnsiCode["bold"] show(join(original), il) AnsiCode["no bold"]
-        if (anything(oPhonetics))
-            r = r "\n" AnsiCode["bold"] "/" join(oPhonetics) "/" AnsiCode["no bold"]
-        r = r "\n"
-        # Dictionary mode
-        if (isarray(oWordClasses) && anything(oWordClasses)) {
-            for (i = 0; i < length(oWordClasses); i++) {
-                r = r "\n" AnsiCode["negative"] AnsiCode["bold"]        \
-                    show(oWordClasses[i], hl) AnsiCode["no bold"] AnsiCode["positive"]
-                if (isarray(oWords[i])) {
-                    # Detailed explanation
-                    for (j = 0; j < length(oWords[i]); j++) {
-                        explanation = oWords[i][j][0]
-                        ref = oWords[i][j][1]
-                        example = oWords[i][j][2]
-
-                        r = r "\n    " AnsiCode["bold"] show(explanation, il) AnsiCode["no bold"]
-                        if (example)
-                            r = r "\n    - \"" show(example, il) "\""
-                        if (ref && isarray(oRefs[ref])) {
-                            r = r "\n    * synonyms: " AnsiCode["underline"] \
-                                show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][0], il) AnsiCode["no underline"]
-                            for (k = 1; k < length(oSynonyms[oRefs[ref][1]][oRefs[ref][2]]); k++)
-                                r = r ", " AnsiCode["underline"]        \
-                                    show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][k], il) AnsiCode["no underline"]
+        if (isarray(wordClasses) && anything(wordClasses))
+            wordMode = 1
+        if (il == tl)
+            wordMode = dictMode = 1
+
+        if (wordMode) {
+            r = AnsiCode["bold"] show(join(original), il) AnsiCode["no bold"]
+            if (anything(oPhonetics))
+                r = r "\n" AnsiCode["bold"] "/" join(oPhonetics) "/" AnsiCode["no bold"]
+            r = r "\n"
+        }
+        if (dictMode) {
+            # Dictionary mode
+            if (isarray(oWordClasses) && anything(oWordClasses)) {
+                for (i = 0; i < length(oWordClasses); i++) {
+                    r = r "\n" AnsiCode["negative"] AnsiCode["bold"]    \
+                        show(oWordClasses[i], hl) AnsiCode["no bold"] AnsiCode["positive"]
+                    if (isarray(oWords[i])) {
+                        # Detailed explanation
+                        for (j = 0; j < length(oWords[i]); j++) {
+                            explanation = oWords[i][j][0]
+                            ref = oWords[i][j][1]
+                            example = oWords[i][j][2]
+
+                            r = r "\n    " AnsiCode["bold"] show(explanation, il) AnsiCode["no bold"]
+                            if (example)
+                                r = r "\n    - \"" show(example, il) "\""
+                            if (ref && isarray(oRefs[ref])) {
+                                r = r "\n    * synonyms: " AnsiCode["underline"] \
+                                    show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][0], il) AnsiCode["no underline"]
+                                for (k = 1; k < length(oSynonyms[oRefs[ref][1]][oRefs[ref][2]]); k++)
+                                    r = r ", " AnsiCode["underline"]    \
+                                        show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][k], il) AnsiCode["no underline"]
+                            }
+                            r = r "\n"
+                        }
+                    } else {
+                        # Synonyms only
+                        for (j = 0; j < length(oSynonyms[i]); j++) {
+                            r = r "\n  * " show(oSynonyms[i][j][0], il)
+                            for (k = 1; k < length(oSynonyms[i][j]); k++)
+                                r = r ", " show(oSynonyms[i][j][k], il)
                         }
                         r = r "\n"
                     }
-                } else {
-                    # Synonyms only
-                    for (j = 0; j < length(oSynonyms[i]); j++) {
-                        r = r "\n  * " show(oSynonyms[i][j][0], il)
-                        for (k = 1; k < length(oSynonyms[i][j]); k++)
-                            r = r ", " show(oSynonyms[i][j][k], il)
-                    }
+                }
+            }
+            # Examples
+            if (isarray(oExamples) && anything(oExamples)) {
+                r = r "\n" "Examples:"
+                for (i = 0; i < length(oExamples); i++) {
+                    example = oExamples[i]
+                    sub(/\u003cb\u003e/, AnsiCode["negative"], example)
+                    sub(/\u003c\/b\u003e/, AnsiCode["positive"], example)
+                    r = r "\n  - " show(example, il)
                     r = r "\n"
                 }
             }
-        }
-        # Examples
-        if (isarray(oExamples) && anything(oExamples)) {
-            r = r "\n" "Examples:"
-            for (i = 0; i < length(oExamples); i++) {
-                example = oExamples[i]
-                sub(/\u003cb\u003e/, AnsiCode["negative"], example)
-                sub(/\u003c\/b\u003e/, AnsiCode["positive"], example)
-                r = r "\n  - " show(example, il)
+            # See also
+            if (isarray(oSeeAlso) && anything(oSeeAlso)) {
+                r = r "\n" "See also:"
+                r = r "\n" AnsiCode["underline"] show(oSeeAlso[0], il) AnsiCode["no underline"]
+                for (k = 1; k < length(oSeeAlso); k++)
+                    r = r ", " AnsiCode["underline"] show(oSeeAlso[k], il) AnsiCode["no underline"]
                 r = r "\n"
             }
         }
-        # See also
-        if (isarray(oSeeAlso) && anything(oSeeAlso)) {
-            r = r "\n" "See also:"
-            r = r "\n" AnsiCode["underline"] show(oSeeAlso[0], il) AnsiCode["no underline"]
-            for (k = 1; k < length(oSeeAlso); k++)
-                r = r ", " AnsiCode["underline"] show(oSeeAlso[k], il) AnsiCode["no underline"]
+
+        if (!wordMode) {
+            r = r AnsiCode["bold"] s(translation, tl) AnsiCode["no bold"] # target language
+            if (anything(phonetics))
+                r = r "\n" AnsiCode["bold"] "/" join(phonetics) "/" AnsiCode["no bold"] # phonetic transcription
             r = r "\n"
         }
-
-
-
-        r = r "\n" AnsiCode["bold"] s(translation, tl) AnsiCode["no bold"] # target language
-        if (anything(phonetics))
-            r = r "\n" AnsiCode["bold"] join(phonetics) AnsiCode["no bold"] # phonetic transcription

         if (isarray(altTranslations[0]) && anything(altTranslations[0])) {
             # List alternative translations

             if (Locale[getCode(hl)]["rtl"] || Locale[getCode(il)]["rtl"])
-                r = r "\n\n" s(sprintf(Locale[getCode(hl)]["message"], join(original))) # caution: mixed languages, BiDi invoked must be implemented correctly (i.e. FriBidi is required)
+                r = r "\n" s(sprintf(Locale[getCode(hl)]["message"], "\"" join(original) "\"")) ":" # caution: mixed languages, BiDi invoked must be implemented correctly (i.e. FriBidi is required)
             else
-                r = r "\n\n" sprintf(Locale[getCode(hl)]["message"], join(original))
+                r = r "\n" sprintf(Locale[getCode(hl)]["message"], "\"" join(original) "\"") ":"
+
             if (Locale[getCode(il)]["rtl"] || Locale[getCode(tl)]["rtl"])
-                r = r "\n" s("(" Locale[getCode(il)]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")") # caution: mixed languages
+                r = r "\n" s("(" Locale[getCode(il)]["endonym"] " -> " Locale[getCode(tl)]["endonym"] ")") # caution: mixed languages
             else
-                r = r "\n" "(" Locale[getCode(il)]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")"
-
-            temp = segments[0] "(" join(altTranslations[0], "/") ")"
-            for (i = 1; i < length(altTranslations); i++)
-                temp = temp " " segments[i] "(" join(altTranslations[i], "/") ")"
-            if (Locale[getCode(il)]["rtl"] || Locale[getCode(tl)]["rtl"])
-                r = r "\n" AnsiCode["bold"] s(temp) AnsiCode["no bold"] # caution: mixed languages
-            else
-                r = r "\n" AnsiCode["bold"] temp AnsiCode["no bold"]
+                r = r "\n" "(" Locale[getCode(il)]["endonym"] " -> " Locale[getCode(tl)]["endonym"] ")"
+
+            if (wordMode) {
+                r = r "\n" AnsiCode["bold"] s(translation, tl) AnsiCode["no bold"] # target language
+                if (anything(phonetics))
+                    r = r "\n" AnsiCode["bold"] "/" join(phonetics) "/" AnsiCode["no bold"] # phonetic transcription
+            } else {
+                temp = segments[0] "(" join(altTranslations[0], "/") ")"
+                for (i = 1; i < length(altTranslations); i++)
+                    temp = temp " " segments[i] "(" join(altTranslations[i], "/") ")"
+                if (Locale[getCode(il)]["rtl"] || Locale[getCode(tl)]["rtl"])
+                    r = r "\n" AnsiCode["bold"] s(temp) AnsiCode["no bold"] # caution: mixed languages
+                else
+                    r = r "\n" AnsiCode["bold"] temp AnsiCode["no bold"]
+            }
         }

         if (isarray(wordClasses) && anything(wordClasses)) {
             # List dictionary entries

-            for (i = 0; i < length(words); i++) {
-                r = r "\n\n" s("[" wordClasses[i] "]", hl) # home language
+            for (i = 0; i < length(wordClasses); i++) {
+                r = r "\n\n" s(AnsiCode["negative"] wordClasses[i] AnsiCode["positive"], hl) # home language
                 for (j = 0; j < length(words[i]); j++) {
                     word = words[i][j][0]
                     explanation = join(words[i][j][1], ", ")


Fix for some variable names not being returned by get_model in fzn2py.awk

--- fzn/fzn2py.awk
+++ fzn/fzn2py.awk
@@ -251,11 +251,9 @@
 	n = asorti(output,varnames);
 	for (i=1; i<=n; i++) {
 		e = varnames[i];
-		if (output[e]!=1 || e == "objective") {
-			if(e != objective) {
-				if(length(output_vars) > 0) output_vars = output_vars ", ";
-				output_vars = output_vars e;
-			}
+		if(e != objective) {
+			if(length(output_vars) > 0) output_vars = output_vars ", ";
+			output_vars = output_vars e;
 		}
 	}
 	print "    output_vars = (" output_vars ",)";


+ Make condor_config more VM Universe aware

--- src/condor_examples/convert_config_to_win32.awk
+++ src/condor_examples/convert_config_to_win32.awk
@@ -31,6 +31,15 @@

 BEGIN {
 	FS = "="
+}
+
+# Make it so we can fill in the VM_GAHP_SERVER option if we choose to in the
+# installer. This is done before the .exe append bellow, because that match
+# seems to block any other matches for that line.
+/^#VM_GAHP_SERVER/ {
+	print "#VM_GAHP_SERVER = $(BIN)/condor_vm-gahp.exe"
+	print "VM_GAHP_SERVER = "
+	next
 }

 # Add a .exe to the end of all condor daemon names
@@ -105,6 +114,19 @@
 	next
 }

+# Make it so we can fill in the VM GAHP options if we choose to in the
+# installer.
+/^#VM_GAHP_CONFIG/ {
+	print "#VM_GAHP_CONFIG = $(RELEASE_DIR)/condor_vmgahp_config.vmware"
+	print "VM_GAHP_CONFIG ="
+	next
+}
+/^#VM_TYPE/ {
+	print "#VM_TYPE = vmware"
+	print "VM_TYPE ="
+	next
+}
+
 # If we made it here, print out the line unchanged
 {print $0}


rnc2rng.awk: Handle RELAX include directive

--- rnc2rng.awk
+++ rnc2rng.awk
@@ -201,6 +201,11 @@
     error = 1;
     exit 1
   }
+  else if (match(line, "^include ")) {
+    print "include appeared out of context on line " FNR | "cat 1>&2";
+    error = 1;
+    exit 1
+  }
   else if (match(line, /^xsd:[A-Za-z_]/)) {
     name = substr(line, 1);
     sub(/^xsd:/, "", name);
@@ -397,6 +402,7 @@
   stack_i = 0;
   paren_i = 0;
   namespaces_i = 0;
+  include_i = 0;
   error = 0;
 }

@@ -458,7 +464,8 @@
     else {
       name = substr($0, 1, nameix - 1);
     }
-    sub(/ /, "", name);
+    sub(/^ */, "", name);
+    sub(/ *$/, "", name);
     if (name == "start")
       stack[++stack_i] = "<start>"
     else {
@@ -474,6 +481,33 @@
   }
   next;
 }
+mode == "grammar" && /^include / {
+  href = substr($0, index($0, "\"") + 1);
+  brace = substr(href, index(href, "\"") + 1);
+  href = substr(href, 1, index(href, "\"") - 1);
+  sub(/^ */, "", brace);
+  sub(/ *$/, "", brace);
+  if (brace == "{") {
+    stack[++stack_i] = "<include href=\"" href "\">";
+    include_i = stack_i;
+  }
+  else {
+    stack[++stack_i] = "<include href=\"" href "\"/>";
+  }
+}
+include_i != 0 && paren_i == 0 && /}/ {
+  if (substr(stack[include_i], 1, 8) == "<include") {
+    stack[++stack_i] = "</include>";
+  }
+  else {
+    print "Mismatched parentheses on line " FNR | "cat 1>&2";
+    error = 1;
+    exit 1
+  }
+  mode = "grammar";
+  include_i = 0;
+  next;
+}
 mode == "pattern" {
   parse_pattern($0);
   next;
@@ -483,7 +517,7 @@
   next;
 }
 # Doesn't handle all comments. That would require modifying
-# parse_pattern and parse_name_class and generall being a lot
+# parse_pattern and parse_name_class and generally being a lot
 # more clever about output. But it handles comments that start
 # a line outside a pattern. Enough for me.
 /#.*/ {


Translators/BingTranslator: fix for Mongolian (Cyrillic)

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -229,6 +229,7 @@
     if (_sl == "hmn")   _sl = "mww" # Bing uses 'mww' for Hmong Daw
     if (_sl == "ku")    _sl = "kmr" # Bing uses 'kmr' for Northern Kurdish
     else if (_sl == "ckb") _sl = "ku" # and 'ku' for Central Kurdish
+    if (_sl == "mn")    _sl = "mn-Cyrl" # Bing uses 'mn-Cyrl' for Mongolian (Cyrillic)
     if (_sl == "no")    _sl = "nb"  # Bing uses Norwegian Bokmål
     # Bing uses 'pt' or 'pt-br' for Brazilian Portuguese, 'pt-pt' for European Portuguese
     if (_sl == "pt-BR") _sl = "pt" # just pt-br
@@ -239,6 +240,7 @@
     if (_tl == "hmn")   _tl = "mww"
     if (_tl == "ku")    _tl = "kmr"
     else if (_tl == "ckb") _tl = "ku"
+    if (_tl == "mn")    _tl = "mn-Cyrl"
     if (_tl == "no")    _tl = "nb"
     if (_tl == "pt-BR") _tl = "pt"
     else if (_tl == "pt-PT") _tl = "pt-pt"


su/tag_dll.awk: fixed whitespace

su/tag_dll.awk: fixed whitespace

darcs-hash:20081127232613-db55f-d8a29c04b6f8dd685f6b1cf6b7c6b07f27b416cc.gz

--- libsofia-sip-ua/su/tag_dll.awk
+++ libsofia-sip-ua/su/tag_dll.awk
@@ -105,18 +105,18 @@
   print "" > REF;
 }

-/^#define TAG_NAMESPACE/ {
-  print "#undef TAG_NAMESPACE" > REF;
-  print $0 > REF;
+/^#define TAG_NAMESPACE/ {
+  print "#undef TAG_NAMESPACE" > REF;
+  print $0 > REF;
   print "" > REF;
-  print "#undef TAG_NAMESPACE" > DLL;
-  print $0 > DLL;
+  print "#undef TAG_NAMESPACE" > DLL;
+  print $0 > DLL;
   print "" > DLL;
 }

-/SU_HAVE_EXPERIMENTAL/ {
-  print $0 > REF;
-  print $0 > DLL;
+/SU_HAVE_EXPERIMENTAL/ {
+  print $0 > REF;
+  print $0 > DLL;
 }

 !DEFS && /^tag_typedef_t/ { DEFS = 1; }
@@ -139,7 +139,7 @@
 END {
   if (LIST) {
     print "\nEXPORT tag_type_t " LIST "[] =\n{" > REF;
-    print "\nEXPORT tag_type_t " LIST "[] =\n{" > DLL;
+    print "\nEXPORT tag_type_t " LIST "[] =\n{" > DLL;
     for (tag in typedefs) {
       if (typedefs[tag] !~ /NSTAG_TYPEDEF/) {
         print "  " tag "," > REF;


added namespaces to enhance params template creation

--- bin/util/header2params2.awk
+++ bin/util/header2params2.awk
@@ -43,6 +43,7 @@
       print "@prefix time:       <http://www.w3.org/2006/time#> ."
       print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
       print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
+      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
    }
    print "@prefix void:       <http://rdfs.org/ns/void#> ."
    print "@prefix scovo:      <http://purl.org/NET/scovo#> ."


2013-06-05: parsegaspriceshtml.awk: Added substitutions for Marathon and BP/Duchess pictures.

--- parsegaspriceshtml.awk
+++ parsegaspriceshtml.awk
@@ -36,6 +36,8 @@
 }

 inGasPriceRecord && expectName && /<a href="/ {
+   gsub("","Marathon")
+   gsub("","Duchess")
    gsub("","Thorntons")
    gsub("","Thorntons")
    expectName=False


Help: display translation engine in getVersion()

--- include/Help.awk
+++ include/Help.awk
@@ -28,6 +28,7 @@
         sprintf("%-22s%s\n", "home language", Option["hl"])             \
         sprintf("%-22s%s\n", "source language", Option["sl"])           \
         sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
+        sprintf("%-22s%s\n", "translation engine", Option["engine"])    \
         sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                 "[NONE]")                                               \
         sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :


Don't fail on 'out og sequence'

Don't fail on 'out og sequence'

--- splitOnRecords.awk
+++ splitOnRecords.awk
@@ -16,7 +16,12 @@
 		active=1
 	} else {
 		print "ERROR - out of sequence!" $0 | "cat 1>&2"
-		exit
+		print "FAILED"
+		print "RECORD START" | "cat 1>&2"
+		split($0, s, "batch ");
+		printf "%s", s[2];
+		printf ", %s", jobtype
+		active=1
 	}
 }

@@ -67,7 +72,7 @@
 		#print "\tFAILED: " $0
 		printf ", FAILED\n";
 		active = 0
-		print "RECORD END" | "cat 1>&s"
+		print "RECORD END" | "cat 1>&2"
 	}
 }


Simplify toloka.awk

Simplify toloka.awk

--- misc/toloka.awk
+++ misc/toloka.awk
@@ -5,11 +5,15 @@
 }
 NR == 1 {
     for (i=1; i<=NF; i++) ix[$i] = i;
+    next;
 }
-NR > 1 && $ix["found"] {
+!$ix["found"] {
+    next;
+}
+{
     hypernyms[$ix["hypernym"] OFS $ix["genitive"]];
 }
-NR > 1 && $ix["found"] && $ix["hyponym"] != hyponym {
+$ix["hyponym"] != hyponym {
     for (hypernym_genitive in hypernyms) {
         print $ix["hyponym"], hypernym_genitive, "", "";
     }


Remove use of graphviz label coordinates for now.

--- share/bin/plain2mp.awk
+++ share/bin/plain2mp.awk
@@ -101,7 +101,7 @@

 	print "enddef;";

-	print "vardef fsmgvedge(expr tail, taildiam, head, headdiam, e, lab, lxy, loops) =";
+	print "vardef fsmgvedge(expr tail, taildiam, head, headdiam, e, lab, loops) =";
 	print "	draw e withpen pencircle scaled 0.25bp withcolor red;";

 	# TODO: better explain this
@@ -133,34 +133,24 @@
 	# TODO: new idea; draw right angle from midpoint, and use that to find intersection with b
 	# this way it should always align for multiple edges, even if on a slant

-	print "			draw lxy withpen pencircle scaled 5bp withcolor red;"
 	print "			draw point (length b / 2) of b withpen pencircle scaled 4bp withcolor blue;"
 #	print "			draw tail -- head withpen pencircle scaled 1bp withcolor green;"
 #	print "			draw ml withpen pencircle scaled 5bp withcolor green;"

-	print "			if g:"
-	print "				bl := lxy;"
-	print "			else:"
-	print "				bl := point (length b / 2) of b;"
-	print "			fi;"
+	print "			bl := point (length b / 2) of b;"

 	# TODO: plus label delta, distancing it from point. make a label function
 	# TODO: maybe an "extend" function, to extend a path in its direction. use dotprod for that?

 	# lft | rt | top | bot | ulft | urt | llft | lrt
+	# TODO: round label coordinates to grid? quantize rather
+
 	# labels ought to be below, if they're below the straight line from head-tail, or above otherwise
-	# TODO: round label coordinates to grid? quantize rather
-
-	# XXX: if using graphviz's point, label exactly on the spot
-	print "			if g:";
-	# TODO: still want to distance by +/- 4bp or so
-	print "				label(lab, bl);"
+	# TODO: find what quadrant the line is, and make this .rt/.lft instead of .top/.bot
+	print "			if ypart bl < ypart ml:" # TODO: threshold for considering "below"
+	print "				label.bot(lab, bl shifted (0, -3bp));"
 	print "			else:"
-	print "				if ypart bl < ypart ml:" # TODO: threshold for considering "below"
-	print "					label.bot(lab, bl shifted (0, -3bp));"
-	print "				else:"
-	print "					label.top(lab, bl shifted (0, +3bp));"
-	print "				fi;"
+	print "				label.top(lab, bl shifted (0, +3bp));"
 	print "			fi;"
 	print "		fi;"

@@ -271,7 +261,10 @@
 		label = "";
 	}

-	printf "fsmgvedge(%s, %s.diam, %s, %s.diam, e, \"%s\", (%fin, %fin), %s.loops);", tail, tail, head, head, label, lx, ly, head;
+	# TODO: find what time (lx,ly) is along graphviz's edge, e
+	# then pass the time and use that for placing our label on our edge
+
+	printf "fsmgvedge(%s, %s.diam, %s, %s.diam, e, \"%s\", %s.loops);", tail, tail, head, head, label, head;

 	if (head == tail) {
 		print "	%s.loops := %s.loops + 1;", head, head


Delete space between function name and '('.

--- CASE-MAPPING.awk
+++ CASE-MAPPING.awk
@@ -41,9 +41,9 @@
 /^[^\#]/ {
   if (FILENAME == "UNIDATA/SpecialCasing.txt" && NF == 5) {
     printf ("0x%s ( ", $1);
-    mtext ($2);
-    mtext ($3);
-    mtext ($4);
+    mtext($2);
+    mtext($3);
+    mtext($4);
     printf (")\n");
     X[$1] = 1;
   }


MONDRIAN: Integrate new checkFile rule from //open/util/bin/checkFile.awk.

[git-p4: depot-paths = "//open/mondrian/": change = 13379]

--- bin/checkFile.awk
+++ bin/checkFile.awk
@@ -356,6 +356,9 @@
         error(fname, FNR, "operator '" a[1] "' must not be at end of line");
     }
 }
+match(s, /^ *(=) /, a) {
+    error(fname, FNR, "operator '" a[1] "' must not be at start of line");
+}
 s ~ /\<[[:digit:][:lower:]][[:alnum:]_]*</ {
     # E.g. "p<" but not "Map<"
     if (!matchFile(fname)) {}


already imported modules are OK

already imported modules are OK

comment better

--- tests/filter.awk
+++ tests/filter.awk
@@ -1,8 +1,14 @@
+# negative tests
+#
 /fails as expected\.$/ {next}
+
+# importing related
+#
 /successfully loaded$/ {next}
 /^ *<-.*loaded\.$/ {next}
-/^->Loading import/ {next}
+/^ *->Loading import/ {next}
 /^loading / {next}
+/^ *Import .* already loaded.$/ {next}

 # empty lines are dull
 #


don't use -1 as exit code

don't use -1 as exit code

--- include/Main.awk
+++ include/Main.awk
@@ -10,7 +10,7 @@
     split(PROCINFO["version"], group, ".")
     if (group[1] < 3 || (group[1] == 3 && group[2] < 1)) {
         e("[ERROR] Oops! Your gawk (version " GawkVersion ") appears to be too old.\nYou need at least gawk 3.1 to run this script.")
-        exit -1
+        exit 1
     }
 }


Simplify logic for ensuring minimum hex length in rctocpp

Simplified the logic for ensuring that a hex string has 8 characters after
the '0x' prefix. Also clarified a comment about potential loss of numeric
precision when using numbers as indices in awk arrays.

--- src/dlls/mscorrc/rctocpp.awk
+++ src/dlls/mscorrc/rctocpp.awk
@@ -15,17 +15,11 @@
     quotient = number;

     hexString = "";
-    while (quotient > 0)
+    for (digitCount = 0; digitCount < 8; digitCount++)
     {
         remainder = quotient % 16;
         quotient = int(quotient / 16);
         hexString = sprintf("%x"hexString, remainder);
-    }
-
-    lengthOfHexString = length(hexString);
-    if (lengthOfHexString < 8)
-    {
-        hexString = substr("00000000", 1, 8 - lengthOfHexString) hexString;
     }

     hexString = "0x" hexString;
@@ -56,8 +50,9 @@
     numEntries++;

     # Use the string representation of the ID as the array index
-    # because the precision of numeric indices can be lost when
-    # iterating over the array in our for-in loop.
+    # because the precision of numeric indices can be lost during
+    # the number -> string -> number conversions that would occur
+    # if numeric indices are used.
     resourceArray[numberToHexString(id)] = str;
 }


Problem: rec-viewer.awk fails on Linux

Problem: rec-viewer.awk fails on Linux

Error message:
```
/usr/bin/env: awk -f: No such file or directory
```

Solution: don't use `/usr/bin/env` in the shebang,
revert to `#!/usr/bin/awk -f`.

--- bin/rec-viewer.awk
+++ bin/rec-viewer.awk
@@ -1,4 +1,4 @@
-#!/usr/bin/env awk -f
+#!/usr/bin/awk -f

 {
     printf("----- %d -----\n", NR);


Fix compare output in regards to capture log section

--- build/CompareOutput.awk
+++ build/CompareOutput.awk
@@ -27,7 +27,7 @@
 $0 ~ /^debug core: rejected policy .* next dissector$/ { next; }
 $0 ~ /^debug core: applying( anonymous)? policy .* next dissector$/ { next; }
 $0 ~ /^debug time: / { next; }
-$0 ~ /^info pcap: progress/ { next; }
+$0 ~ /^info capture: progress/ { next; }
 $0 ~ /^debug core: memory report/ { next; }
 $0 ~ /^warn core: rule .*at .* uses 'hook' keyword which is deprecated and should be replaced by 'on'$/ { next; }


add indels, more betas

add indels, more betas


Patch from Stefan Allius:

Patch from Stefan Allius:

    Hi Erik,

    I patched the initfini.awk script, so we can build crt[in].S for the SuperH
    targets with gmon-support.

    The patch moves all labels into the wright sections, and add some assembler
    statements to jump over them. (only activ on SH targets)

    The patch also avoid any references of _GLOBAL_OFFSET_TABLE_ or *gmon_start*
    in the crtn.S file. This should work on every target.

    Bye Stefan

--- extra/scripts/initfini.awk
+++ extra/scripts/initfini.awk
@@ -12,7 +12,6 @@
   system("/bin/rm -f crt[in].S");
   omitcrti=0;
   omitcrtn=0;
-  do_sh_specials = 0;
   glb_idx = 0;
   while(getline < "initfini.S")
   { if(/\.endp/) {endp=1}
@@ -20,22 +19,37 @@
     if(/\.align/) {alignval=$2}
 # here comes some special stuff for the SuperH targets
 #  We search for all labels, which uses the _GLOBAL_OFFSET_TABLE_
-#  definition, and store them in the glb_label array.
-    if(/_GLOBAL_OFFSET_TABLE_/) {
-      glb_label[glb_idx] = last;
-      glb_idx += 1;
-      glb = $0;
+#  or a call_gmon_start function reference, and store
+#  them in the glb_label array.
+    if(/_init_EPILOG_BEGINS/) {glb_idx=1;glb_idx_arr[glb_idx]=0}
+    if(/_fini_EPILOG_BEGINS/) {glb_idx=2;glb_idx_arr[glb_idx]=0}
+    if(/EPILOG_ENDS/)         {glb_idx=0}
+    if(/_GLOBAL_OFFSET_TABLE_/||/call_gmon_start/) {
+      glb_label[glb_idx,glb_idx_arr[glb_idx]] = last;
+      glb_idx_arr[glb_idx] += 1;
+      glb_label[glb_idx,glb_idx_arr[glb_idx]] = $0;
+      glb_idx_arr[glb_idx] += 1;
     }
     last = $1;
   }
   close("initfini.S");
 }
 # special rules for the SuperH targets (They do nothing on other targets)
-/SH_GLB_BEGINS/ && glb_idx==0 {omitcrti +=1;do_sh_specials++}
-/_init_SH_GLB/  && glb_idx>=1 {print glb_label[0] glb >> "crti.S"}
-/_fini_SH_GLB/  && glb_idx>=2 {print glb_label[1] glb >> "crti.S"}
-/SH_GLB_ENDS/   && glb_idx==0 {omitcrti -=1}
-/SH_GLB/ || /_GLOBAL_OFFSET_TABLE_/ && do_sh_specials>=1 {getline}
+/SH_GLB_BEGINS/ && glb_idx_arr[1]==0 && glb_idx_arr[2]==0 {omitcrti +=1}
+/_init_SH_GLB/  {glb_idx=1}
+/_fini_SH_GLB/  {glb_idx=2}
+/SH_GLB_ENDS/ {omitcrti -=1}
+/SH_GLB/ \
+{
+  if (glb_idx>0)
+    {
+      for (i=0;i<glb_idx_arr[glb_idx];i+=1) {
+	print glb_label[glb_idx,i] >> "crti.S";
+      }
+      glb_idx = 0;
+    }
+  next;
+}
 # special rules for H8/300 (sorry quick hack)
 /.h8300h/ {end=0}

@@ -54,6 +68,12 @@
 /GMON_STUFF_PAUSES/{omitcrtn=0;getline}
 /GMON_STUFF_UNPAUSES/{omitcrtn=1;getline}
 /GMON_STUFF_ENDS/{omitcrtn=0;getline}
+
+/_GLOBAL_OFFSET_TABLE_/||/gmon_start/ \
+{
+  if(omitcrti==0) {print >> "crti.S";}
+  next;  # no gmon_start or GLOBAL_OFFSET_TABLE references in crtn.S
+}

 /END_INIT/ \
 { if(endp)


cfg80211: parse dfs region for internal regdb option

Add support for parsing and setting the dfs region (ETSI, FCC, JP)
when the internal regulatory database is used. Before this
the DFS region was being ignored even if present on the used
db.txt

Signed-off-by: Janusz Dziedzic <a4201e08a607951e17dcd5a575c0fe6bceb34e87@tieto.com>
Reviewed-by: Luis R. Rodriguez <d4fa18192c5a479e564d29c2300c2e15afd1d8d7@do-not-panic.com>
Signed-off-by: Johannes Berg <bff32994ff0f8d048f262a8388145a71b6071bfe@intel.com>

--- net/wireless/genregdb.awk
+++ net/wireless/genregdb.awk
@@ -46,6 +46,12 @@
 	sub(/:/, "", country)
 	printf "static const struct ieee80211_regdomain regdom_%s = {\n", country
 	printf "\t.alpha2 = \"%s\",\n", country
+	if ($NF ~ /DFS-ETSI/)
+		printf "\t.dfs_region = NL80211_DFS_ETSI,\n"
+	else if ($NF ~ /DFS-FCC/)
+		printf "\t.dfs_region = NL80211_DFS_FCC,\n"
+	else if ($NF ~ /DFS-JP/)
+		printf "\t.dfs_region = NL80211_DFS_JP,\n"
 	printf "\t.reg_rules = {\n"
 	active = 1
 	regdb = regdb "\t&regdom_" country ",\n"


Include header files whose path begins with ../ .

git-svn-id: 12ae5aeef08fd453d75833463da4cc20df82a94e@2159 8af8166b-12a6-a448-a533-9086ace3f9f6


Add missing } and fix indentation of last line

--- jgivengherkin.awk
+++ jgivengherkin.awk
@@ -1,4 +1,5 @@
 #!/usr/local/bin/gawk -f
+
 BEGIN {
   INDENTATION = "    ";
   prev = "";
@@ -109,6 +110,7 @@
 }

 END {
-  print prev ";"
+  print INDENTATION prev ";"
+  print INDENTATION "}"
   print "}"
 }


gawk is awkward

gawk is awkward

--- build.awk
+++ build.awk
@@ -146,6 +146,9 @@
         while (getline line < fileName) {
             match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
             if (RSTART) { # @include
+                if (group[1] ~ /\.awk$/)
+                    append(Includes, group[1])
+
                 if (ret) ret = ret RS
                 ret = ret readSqueezed(group[1], squeezed)
             } else if (!squeezed || line = squeeze(line)) { # effective LOC
@@ -173,15 +176,6 @@
     # Default target: bash
     if (!target) target = "bash"

-    # Rebuild EntryScript
-    findIncludes(EntryPoint)
-    print "#!/bin/sh" > EntryScript
-    print "TRANS_DIR=`dirname $0`" > EntryScript
-    print "gawk \\" > EntryScript
-    for (i = 0; i < length(Includes) - 1; i++)
-        print "-i \"${TRANS_DIR}/" Includes[i] "\" \\" > EntryScript
-    print "-f \"${TRANS_DIR}/" Includes[i] "\" -- \"$@\"" > EntryScript
-
     ("mkdir -p " parameterize(BuildPath)) | getline

     if (target == "bash" || target == "zsh") {
@@ -220,6 +214,15 @@
         print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

         ("chmod +x " parameterize(Trans)) | getline
+
+        # Rebuild EntryScript
+        print "#!/bin/sh" > EntryScript
+        print "TRANS_DIR=`dirname $0`" > EntryScript
+        print "gawk \\" > EntryScript
+        for (i = 0; i < length(Includes) - 1; i++)
+            print "-i \"${TRANS_DIR}/" Includes[i] "\" \\" > EntryScript
+        print "-f \"${TRANS_DIR}/" Includes[i] "\" -- \"$@\"" > EntryScript
+        ("chmod +x " parameterize(EntryScript)) | getline
         return 0

     } else if (target == "awk" || target == "gawk") {


less chatty mostly

less chatty mostly

--- scripts/ntriple2dot.awk
+++ scripts/ntriple2dot.awk
@@ -28,12 +28,14 @@

 # break URI into tokens to use a a path in
 # multi-dim array pointing to a curie-prefix
+# base URI include "!#&-=?_"
+# which are all also present in localID as well
 function tokenize (str){
     # b/c found in local IDs leave .:
-    gsub(/[?&:#=/_-]+/, SUBSEP, str)
+    gsub(/[!&:#=?/_-]+/, SUBSEP, str)
     # discard the first token it is only low info content  http|https|ftp
     # discard any empty trailing tokens
-    tail = match(str, ".*" SUBSEP "$")  # side effects RSTART & RLENGTH
+    tail = match(str, ".*" SUBSEP)  # side effects RSTART & RLENGTH
     # print "tail = " tail " tail length = " RLENGTH
     finish = 0
     start = index(str, SUBSEP) + 1
@@ -48,11 +50,11 @@
 }

 # remove final token from token path delimited with SUBSEP
-# this ends up matching chars included in local identifiers
+# this ends up matching chars included in localID
 # ._-;():%#,?=@*\&+   (yea ... nope)

 function detail(key,   start) {
-    start = match(key, ".+" SUBSEP)
+    start = match(key, ".+" SUBSEP)
     if(start == 1)
         return substr(key, 1, RLENGTH -1)
     else
@@ -90,7 +92,7 @@
         return prefix[u]
     else{
         printf("WARN  %s\n", uri) > "/dev/stderr"
-        printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
+        # printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
         return simplify(uri)
     }
 }


Add gensub examples #chunks

Add gensub examples #chunks

--- chunks/chunks.awk
+++ chunks/chunks.awk
@@ -4,6 +4,10 @@
 match($0, /^mix ([^ ]+) +# +(.*)/, arr) {
   print arr[1] "\t" arr[2]
 }
+
+filename=gensub(" ", "-", "g", $0)
+# only once
+filename=gensub(" ", "-", 1, $0)

 # https://www.gnu.org/software/gawk/manual/html_node/String-Functions.html
 #


change -d to -D as shorthand for option -debug

--- include/Main.awk
+++ include/Main.awk
@@ -116,8 +116,8 @@
             exit
         }

-        # -d, -debug
-        match(ARGV[pos], /^--?d(e(b(ug?)?)?)?$/)
+        # -D, -debug
+        match(ARGV[pos], /^--?(debug|D)$/)
         if (RSTART) {
             Option["debug"] = 1
             continue


Cleanup diagnostic print statement comments

--- getBBoxCatalogFromSitemap.awk
+++ getBBoxCatalogFromSitemap.awk
@@ -215,9 +215,7 @@
             title = "Porridge (2016-2017)"
         }
         # print "==> title = " title > "/dev/stderr"
-        # print "==> contentId = " contentId > "/dev/stderr"
         # print "==> EntityId = " EntityId > "/dev/stderr"
-        # print "---" > "/dev/stderr"
     }

     # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
@@ -247,10 +245,7 @@
             title = "Maigret (2016–2017)"
         }
         # print "==> title = " title > "/dev/stderr"
-        # print "==> originalDate = " originalDate > "/dev/stderr"
-        # print "==> contentId = " contentId > "/dev/stderr"
         # print "==> EntityId = " EntityId > "/dev/stderr"
-        # print "---" > "/dev/stderr"
     }

     if (EntityId == "") {


Typo.

Typo.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@31414 4f837ed2-42f5-46e7-a7a5-fa17313484d4

--- scripts/tdb/statuses_gln.awk
+++ scripts/tdb/statuses_gln.awk
@@ -36,7 +36,7 @@
           r[pn] = "4"
 #        } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Pz")) {
 #          r[pn] = "5"
-        } else if (substr(p[pn],1,1) >= "IA") {
+        } else if (substr(p[pn],1,1) >= "Ia") {
           r[pn] = "8"
         }
       }


Script: upgrade(): change "download-from" address to HTTPS

--- include/Script.awk
+++ include/Script.awk
@@ -60,7 +60,7 @@
     if (newerVersion(newVersion, Version)) {
         w("Current version: \t" Version)
         w("New version available: \t" newVersion)
-        w("Download from: \t" "http://www.soimort.org/translate-shell/trans")
+        w("Download from: \t" "https://www.soimort.org/translate-shell/trans")
     } else {
         w("Current version: \t" Version)
         w("Already up-to-date.")


LanguageHelper: scriptName(): add Cans

LanguageHelper: scriptName(): add Cans

--- include/LanguageHelper.awk
+++ include/LanguageHelper.awk
@@ -128,6 +128,7 @@
     case "Arab": return "Arabic"
     case "Armn": return "Armenian"
     case "Beng": return "Bengali"
+    case "Cans": return "Canadian Aboriginal Syllabics"
     case "Cher": return "Cherokee"
     case "Cyrl": return "Cyrillic"
     case "Deva": return "Devanagari"


[as] Fix list of arguments in gen.awk

[as] Fix list of arguments in gen.awk

Only the last argument was printed to the intermediate
file because the concatenation was lost.

--- as/target/gen.awk
+++ as/target/gen.awk
@@ -67,11 +67,11 @@
 	for (i in args) {
 		a = args[i]
 		if (match(a, /^imm8/)) {
-			out = "AIMM8"
+			out = out "AIMM8"
 		} else if (match(a, /^imm16/)) {
-			out = "AIMM16"
+			out = out "AIMM16"
 		} else if (match(a, /^imm32/)) {
-			out = "AIMM32"
+			out = out "AIMM32"
 		} else if (match(a, /^imm64/)) {
 			out = "AIMM64"
 		} else {


fix for invalid FILETRANSFER_PLUGINS in default config on windows #4012 ===VersionHistory:Pending===

--- src/condor_examples/convert_config_to_win32.awk
+++ src/condor_examples/convert_config_to_win32.awk
@@ -37,27 +37,50 @@
 # installer. This is done before the .exe append bellow, because that match
 # seems to block any other matches for that line.
 /^#VM_GAHP_SERVER/ {
-	print "#VM_GAHP_SERVER = $(BIN)/condor_vm-gahp.exe"
+	print "#VM_GAHP_SERVER = $(BIN)\\condor_vm-gahp.exe"
 	print "VM_GAHP_SERVER = "
 	next
 }
-
-# Add a .exe to the end of all condor daemon names in $(BIN) and $(SBIN)
-/BIN)\/condor_/ {
-	printf "%s.exe\n",$0
-	next
-}
-
-# Add a .exe to the end of all condor daemon names in $(LIBEXEC)
-/LIBEXEC)\/condor_/ {
-	printf "%s.exe\n",$0
+# We DONT want to add a .exe extension to the vmware script, it's a perl script.
+/VMWARE_SCRIPT[ \t]*=/ {
+	gsub(/\//, "\\", $0)
+	printf "%s\n",$0
+	next
+}
+
+# Remove SHADOW_STANDARD (the standard universe shadow)
+#
+/^SHADOW_STANDARD/ {
+	next
+}
+/^SHADOW_LIST/ {
+    printf "SHADOW_LIST = SHADOW\n"
+    next
+}
+
+# Remove STARTER_STANDARD (the standard universe starter)
+#
+/^STARTER_STANDARD/ {
+    next
+}
+/^STARTER_LIST/ {
+    printf "STARTER_LIST = STARTER\n"
+    next
+}
+
+/^FILETRANSFER_PLUGINS[ \t]*=/ {
+	gsub(/\//, "\\", $0)
+	gsub(/_plugin,/, "_plugin.exe,", $0)
+	gsub(/_plugin$/, "_plugin.exe", $0)
+	printf "#%s\n",$0
+	printf "FILETRANSFER_PLUGINS = $(LIBEXEC)\\data_plugin.exe\n";
 	next
 }

 # Email Settings
 # Use condor_mail.exe for MAIL, and put in a stub for SMTP_SERVER
 /^MAIL/ {
-	printf "MAIL = $(BIN)/condor_mail.exe\n\n"
+	printf "MAIL = $(BIN)\\condor_mail.exe\n\n"
 	printf "## For Condor on Win32 we need to specify an SMTP server so\n"
 	printf "## that Condor is able to send email.\n"
 	printf "SMTP_SERVER =\n\n"
@@ -71,26 +94,6 @@
 	next
 }

-# Remove SHADOW_STANDARD (the standard universe shadow)
-#
-/^SHADOW_STANDARD/ {
-	next
-}
-/^SHADOW_LIST/ {
-    printf "SHADOW_LIST = SHADOW\n"
-    next
-}
-
-# Remove STARTER_STANDARD (the standard universe starter)
-#
-/^STARTER_STANDARD/ {
-    next
-}
-/^STARTER_LIST/ {
-    printf "STARTER_LIST = STARTER\n"
-    next
-}
-
 # Choose a Windows-reasonable value for RELEASE_DIR and LOCAL_DIR
 #
 /^RELEASE_DIR[ \t]*=/ {
@@ -143,8 +146,8 @@
 }

 # Clean out the initial guess of /usr/bin/java for JAVA... ;)
-# Note the space after JAVA below, so we do not change other JAVA settings.
-/^JAVA / {
+# wherever it is on Windows, it's not there.
+/^JAVA[ \t]*=/ {
 	printf "JAVA =\n"
 	next
 }
@@ -227,5 +230,36 @@
 	next
 }

+# Add a .exe to the end of all condor daemon names in $(BIN) and $(SBIN)
+/BIN)\/condor_/ {
+	gsub(/\//, "\\", $0)
+	printf "%s.exe\n",$0
+	next
+}
+
+# Add a .exe to the end of all condor daemon names in $(LIBEXEC)
+/LIBEXEC)\/condor_/ {
+	gsub(/\//, "\\", $0)
+	printf "%s.exe\n",$0
+	next
+}
+
+# convert paths relative to LOCAL_DIR, RELEASE_DIR and LOG to windows path separators
+/[ \t]*=[ \t]*\$\(LOCAL_DIR\)\// {
+	gsub(/\//, "\\", $0)
+	printf "%s\n",$0
+	next
+}
+/[ \t]*=[ \t]*\$\(RELEASE_DIR\)\// {
+	gsub(/\//, "\\", $0)
+	printf "%s\n",$0
+	next
+}
+/[ \t]*=[ \t]*\$\(LOG\)\// {
+	gsub(/\//, "\\", $0)
+	printf "%s\n",$0
+	next
+}
+
 # If we made it here, print out the line unchanged
 {print $0}


Theme: make random theme work again

Theme: make random theme work again

--- include/Theme.awk
+++ include/Theme.awk
@@ -118,7 +118,8 @@

 # Set theme using corresponding option.
 function setTheme(    file, line, script) {
-    if (Option["theme"] && Option["theme"] != "default") {
+    if (Option["theme"] && Option["theme"] != "default"                 \
+        && Option["theme"] != "none" && Option["theme"] != "random") {
         file = Option["theme"]
         if (!fileExists(file)) {
             file = ENVIRON["HOME"] "/.translate-shell/" Option["theme"]


LanguageHelper: improve getDetails() for writing systems

--- include/LanguageHelper.awk
+++ include/LanguageHelper.awk
@@ -180,6 +180,25 @@
     return r
 }

+# Return the regions that a script is written in, as an English string.
+function getWrittenIn(code,    i, j, r, regions, str) {
+    r = NULLSTR
+    str = Locale[getCode(code)]["written-in"]
+    if (str) {
+        split(str, regions, /\s?;\s?/)
+        j = 0
+        for (i in regions) {
+            r = r regions[i]
+            j++
+            if (j < length(regions) - 1)
+                r = r ", "
+            else if (j == length(regions) - 1)
+                r = r " and "
+        }
+    }
+    return r
+}
+
 # Return 1 if a language is supported by Google; otherwise return 0.
 function isSupportedByGoogle(code,    engines, i, str) {
     str = Locale[getCode(code)]["supported-by"]
@@ -203,7 +222,7 @@
 }

 # Return detailed information of a language as a string.
-function getDetails(code,    article, desc, group, iso, script) {
+function getDetails(code,    article, desc, group, iso, name, names, script, writing) {
     if (code == "auto" || !getCode(code)) {
         e("[ERROR] Language not found: " code "\n"                      \
           "        Run '-reference / -R' to see a list of available languages.")
@@ -212,22 +231,34 @@

     script = scriptName(getScript(code))
     if (isRTL(code)) script = script " (R-to-L)"
+
     split(getISO(code), group, "-")
     iso = group[1]
+
+    name = getName(code)
+    names = getNames(code)
+    if (match(name, /\(.*\)/)) {
+        writing = substr(name, match(name, /\(.*\)/) + 1)
+        writing = substr(writing, 1, length(writing) - 1)
+        name = substr(name, 1, match(name, /\(.*\)/) - 2)
+    }

     if (getBranch(code)) {
         article = match(tolower(getBranch(code)), /^[aeiou]/) ? "an" : "a"
         desc = sprintf("%s is %s %s language spoken mainly in %s.",
-                       getNames(code), article, getBranch(code), getSpokenIn(code))
+                       names, article, getBranch(code), getSpokenIn(code))
     } else if (getFamily(code) == NULLSTR || tolower(getFamily(code)) == "language isolate") {
         desc = sprintf("%s is a language spoken mainly in %s.",
-                       getNames(code), getSpokenIn(code))
+                       names, getSpokenIn(code))
     } else
         desc = sprintf("%s is a language of the %s family, spoken mainly in %s.",
-                       getNames(code), getFamily(code), getSpokenIn(code))
+                       names, getFamily(code), getSpokenIn(code))
+    if (writing && getWrittenIn(code))
+        desc = desc sprintf(" The %s writing system is officially used in %s.",
+                            tolower(writing), getWrittenIn(code))

     return ansi("bold", sprintf("%s\n", getDisplay(code)))              \
-        sprintf("%-22s%s\n", "Name", ansi("bold", getNames(code)))      \
+        sprintf("%-22s%s\n", "Name", ansi("bold", names))               \
         sprintf("%-22s%s\n", "Family", ansi("bold", getFamily(code)))   \
         sprintf("%-22s%s\n", "Writing system", ansi("bold", script))    \
         sprintf("%-22s%s\n", "Code", ansi("bold", getCode(code)))       \


Utils: add a short note for dump()

Utils: add a short note for dump()

--- include/Utils.awk
+++ include/Utils.awk
@@ -155,6 +155,7 @@
 }

 # Dump a Unicode string into a byte array. Return the length of the array.
+# NOTE: can only be ran once for each text! Build a cache.
 function dump(text, group,    command, temp) {
     command = "hexdump" " -e'1/1 \"%03d\" \" \"'"
     ("echo " parameterize(text) PIPE command) | getline temp


Fix bugs

Fix bugs

--- scripts/rules2flows.awk
+++ scripts/rules2flows.awk
@@ -22,6 +22,6 @@
 		l4_match = l4_match","metadata
 	    }
     }
-    str = "{"table_id","priority","cookie","actions",\"match\":{\"in_port\":1,"dl_type","ip_src","ip_dst""l4_match"}}";
+    str = "{"table_id","priority","cookie","actions",\"match\":{"dl_type","ip_src","ip_dst""l4_match"}}";
     print str;
 }


Reset the len value

Reset the len value

--- impl/delabel.awk
+++ impl/delabel.awk
@@ -11,6 +11,8 @@

     for (i = 1; i <= len; i++) uniq[words[i]];

+    len = 0;
+
     for (word in uniq) {
         synset = synset sep word;
         sep = ", ";


dynamic regexp

dynamic regexp

Effective awk programming : universal text processing and pattern matching / Arnold Robbins

--- REGEX/txt.awk
+++ REGEX/txt.awk
@@ -1 +1,5 @@
 echo aaaabcd | awk '{ sub(/a+/, "<A>"); print }'
+
+
+BEGIN { digits_regexp = "[[:digit:]]+" }
+$0 ~ digits_regexp    { print }


Indent line after } else {

Indent line after } else {

--- getMHzFrom-episodePages.awk
+++ getMHzFrom-episodePages.awk
@@ -167,7 +167,7 @@
             if (episodeType == "T") {
                 episodeDescription = episodeTitle
             } else {
-            print "==> No description: " shortURL >> ERROR_FILE
+                print "==> No description: " shortURL >> ERROR_FILE
             }
         }
         sub (/^PR \| /,"",episodeDescription)


Utils: fix curl()

Utils: fix curl()

--- include/Utils.awk
+++ include/Utils.awk
@@ -136,7 +136,7 @@
         command = Curl " --location --silent " url
         content = NULLSTR
         while ((command |& getline line) > 0)
-            content = content line "\n"
+            content = (content ? content "\n" : NULLSTR) line
         return content
     }
 }


space/tab

space/tab

--- scripts/label_dot_edge.awk
+++ scripts/label_dot_edge.awk
@@ -34,7 +34,7 @@
 		for(i=2;i<=length(term);i++)
 			row[4] = row[4] "\"" term[i]
 	}
-    line = row[1]
+	line = row[1]
 	for(i=2;i<=length(row);i++)
 		line = line " " row[i]
 	print  line


added dgtwc: http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#  to enhancement template maker

--- bin/util/header2params2.awk
+++ bin/util/header2params2.awk
@@ -58,6 +58,7 @@
       print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
       print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
       print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
+      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
    }
    print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
    print "@prefix void:          <http://rdfs.org/ns/void#> ."


Bell-Labs AWK does not support POSIX "bracket expressions" (POSIXese for
"character classes", basically).  So change them to their character
representation.

Submitted by:	David Wolfskill <david@catwhisker.org>


Commons: bold red error e()

Commons: bold red error e()

--- include/Commons.awk
+++ include/Commons.awk
@@ -269,7 +269,7 @@

 # Print error message.
 function e(text) {
-    print ansi("red", text) > "/dev/stderr"
+    print ansi("bold", ansi("red", text)) > "/dev/stderr"
 }

 # Print debugging message.


fix condition in iteration getReply

fix condition in iteration getReply

--- gawk/ch04/ch04.awk
+++ gawk/ch04/ch04.awk
@@ -164,6 +164,6 @@
       zremrangebyrank(p,"viewed:"token, 0, -26)
       zincrby(p,"viewed:", -1, item)
     }
-    for(ERRNO="" ; ERRNO="" ; var=getReply(p,REPLY)) ;
-    ERRNO==""
+    for(ERRNO="" ; ERRNO=="" ; getReply(p,REPLY))
+      ;
 }


Fix ofmtstrnum

Fix ofmtstrnum

This is a weird one: Go splitting on an empty regex gives the individual
characters in the string (like JavaScript). AWK and Python give the
original string. So change the regex to "x" to mimic the latter behavior.

--- testdata/gawk/ofmtstrnum.awk
+++ testdata/gawk/ofmtstrnum.awk
@@ -1,5 +1,5 @@
 BEGIN {
-	split(" 1", f, "|")
+	split(" 1", f, "x")
 	OFMT = "%.1f"
 	print f[1]
 	x = f[1] + 0


Generalize series mismatch fix

Generalize series mismatch fix

--- getAcornFrom-seriesPages.awk
+++ getAcornFrom-seriesPages.awk
@@ -243,7 +243,7 @@
     # The season number should match that in the URL
     # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, and Vera have known problems
     if (episodeURL ~ \
-           /\/birdsfeather\/series[0-9]{1,2}\/|\/docmartin\/series[0-9]{1,2}\/|\/murdoch\/season[0-9]{1,2}|\/poirot\/series[0-9]{1,2}\/|\/rebus\/series[0-9]{1,2}\/|\/vera\/series[0-9]{1,2}\//) {
+           /\/series[0-9]{1,2}\/|\/murdoch\/season[0-9]{1,2}/) {
         split (episodeURL, part, "/")
         URLseasonNumber = part[5]
         sub (/christmasspecial/,"",URLseasonNumber)


Replace GNU-specific character classes with MAWK-compatible expressions.

--- awkgward/awkgward.awk
+++ awkgward/awkgward.awk
@@ -40,8 +40,8 @@

 function create_guard_macro(prefix, filename)
 {
+    double_underscore = toupper(filename);
     # just in case, double single underscores
-    double_underscore = toupper(filename);
     gsub(/_/, "__", double_underscore);
     # \/ = slash (/), \. = dot (.), \\ = backslash (\) - replace them with underscore.
     gsub(/[\/\.\\]/, "_", double_underscore);
@@ -83,7 +83,7 @@
     state = S_START;
 }

-/^#ifndef[[:space:]].*/ {
+/^#ifndef[\t ].*/ {
     if (state == S_START) {
         ifndef_buf = $0;
         state = S_FIRST_IFNDEF;
@@ -91,7 +91,7 @@
     }
 }

-/^#define[[:space:]].*/ {
+/^#define[\t ].*/ {
     if (state == S_FIRST_IFNDEF) {
         #eat old guard and replace it with new one
         print_or_buf(create_guard(prefix, FILENAME));
@@ -102,7 +102,7 @@
         state = S_END;
 }

-/^#endif[[:space:]].*/ {
+/^#endif[\t ].*/ {
     #flush old buffer and whatever was in endif line
     if (global_buf_mode) {
         print endif_line;


addr_utils.awk		-- added code to upcase 1st letter of words in street -- deals el Camino, de las Pulgas, etc

--- lib/addr_utils.awk
+++ lib/addr_utils.awk
@@ -26,7 +26,6 @@
 		}
 	}

-	# TODO: move this to a separate function, may AU_expand()
 	if(!options["rply"]){
 		# As a convenience to the user, given that nearly all the addresses
 		# are in small set of nearby towns, the minimal query address can be
@@ -115,22 +114,16 @@
 		return 1
 	}

-	# Street Hacks: Probably should just compare ignore case but not ready for that step yet
-	# change el Camino X to El Camino X
-	# change el Monte X  to El Monte X
-	if(options["do_subs"]){
-		if(nf2 > 3){
-			if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Camino")
-				ary2[nf2-2] = "El"
-			else if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Monte")
-				ary2[nf2-2] =  "El"
-		}
+	# If a word in street begins w/lower case, upper case it.  Convert, say el Camino to El Camino, etc
+	for(i = 2; i <= nf2; i++){
+		if(substr(ary2[i], 1, 1) ~ /^[a-z]/)
+			ary2[i] = toupper(substr(ary2[i], 1, 1)) substr(ary2[i], 2)
 	}

 	# Street should be num [ dir ] str [ st ] [ dir ]
 	street = ary2[1]
 	if(options["do_subs"]){
-		# This is amusing.  What is South Court? Is is S. Court or South Ct.  No idea so leave such streets in long form
+		# This is amusing.  What is South Court? Is it S. Court or South Ct.  No idea so leave such streets in long form
 		if(nf2 == 3 && (ary2[2] in dirs) && (ary2[3] in st_types)){
 			for(i = 2; i <= nf2; i++)
 				street = street " " ary2[i]
@@ -159,7 +152,6 @@
 			street = street " " ary2[i]
 	}

-	# TODO: this belongs in a separate function
 	if(options["rply"]){
 		if(town in towns)
 			town = towns[town]


parsegaspriceshtml.awk: Count nested <tr> for inGasPriceRecord. (Fix for UDF.)

--- parsegaspriceshtml.awk
+++ parsegaspriceshtml.awk
@@ -41,16 +41,18 @@
    gsub("","Thorntons")
    gsub("","Certified")
    gsub("","Marathon")
-   # gsub("","UDF")
+   gsub("","UDF")
    expectName=False
    sub("[ ]*$","")
    sub("^.*>","")
    sub("&","\\&")
+   # print "name", i, name[i], $0
    name[i]=$0
    # print
 }

 inGasPriceRecord && / class="p_area">/ {
+   # print "area", i, name[i], $0
    sub("^.* class=\"p_area\">","")
    sub("[ ]*$","")
    area[i]=$0
@@ -69,11 +71,18 @@
    # print
 }

+inGasPriceRecord && /
/ {
+   inGasPriceRecord += 1
+}
+
 inGasPriceRecord && /<\/tr>/ {
-   i ++
-   # print "EOR"
-   # print $0
-   inGasPriceRecord=False
+   inGasPriceRecord -= 1
+   if (!inGasPriceRecord) {
+      i ++
+      # print "EOR"
+      # print $0
+      inGasPriceRecord=False
+   }
 }

 END {


Do not use sort -s in lexicon.awk

Do not use sort -s in lexicon.awk

--- lexicon.awk
+++ lexicon.awk
@@ -7,5 +7,5 @@
 }
 {
     gsub(/, /, ORS, $3);
-    print $3 | "sort --parallel=$(nproc) -S1G -us";
+    print $3 | "sort --parallel=$(nproc) -S1G -u";
 }


SImplify first awk script

SImplify first awk script

--- getMhzFrom-browsePage.awk
+++ getMhzFrom-browsePage.awk
@@ -1,33 +1,36 @@
-# Given the output of:
-#    curl -s https://mhzchoice.vhx.tv/series https://mhzchoice.vhx.tv/series?page=2
-# create lists of titles, urls and seasons to be pasted into a spreadsheet
+# Generate series URLs, Titles, Number of Seasons from MHz "Browse" page
 #
-# e.g.
-# A French Village
-#
6 seasons

+# INVOCATION:
+#       curl -s https://mhzchoice.vhx.tv/series https://mhzchoice.vhx.tv/series?page=2 \
+#           | awk -v URL_FILE=$URL_FILE -v TITLE_FILE=$TITLE_FILE \
+#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -f getMHzFrom-browsePage.awk
 #
+# INPUT:
+#       A French Village
+#   ---
+#
6 seasons

 #
-# Invoked with file parameters as follows:
-# awk -v URL_FILE=$URL_FILE -v TITLE_FILE=$TITLE_FILE \
-#     -v SEASONS_FILE=$SEASONS_FILE -f fetchMHz-series.awk
+# OUTPUT:
+#       $URL_FILE, $TITLE_FILE, $NUM_SEASONS_FILE

 /https:\/\/mhzchoice.vhx.tv\/.*strong title/ {
-    sub (/.*href="/,"")
-    endTitle = match($0,/"/ ) -1
-    print substr($0,1,endTitle) "/" >> URL_FILE
+    split($0,fld,"\"")
+    URL = fld[2]
+    TITLE = fld[4]

-    sub (/.*strong title="/,"")
-    sub (/".*$/,"")
-    gsub (/'/,"'")
-    if (match ($0, /^The /)) {
-        $0 = substr($0, 5) ", The"
+    print URL "/" >> URL_FILE
+
+    # Canonicalize Title
+    gsub (/'/,"'",TITLE)
+    if (match (TITLE, /^The /)) {
+        TITLE = substr(TITLE, 5) ", The"
     }
-    print >> TITLE_FILE
+    print TITLE >> TITLE_FILE
 }

 /h4 class="transparent.*media-count/ {
     sub (/.*media-count'>/,"")
     sub (/ season.*$/,"")
-    print $0 >> SEASONS_FILE
+    print $0 >> NUM_SEASONS_FILE
 }


Translators/BingTranslator: fix bingTTSUrl() (new API, #218)


Translators/BingTranslator: support phonetics (transliteration)


Allow testing of back content

Allow testing of back content

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@23463 4f837ed2-42f5-46e7-a7a5-fa17313484d4

--- scripts/tdb/statuses_clockss.awk
+++ scripts/tdb/statuses_clockss.awk
@@ -1,5 +1,5 @@
 #!/usr/bin/awk -f
-# input: publisher\tplugin\tcontract_year\tyear\ttester\tstatus
+# cat *.tdb | tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back]

 BEGIN {
   FS="\t"
@@ -10,10 +10,15 @@
   # add a loop to add line only if ending year is gt or eq to contract year
   end_year = 0
   incontract = 0
+  if ($7 > 1900) {
+    test_year = $7
+  } else {
+    test_year = $3
+  }
   if (length($4) > 3) {
     end_year = substr($4,length($4)-3,4)
   }
-  if (end_year >= $3) {
+  if (end_year >= test_year) {
     incontract = 1
   }

@@ -21,17 +26,18 @@
   if (incontract == 1) {
       nn = split($2,na,/\./)
       lp2 = na[nn]
-    if (!(($1,lp2,$3,$4) in b)) {
+    if (!(($1,lp2,$3,$7,$4) in b)) {
       p[pn] = $1
       n[pn] = lp2
   #    n[pn] = $2
       t[pn] = $3
+      k[pn] = $7
       d[pn] = $4
       r[pn] = $5
       pn++
     }
-    b[$1,lp2,$3,$4]++
-    c[$1,lp2,$3,$4,$6]++
+    b[$1,lp2,$3,$7,$4]++
+    c[$1,lp2,$3,$7,$4,$6]++
     x[$6]++
     tt++
   }
@@ -65,7 +71,7 @@
   scn = 11

   #print out header
-  printf "Publisher\tPlugin\tContr\tYear\tT\tTotal"
+  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tTotal"
   for (j = 0 ; j < scn ; j++) {
     if (x[s[j]] > 0) {
     printf "\t%s", sc[j]
@@ -73,22 +79,22 @@
   }
   printf "\n"

-  #print out publisher, plugin, contract year, year, total aus
+  #print out publisher, plugin, contract year, back, year, tester, total aus
   for (i = 0 ; i < pn ; i++) {
-    printf "%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], d[i], r[i], b[p[i],n[i],t[i],d[i]]
+    printf "%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],t[i],d[i]]
     for (j = 0 ; j < sn ; j++) {
       if (x[s[j]] > 0){
-      if (c[p[i],n[i],t[i],d[i],s[j]] == 0) {
+      if (c[p[i],n[i],t[i],k[i],d[i],s[j]] == 0) {
       printf "\t.."
     } else {
-        printf "\t%d", c[p[i],n[i],t[i],d[i],s[j]]
+        printf "\t%d", c[p[i],n[i],t[i],k[i],d[i],s[j]]
       }
     }
     }
     printf "\n"
   }
     #print out bottom line sums
-    printf "Publisher\tPlugin\tContr\tYear\tT\t%d", tt
+    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\t%d", tt
     for (j = 0 ; j < sn ; j++) {
       if (x[s[j]] > 0) {
         printf "\t%d", x[s[j]]


improve utility when not processing dipper ingest files

--- scripts/deltadot.awk
+++ scripts/deltadot.awk
@@ -34,6 +34,8 @@
     n = match($0, / [(][[:digit:]]*[)]>];$/)
     if(n>0)
         arr[substr($0,1,n)] = substr($0, n+2, length($0)-n-5)
+	else
+		arr[$1 " " $2 " " $3 " "]++
 }

 # strip metadata from filename
@@ -74,7 +76,7 @@
 	}
 	# only edges not in first file
 	for(e in edge2)
-		edges[e "(" edge2[e] ")>, color=\"blue\"];"];
+		edges[e "(" edge2[e] ")>, color=\"blue\"];"];  # expects label=<tag> to close

 	print "digraph {"
 	print "rankdir=LR;"


INTEGRATION: CWS i18n28 (1.3.178); FILE MERGED 2006/12/08 15:53:16 er 1.3.178.1: #i64400# add getBreakIteratorRules_

--- i18npool/source/localedata/data/linkermapfile-check.awk
+++ i18npool/source/localedata/data/linkermapfile-check.awk
@@ -16,6 +16,7 @@
     sPublic[nPublics++] = "getAllCalendars_"
     sPublic[nPublics++] = "getAllCurrencies_"
     sPublic[nPublics++] = "getAllFormats_"
+    sPublic[nPublics++] = "getBreakIteratorRules_"
     sPublic[nPublics++] = "getCollationOptions_"
     sPublic[nPublics++] = "getCollatorImplementation_"
     sPublic[nPublics++] = "getContinuousNumberingLevels_"


Add missing field

Add missing field

--- getBritBoxFrom-scrapy_xml.awk
+++ getBritBoxFrom-scrapy_xml.awk
@@ -97,6 +97,7 @@
     seasonNumber = $11
     seasonYear = $15
     seasonEpisodes = $19
+    seasonDescription = $23

     if (seasonNumber != "") {
         printf \


Fix an output formatting bug in the rewriter script.

--- log_rewriter.awk
+++ log_rewriter.awk
@@ -6,11 +6,13 @@
 {
         split($1, DATE, "/")
         split($2, TIME, ":")
-        $1 = ""
-        $2 = ""
         t = mktime(DATE[1] " " DATE[2] " " DATE[3] " " TIME[1] " " TIME[2] " " TIME[3])
         if (delta == "") {
                 delta = now - t
         }
-        print strftime(format, t + delta) $0
+        out = strftime(format, t + delta)
+        for (i = 3; i <= NF; i++) {
+                out = out OFS $i
+        }
+        print out
 }


Languages: add Dzongkha language

Languages: add Dzongkha language

--- include/Languages.awk
+++ include/Languages.awk
@@ -78,6 +78,19 @@
     Locale["co"]["iso"]                = "cos"
     Locale["co"]["glotto"]             = "cors1242"
     Locale["co"]["script"]             = "Latn"
+
+    #? Dzongkha
+    Locale["dz"]["name"]               = "Dzongkha"
+    Locale["dz"]["endonym"]            = "རྫོང་ཁ"
+    #Locale["dz"]["translations-of"]
+    #Locale["dz"]["definitions-of"]
+    #Locale["dz"]["synonyms"]
+    #Locale["dz"]["examples"]
+    #Locale["dz"]["see-also"]
+    Locale["dz"]["family"]             = "Sino-Tibetan"
+    Locale["dz"]["iso"]                = "dzo"
+    Locale["dz"]["glotto"]             = "nucl1307"
+    Locale["dz"]["script"]             = "Tibt"

     #1 Afrikaans
     Locale["af"]["name"]               = "Afrikaans"


Commons: add getGitHead()

Commons: add getGitHead()

--- include/Commons.awk
+++ include/Commons.awk
@@ -350,14 +350,29 @@
     return (prog " " arg SUPERR | getline) ? prog : NULLSTR
 }

-# Return non-zero if file exists, otherwise return 0.
+# Return non-zero if file exists; otherwise return 0.
 function fileExists(file) {
     return !system("test -f " parameterize(file))
 }

-# Return non-zero if file exists and is a directory, otherwise return 0.
+# Return non-zero if file exists and is a directory; otherwise return 0.
 function dirExists(file) {
     return !system("test -d " parameterize(file))
+}
+
+# Return the HEAD revision if the current directory is a git repo;
+# Otherwise return a null string.
+function getGitHead(    line, group) {
+    if (fileExists(".git/HEAD")) {
+        getline line < ".git/HEAD"
+        match(line, /^ref: (.*)$/, group)
+        if (fileExists(".git/" group[1])) {
+            getline line < (".git/" group[1])
+            return substr(line, 1, 7)
+        } else
+            return NULLSTR
+    } else
+        return NULLSTR
 }

 # Initialize `UriSchemes`.


parse_address.awk	-- superceded by addr_utils.awk

--- lib/parse_address.awk
+++ lib/parse_address.awk
@@ -1,165 +0,0 @@
-function parse_address(addr, result, dirs, st_abbrevs, st_quals, towns,   nf, ary, i, f_st, f_town, name) {
-
-	result["status"] = "B"
-	result["emsg"  ] = ""
-	result["name"  ] = ""
-	result["street"] = ""
-	result["town"  ] = ""
-	result["state" ] = ""
-
-	nf = split(addr, ary, ",")
-	for(i = 1; i <= nf; i++){
-		gsub(/^  */, "", ary[i])
-		gsub(/  *$/, "", ary[i])
-	}
-	f_st = get_street(nf, ary, result, dirs, st_abbrevs)
-	if(f_st == 0)
-		return 1
-
-	f_town = get_town(nf, ary, result, f_st, st_quals, towns)
-	if(f_town == 0)
-		return 1
-
-	# If we get here, addr is good, set the name
-	result["status"] = "G"
-	if(f_st == 1)
-		result["name"] = "Residence"
-	else{
-		name = ary[1]
-		for(i = 2; i < f_st; i++){
-			name = name ", " ary[i]
-		}
-		result["name"] = name
-	}
-	return 0
-}
-function get_street(nf, ary, result, dirs, st_abbrevs,    f_st, i, street, l_street, ab, l_ab, ix, nf2, ary2, work) {
-
-	# find the street part of the address
-	f_st = 0
-	if(nf == 1){	# minimum address is street, town
-		result["status"] = "B"
-		result["emsg"] = "no town"
-		return 0
-	}else{
-		# street is the first field that matches one of these patterns
-		for(i = 1; i < nf; i++){
-			if(ary[i] ~ /^[1-9][0-9]* /){	# Simple number prefix
-				f_st = i
-				break
-			}else if(ary[i] ~ /^[1-9][0-9]*-[1-9]/){	# Number range prefix
-				f_st = i
-				break
-			}else if(ary[i] ~ /^[1-9][0-9]*[a-zA-Z] /){	# Number + 1 letter
-				f_st = i
-				break
-			}
-		}
-		# ERROR: No street found
-		if(f_st == 0){
-			result["status"] = "B"
-			result["emsg"] = "street?"
-			return 0
-		}
-	}
-
-	# ERROR: f_st is last field: no town
-	if(f_st == nf){
-		result["status"] = "B"
-		result["emsg"] = "f_st=$"
-		return 0
-	}
-
-	# replace str abbrev with their long forms
-	street = ary[f_st]
-	l_street = length(street)
-	for(ab in st_abbrevs){
-		l_ab = length(ab)
-		if(l_ab > l_street)
-			continue
-		ix = l_street - l_ab + 1
-		if(substr(street, ix) == ab){
-			ary[f_st] = substr(street, 1, ix - 1) st_abbrevs[ab]
-			break
-		}
-	}
-
-	# replace single letter directions with their long forms
-	nf2 = split(ary[f_st], ary2, /  */)
-	work = ""
-	for(i = 1; i <= nf2; i++){
-		ary2[i] = ary2[i] in dirs ? dirs[ary2[i]] : ary2[i]
-		work = work == "" ? ary2[i] : work " " ary2[i]
-	}
-	result["street"] = ary[f_st] = work
-	return f_st
-}
-function get_town(nf, ary, result, f_st, st_quals, towns,   i, f_town) {
-
-	# find the town:
-	# 1. Set any street qualifiers (Apt 8, Bldg 2, etc) to ""
-	# 2. The first field that is not "" is the town.
-	for(i = f_st + 1; i <= nf; i++){
-		ary[i] = rm_st_qual(ary[i], st_quals)
-	}
-	f_town = 0
-	for(i = f_st + 1; i <= nf; i++){
-		if(ary[i] != ""){
-			f_town = i
-			break
-		}
-	}
-	if(f_town == 0){				# no town
-		result["status"] = "B"
-		result["emsg"  ] = "town?"
-		return 0
-	}
-
-	# handle possible CA zip
-	if(f_town == nf){				# town is last field, look closer
-		if(ary[f_town] == "CA 94305"){		# Stanford is special
-			ary[f_town] = "Stanford University"
-		}else if(ary[f_town] ~ /^CA 9/){	# town cannot be a a zip code
-			result["status"] = "B"
-			result["emsg"  ] = "town=zip"
-			return 0
-		}
-	}else if(f_town == nf - 1){			# town followed by 1 more field, look closer
-		if(ary[nf] ~ /^CA [0-9]/){		# town followed by state zip, drop it
-			ary[nf] = ""
-		}else if(ary[nf] == "CA"){		# town followed by state, drop this field
-			ary[nf] = ""
-		}else{					# no idea
-			result["status"] = "B"
-			result["emsg"  ] = "town2?"
-			return 0
-		}
-	}else{ 						# town followed by at least 2 fields, who knows?
-		result["status"] = "B"
-		result["emsg"  ] = "town3?"
-		return 0
-	}
-
-	# Good to go
-	# Replace abbreviated town w/full name
-	ary[f_town] = (ary[f_town] in towns ? towns[ary[f_town]] : ary[f_town])
-	result["town"] = ary[f_town]
-	result["state"] = "CA"	# for now
-
-	return f_town
-}
-function rm_st_qual(str, st_quals,   l_str, lc_str, u) {
-
-	if(str ~ /^[#1-9]/)
-		return ""
-	l_str = length(str)
-	lc_str = tolower(str)
-	for(u in st_quals){
-		l_u = length(u)
-		if(l_str < l_u)
-			continue
-		if(substr(lc_str, 1, l_u) == u)
-			return ""
-        }
-	return str
-}


Translate: getResponse() returns json only (fix #64)

--- include/Translate.awk
+++ include/Translate.awk
@@ -74,7 +74,7 @@

     print header |& HttpService
     while ((HttpService |& getline) > 0) {
-        if (length($0) > 1) content = $0
+        if ($0 ~ /^\[.*\]/) content = $0
         l(sprintf("%4s bytes > %s", length($0), $0))
     }
     close(HttpService)


Change letter ranges.

Change letter ranges.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@26755 4f837ed2-42f5-46e7-a7a5-fa17313484d4


Remove unnecessary leading zeros from the generated Java constants. (dm)

git-svn-id: 30a5f035a20f1bc647618dbad7eea2a951b61b7c@2661 91a5dbb7-01b9-0310-9b5f-b28072856b6e

--- Bindings/Java/constants.awk
+++ Bindings/Java/constants.awk
@@ -95,7 +95,8 @@
 function hexadecimalValue(value) {
   value = tolower(value)
   gsub("u?l*$", "", value)
-  gsub("x0+", "x0", value)
+  gsub("x0+", "x", value)
+  gsub("x$", "x0", value)
   return value
 }


Translators/Apertium: unquote before invoking uprintf()

--- include/Translators/Apertium.awk
+++ include/Translators/Apertium.awk
@@ -64,7 +64,7 @@
     }

     # FIXME: build a cache for recurring invocation of uprintf()
-    translation = uprintf(unparameterize(ast[0 SUBSEP "responseData" SUBSEP "translatedText"]))
+    translation = uprintf(unquote(unparameterize(ast[0 SUBSEP "responseData" SUBSEP "translatedText"])))

     returnIl[0] = il = _sl
     if (Option["verbose"] < 0)


bump version to 0.9.1-dev

bump version to 0.9.1-dev

--- metainfo.awk
+++ metainfo.awk
@@ -1,7 +1,7 @@
 BEGIN {
     Name        = "Translate Shell"
     Description = "Google Translate to serve as a command-line tool"
-    Version     = "0.9.0.6"
+    Version     = "0.9.1-dev"
     ReleaseDate = "2015-05-29"
     Command     = "trans"
     EntryPoint  = "translate.awk"


Comma caution.

Comma caution.

--- src/cloudfront.awk
+++ src/cloudfront.awk
@@ -38,6 +38,7 @@

 /^#Version: / {
     printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
+    logLineSeen = 0
 }

 /^#Fields: / {
@@ -51,13 +52,21 @@
 }

 ! /^#/ {
+    if (logLineSeen == 1) {
+        printf ",\n"
+    }
     split($0, fieldsThisLine, " ")
     if (NF == numFields) {
         print "\t{"
         for(i = 1; i < NF; i++)
         {
-            printf "\t\t\"%s\": \"%s\",\n", fields[i], fieldsThisLine[i]
+            printf "\t\t\"%s\": \"%s\"", fields[i], fieldsThisLine[i]
+            if (i + 1 < NF) {
+                printf ","
+            }
+            printf "\n"
         }
-        print "\t},"
+        printf "\t}"
     }
+    logLineSeen = 1;
 }


make better gv file

make better gv file


Add show durations

Add show durations

--- getAcornFrom-showPages.awk
+++ getAcornFrom-showPages.awk
@@ -99,6 +99,27 @@
     next
 }

+# Extract the episode duration
+/<meta itemprop="timeRequired"/ {
+    durationLinesFound += 1
+    split ($0,fld,"\"")
+    split (fld[4],tm,/[TMS]/)
+    secs = tm[3]
+    mins = tm[2] + int(secs / 60)
+    hrs =  int(mins / 60)
+    secs %= 60; mins %= 60
+    #
+    showSecs += secs
+    showMins += mins + int(showSecs / 60)
+    showHrs += hrs + int(showMins / 60)
+    showSecs %= 60; showMins %= 60
+    #
+    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
+    if (episodeDuration == "00:00:00")
+        printf ("==> Blank episode duration: %s  %s\n", shortURL, showTitle) >> ERRORS
+    next
+}
+
 /<footer>/ {
     if (episodeLinesFound == 0) {
         printf ("==> No numberOfEpisodes: %s\t%s\n", shortURL, showTitle) >> ERRORS
@@ -109,10 +130,13 @@
     if (descriptionLinesFound == 0) {
         printf ("==> No franchise-description: %s\t%s\n", shortURL, showTitle) >> ERRORS
     }
+    if (durationLinesFound == 0) {
+        printf ("==> No durations: %s\t%s\n", shortURL, showTitle) >> ERRORS
+    }
     showLink = "=HYPERLINK(\"" showURL "\";\"" showTitle "\")"
+    showDuration = sprintf ("%02d:%02d:%02d", showHrs, showMins, showSecs)
     # Title	Seasons	Episodes	Duration	Description
-    printf ("%s\t%s\t=%s\t%s\n", showLink, showSeasons, seasonEpisodes, showDescription)
-    # printf ("%s\t%s\t=%s\t%s\t%s\n", showLink, showSeasons, seasonEpisodes, showDuration, showDescription)
+    printf ("%s\t%s\t=%s\t%s\t%s\n", showLink, showSeasons, seasonEpisodes, showDuration, showDescription)
     # printf ("%s\t%s\t=%s\t%s\t%s\n", showLink, showSeasons, showEpisodes, showDuration, showDescription)
     # Make sure there is no carryover
     showTitle = ""
@@ -122,6 +146,9 @@
     showSeasons = ""
     showEpisodes = ""
     seasonEpisodes = ""
+    showSecs = 0
+    showMins = 0
+    showHrs = 0
     showDuration = ""
     showDescription = ""
     #


Bugfix

Bugfix


Fix tests

Fix tests

Due to some formating changes, the output comparison was not working.

--- build/CompareOutput.awk
+++ build/CompareOutput.awk
@@ -22,8 +22,11 @@
 }

 {
-	if (show != 0) {
+	if (show == 2) {
 		print;
+	}
+	else if (show == 1) {
+		show = 2
 	}
 }


Correct error in number variables printed

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@23949 4f837ed2-42f5-46e7-a7a5-fa17313484d4

--- scripts/tdb/statuses_clockss.awk
+++ scripts/tdb/statuses_clockss.awk
@@ -81,7 +81,7 @@

   #print out publisher, plugin, contract year, back, year, tester, total aus
   for (i = 0 ; i < pn ; i++) {
-    printf "%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],t[i],d[i]]
+    printf "%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],t[i],k[i],d[i]]
     for (j = 0 ; j < sn ; j++) {
       if (x[s[j]] > 0){
       if (c[p[i],n[i],t[i],k[i],d[i],s[j]] == 0) {


Translators/BingTranslator: fix #110

Translators/BingTranslator: fix #110

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -2,15 +2,15 @@
 # BingTranslator.awk                                               #
 ####################################################################
 #
-# Last Updated: 26 Dec 2015
-# https://www.bing.com/translator/dynamic/226010/js/LandingPage.js
+# Last Updated: 11 Mar 2016
+# http://ssl.microsofttranslator.com/dynamic/226010/js/LandingPage.js
 BEGIN { provides("bing") }

 function genRTTAppId(    content, group, header, isBody) {
     HttpProtocol = "http://"
-    HttpHost = "www.bing.com"
+    HttpHost = "ssl.microsofttranslator.com"
     HttpPort = 80
-    LandingPage = "/translator/dynamic/226010/js/LandingPage.js"
+    LandingPage = "/dynamic/226010/js/LandingPage.js"

     if (Option["proxy"]) {
         match(Option["proxy"], /^(http:\/*)?([^\/]*):([^\/:]*)/, HttpProxySpec)


Remove reorder.awk script

Remove reorder.awk script

--- reorder.awk
+++ reorder.awk
@@ -1,39 +0,0 @@
-#!/usr/bin/awk -f
-BEGIN { FS=","; OFS=","; }
-
-(FNR > 1) {
-	stdCov[$1] = $4;
-	dynCov[$1] = $14;
-	caddyCov[$1] = $24;
-
-	stdPapiTime[$1] = $5;
-	dynPapiTime[$1] = $15;
-	caddyPapiTime[$1] = $25;
-
-	stdTime[$1]= $6;
-	dynTime[$1]= $16;
-	caddyTime[$1]= $26;
-
-	stdCalls[$1] = $10;
-	dynCalls[$1] = $20;
-	caddyCalls[$1] = $30;
-}
-
-END {
-	printf "%s, %s, %s, %s, %s, %s\n",
-		"Name", "Class", "ExecCov", "Time", "PapiTime", "Calls";
-
-	for (p in stdCov) {
-		printf "%s, %s, %s, %s, %s, %s\n",
-			p, "Static", stdCov[p], stdTime[p], stdPapiTime[p], stdCalls[p];
-
-		printf "%s, %s, %s, %s, %s, %s\n",
-		    p, "Algebraic", caddyCov[p], caddyTime[p], caddyPapiTime[p], caddyCalls[p];
-
-		printf "%s, %s, %s, %s, %s, %s\n",
-			p, "Dynamic" , dynCov[p], dynTime[p], dynPapiTime[p], dynCalls[p];
-
-		#printf "%s, %s, %s, %s, %s, %s\n",
-		#	p, "Raw"     , "0", rawTime[p], "0", "0";
-	}
-}


Support perforce-style diffs in determining modified line ranges.

PiperOrigin-RevId: 304687721

--- verilog/tools/formatter/diff-to-changed-lines.awk
+++ verilog/tools/formatter/diff-to-changed-lines.awk
@@ -65,11 +65,20 @@
   active_range = 0;
 }

+# seen in 'git diff'
 # diff before-file after-file
 /^diff/ {
   auto_flush_range();
   expect_filename = 1;
 }
+
+# seen in 'p4 diff'
+# ==== //depot/...#N - /local/path/to/file ====
+/^==== .* ====$/ {
+  auto_flush_range();
+  expect_filename = 1;
+}
+

 # + lines are after change only
 /^+/ {


Added from to the prints.

Added from to the prints.

--- timeCalculator.awk
+++ timeCalculator.awk
@@ -1,11 +1,17 @@
 #![$2] += $NF }
 $0 ~ "^Time edu.berkeley.cs.amplab.carat." {
-  sums[$2] += $NF
-  count[$2] += 1
+  k = $2
+  if ($0 ~ "^Time edu.berkeley.cs.amplab.carat.* from ")
+    k = $2" "$4
+  sums[k] += $NF
+  count[k] += 1
 }
 $0 ~ "^Time spark.timeseries." {
-  sums[$2] += $NF
-  count[$2] += 1
+  k = $2
+  if ($0 ~ "^Time spark.timeseries.* from ")
+    k = $2" "$4
+  sums[k] += $NF
+  count[k] += 1
 }

 END {