Help: fix getReference()

Help: fix getReference()

--- include/Help.awk
+++ include/Help.awk
@@ -192,100 +192,114 @@
 #     displayName = "endonym" or "name"
 function getReference(displayName,
                       ####
-                      code, col, cols, i, j, name, r, rows, saveSortedIn, t, tt) {
-    rows = int(length(Locale) / 3) + 1
+                      code, col, cols, i, j, name, num, r, rows, saveSortedIn,
+                      t1, t2) {
+    # number of language codes with stable support
+    num = 0
+    for (code in Locale)
+        if (Locale[code]["support"] != "unstable")
+            num++
+    rows = int(num / 3) + 1
     cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
     i = 0
     saveSortedIn = PROCINFO["sorted_in"]
     PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
         "compName"
     for (code in Locale) {
-        col = int(i / rows)
-        append(cols[col], code)
-        i++
+        # show languages only with stable support
+        if (Locale[code]["support"] != "unstable") {
+            col = int(i / rows)
+            append(cols[col], code)
+            i++
+        }
     }
     PROCINFO["sorted_in"] = saveSortedIn

     if (displayName == "endonym") {
-        r = "┌" replicate("─", 22) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
+        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
         for (i = 0; i < rows; i++) {
-            r = r "│ "
+            r = r "│"
             for (j = 0; j < 3; j++) {
                 if (cols[j][i]) {
-                    t = cols[j][i] == "bo" ||
-                        cols[j][i] == "dz" ||
-                        cols[j][i] == "he" ||
-                        cols[j][i] == "hi" ||
-                        cols[j][i] == "hu" ||
-                        cols[j][i] == "la" ||
-                        cols[j][i] == "ml" ||
-                        cols[j][i] == "mn" ||
-                        cols[j][i] == "ne" ||
-                        cols[j][i] == "ny" ||
-                        cols[j][i] == "pa" ||
-                        cols[j][i] == "pl" ||
-                        cols[j][i] == "ro" ||
-                        cols[j][i] == "sr" ||
-                        cols[j][i] == "te" ||
-                        cols[j][i] == "tg" ||
-                        cols[j][i] == "tr" ||
-                        cols[j][i] == "ur" ||
-                        cols[j][i] == "wo" ||
-                        cols[j][i] == "yi" ||
-                        cols[j][i] == "yo" ||
-                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW" &&
-                         length(getEndonym(cols[j][i])) < 6) ? "\t\t " :
-                        cols[j][i] == "id" ? "" :
-                        cols[j][i] == "haw" ? " " :
-                        "\t "
-                    tt = length(cols[j][i]) == 3 ? " │" :
-                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW") ? "  │" : ""
-                    r = r getDisplay(cols[j][i]) t "- " ansi("bold", cols[j][i]) tt " "
+                    t1 = getDisplay(cols[j][i])
+                    switch (cols[j][i]) { # fix rendered text width
+                    case "he":
+                        t1 = sprintf(" %-18s", t1)
+                        break
+                    case "ur":
+                        t1 = sprintf(" %-17s", t1)
+                        break
+                    case "hi": case "gu": case "km": case "kn":
+                    case "my": case "ne": case "pa": case "si":
+                    case "ta": case "te": case "yi":
+                        t1 = sprintf(" %-16s", t1)
+                        break
+                    case "ja": case "ko":
+                        t1 = sprintf(" %-12s", t1)
+                        break
+                    case "zh-CN": case "zh-TW":
+                        t1 = sprintf(" %-11s", t1)
+                        break
+                    default:
+                        if (length(t1) <= 15)
+                            t1 = sprintf(" %-15s", t1)
+                    }
+                    switch (length(cols[j][i])) {
+                    case 1: case 2: case 3: case 4:
+                        t2 = sprintf("- %s |", ansi("bold", sprintf("%4s", cols[j][i])))
+                        break
+                    case 5:
+                        t2 = sprintf("- %s|", ansi("bold", cols[j][i]))
+                        break
+                    case 6:
+                        t2 = sprintf("-%s|", ansi("bold", cols[j][i]))
+                        break
+                    case 7:
+                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
+                        break
+                    default:
+                        t2 = ansi("bold", cols[j][i])
+                    }
+                    r = r t1 t2
                 } else
-                    r = r "\t\t       │"
+                    r = r sprintf("%23s│", NULLSTR)
             }
             r = r RS
         }
-        r = r "└" replicate("─", 22) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
+        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
     } else {
-        r = "┌" replicate("─", 22) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
+        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
         for (i = 0; i < rows; i++) {
-            r = r "│ "
+            r = r "│"
             for (j = 0; j < 3; j++) {
                 if (cols[j][i]) {
-                    t = cols[j][i] == "he" ||
-                        cols[j][i] == "kk" ||
-                        cols[j][i] == "ko" ||
-                        cols[j][i] == "ky" ||
-                        cols[j][i] == "ne" ||
-                        cols[j][i] == "pl" ||
-                        cols[j][i] == "ps" ||
-                        cols[j][i] == "sd" ||
-                        cols[j][i] == "sk" ||
-                        cols[j][i] == "sm" ||
-                        cols[j][i] == "so" ||
-                        cols[j][i] == "te" ||
-                        cols[j][i] == "ug" ||
-                        cols[j][i] == "yo" ||
-                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW" &&
-                         length(getName(cols[j][i])) < 6) ? "\t\t " :
-                        cols[j][i] == "fy" ||
-                        cols[j][i] == "gd" ||
-                        cols[j][i] == "ht" ? " " :
-                        "\t "
-                    tt = length(cols[j][i]) == 3 ? " │" :
-                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW") ? "  │" : ""
-                    name = getName(cols[j][i])
-                    if (cols[j][i] == "zh-CN" ||
-                        cols[j][i] == "zh-TW")
-                        name = substr(name, 1, 12) "."
-                    r = r name t "- " ansi("bold", cols[j][i]) tt " "
+                    t1 = getName(cols[j][i])
+                    if (length(t1) > 15)
+                        t1 = substr(t1, 1, 12) "..."
+                    t1 = sprintf(" %-15s", t1)
+                    switch (length(cols[j][i])) {
+                    case 1: case 2: case 3: case 4:
+                        t2 = sprintf("- %s |", ansi("bold", sprintf("%4s", cols[j][i])))
+                        break
+                    case 5:
+                        t2 = sprintf("- %s|", ansi("bold", cols[j][i]))
+                        break
+                    case 6:
+                        t2 = sprintf("-%s|", ansi("bold", cols[j][i]))
+                        break
+                    case 7:
+                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
+                        break
+                    default:
+                        t2 = ansi("bold", cols[j][i])
+                    }
+                    r = r t1 t2
                 } else
-                    r = r "\t\t       │"
+                    r = r sprintf("%23s│", NULLSTR)
             }
             r = r RS
         }
-        r = r "└" replicate("─", 22) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
+        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
     }
     return r
 }


Fix for shows listed but with missing target

--- getAcornFrom-seriesPages.awk
+++ getAcornFrom-seriesPages.awk
@@ -82,6 +82,9 @@
         $0 = substr($0, 5) ", The"
     }
     episodeLinesFound = 0
+    seasonLinesFound = 0
+    descriptionLinesFound = 0
+    durationLinesFound = 0
     numEpisodesStr = ""
     seriesTitle = $0
     print seriesTitle >> TITLE_FILE
@@ -90,8 +93,8 @@

 # Extract the number of episodes in the series
 /itemprop="numberOfEpisodes"/ {
-    split ($0,fld,"\"")
     episodeLinesFound += 1
+    split ($0,fld,"\"")
     if (episodeLinesFound == 1)
         totalEpisodes = fld[4]
     if (episodeLinesFound != 1)
@@ -101,6 +104,7 @@

 # Extract the number of seasons in the series
 /itemprop="numberOfSeasons"/ {
+    seasonLinesFound += 1
     split ($0,fld,"\"")
     numSeasons = fld[4]
     if ((numSeasons + 0) == 0)
@@ -115,6 +119,7 @@

 # Extract the series description
 /id="franchise-description"/ {
+    descriptionLinesFound += 1
     # get rid of boilerplate
     split ($0,fld,"[<>]")
     description = fld[3]
@@ -177,6 +182,7 @@

 # Extract the episode duration
 /<meta itemprop="timeRequired"/ {
+    durationLinesFound += 1
     split ($0,fld,"\"")
     split (fld[4],tm,/[TMS]/)
     secs = tm[3]
@@ -304,15 +310,38 @@
 }

 /<footer>/ {
-    print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
-    if (description == "")
-        printf ("==> No description: %d\t%s  %s\n", SERIES_NUMBER, shortURL, seriesTitle) >> ERROR_FILE
+    if (episodeLinesFound == 0) {
+        print 0  >> NUM_EPISODES_FILE
+        printf ("==> No numberOfEpisodes: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
+    } else {
+        print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
+    }
+    #
+    if (seasonLinesFound == 0) {
+        print seasonLinesFound >> NUM_SEASONS_FILE
+        printf ("==> No numberOfSeasons: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
+    }
+    #
+    if (descriptionLinesFound == 0) {
+        print "-- No Description --" >> DESCRIPTION_FILE
+        printf ("==> No franchise-description: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
+    } else {
+        if (description == "")
+            printf ("==> No description: %d\t%s  %s\n", SERIES_NUMBER, shortURL, \
+                    seriesTitle) >> ERROR_FILE
+    }
     description = ""
     #
-    seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
-    print seriesDuration >> DURATION_FILE
-    if (seriesDuration == "00:00:00")
-        printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, shortURL, seriesTitle) >> ERROR_FILE
+    if (durationLinesFound == 0) {
+        print "00:00:00"  >> DURATION_FILE
+        printf ("==> No timeRequired: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
+    } else {
+        seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
+        print seriesDuration >> DURATION_FILE
+        if (seriesDuration == "00:00:00")
+            printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, shortURL, i\
+                    seriesTitle) >> ERROR_FILE
+   }
     seriesSecs = 0
     seriesMins = 0
     seriesHrs = 0


Help: header comment

Help: header comment


Main: support option aliases -hl, -sl and -tl

--- include/Main.awk
+++ include/Main.awk
@@ -488,17 +488,17 @@

         ## Language preference options

-        # -l CODE, -lang CODE
-        match(ARGV[pos], /^--?l(a(ng?)?)?(=(.*)?)?$/, group)
-        if (RSTART) {
-            Option["hl"] = group[3] ?
-                (group[4] ? group[4] : Option["hl"]) :
-                ARGV[++pos]
-            continue
-        }
-
-        # -s CODE, -source CODE
-        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?)?(=(.*)?)?$/, group)
+        # -l CODE, -hl CODE, -lang CODE
+        match(ARGV[pos], /^--?(l(a(ng?)?)?|hl)(=(.*)?)?$/, group)
+        if (RSTART) {
+            Option["hl"] = group[4] ?
+                (group[5] ? group[5] : Option["hl"]) :
+                ARGV[++pos]
+            continue
+        }
+
+        # -s CODE, -sl CODE, -source CODE
+        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?|l)?(=(.*)?)?$/, group)
         if (RSTART) {
             Option["sl"] = group[5] ?
                 (group[6] ? group[6] : Option["sl"]) :
@@ -506,8 +506,8 @@
             continue
         }

-        # -t CODES, -target CODES
-        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?)?(=(.*)?)?$/, group)
+        # -t CODES, -tl CODE, -target CODES
+        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?|l)?(=(.*)?)?$/, group)
         if (RSTART) {
             if (group[5]) {
                 if (group[6]) split(group[6], Option["tl"], "+")


Handle language "other".

Handle language "other".

--- tbl2mim.awk
+++ tbl2mim.awk
@@ -27,7 +27,11 @@

 /^ICON *=/ { ICON = " \""$3"\""; next; }
 /^NAME *=/ { NAME = tolower($3); next; }
-/^LANGUAGES *=/ { LANG = $3; next; }
+/^LANGUAGES *=/ {
+    LANG = $3;
+    if (LANG == "other")
+	LANG = "t";
+    next; }
 /^STATUS_PROMPT *=/ { TITLE = $3; next; }
 /^BEGIN_TABLE/ {
     printf "(input-method %s %s)\n", LANG, NAME;


fix filter-showIncludes

fix filter-showIncludes

We want what is after the prefix, not the prefix itself.
80cf1c22b629d524de18f20dcfd155d889ee5d2f could not have possibly worked.

Change-Id: Ic2ab84d8c60ac608a773368b873ea5f98be4cd29

--- solenv/gbuild/platform/filter-showIncludes.awk
+++ solenv/gbuild/platform/filter-showIncludes.awk
@@ -42,7 +42,7 @@
 {
     sub(/^ */, "")
     if (index($0, showincludes_prefix) == 1) {
-        $0 = substr($0, 1, length(showincludes_prefix))
+        $0 = substr($0, length(showincludes_prefix) + 1)
         sub(/^ */, "")
         gsub(/\\/, "/")
         gsub(/ /, "\\ ")


corrected some whitespace.

corrected some whitespace.

--- bin/util/header2params2.awk
+++ bin/util/header2params2.awk
@@ -45,7 +45,7 @@
       print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
       RDFS="todo"
    }
-   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
+   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
    if(length(conversionID)) {
       print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
       print "@prefix vann:          <http://purl.org/vocab/vann/> ."


only concatenate when replacing a character

--- csvquote.awk
+++ csvquote.awk
@@ -18,8 +18,8 @@
     maybe_escaped_quote_char = 0; # false
 }

-function replace_special_chars(c) {
-    answer = c; # default to pass-through
+function suggest_replacementchar(c) {
+    answer = ""; # default to pass-through, that's what empty string means
     if (maybe_escaped_quote_char) {
         if (c != quotechar) {
             # this is the end of a quoted field
@@ -59,12 +59,21 @@
         print $0;
     } else {
         # otherwise we scan the line for quote characters
-        writebuf = "";
-        numchars = length($0);
+        writebuf = $0 RS; # concat. the record separator needs to be included
+        numchars = length(writebuf);
         for (i=1;i<=numchars;i++) {
-            writebuf = writebuf (replace_special_chars(substr($0,i,1)));
+            replacementchar = suggest_replacementchar(substr(writebuf,i,1));
+            if (replacementchar != "") {
+                # we need to modify the current character
+                if (i == 1) {
+                    writebuf = replacementchar substr(writebuf, i + 1); # concat
+                } else if (i == numchars) {
+                    writebuf = substr(writebuf, 1, i - 1) replacementchar; # concat
+                } else {
+                    writebuf = substr(writebuf, 1, i - 1) replacementchar substr(writebuf, i + 1); # concat
+                }
+            }
         }
-        writebuf = writebuf (replace_special_chars(RS));
         printf("%s", writebuf);
     }
 }


fallback to basic interactive mode (no wrapper) when $TRANS_PROGRAM and EntryPoint do not exist

--- include/Main.awk
+++ include/Main.awk
@@ -255,7 +255,7 @@
         # Interactive mode
         initRlwrap() # initialize Rlwrap

-        if (Rlwrap) {
+        if (Rlwrap && (ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
             command = Rlwrap " " Gawk " " (ENVIRON["TRANS_PROGRAM"] ?
                                            "\"${TRANS_PROGRAM}\"" :
                                            "-f " EntryPoint) " -" \
@@ -275,26 +275,29 @@
         # Emacs interface
         Emacs = "emacs"

-        params = ""
-        for (i = 1; i < length(ARGV); i++)
-            if (ARGV[i])
-                params = params " " (parameterize(ARGV[i], "\""))
-        if (ENVIRON["TRANS_PROGRAM"]) {
-            el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
-                "(setq explicit-shell-file-name \"" Gawk "\") " \
-                "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
-                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
-        } else {
-            el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
-                "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
-                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
-        }
-        command = Emacs " --eval " parameterize(el)
-
-        if (!system(command))
-            exit
-        else
-            exit 1
+        if (ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint)) {
+            params = ""
+            for (i = 1; i < length(ARGV); i++)
+                if (ARGV[i])
+                    params = params " " (parameterize(ARGV[i], "\""))
+            if (ENVIRON["TRANS_PROGRAM"]) {
+                el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
+                    "(setq explicit-shell-file-name \"" Gawk "\") " \
+                    "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
+                    "(command-execute 'shell) (rename-buffer \"" Name "\"))"
+            } else {
+                el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
+                    "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
+                    "(command-execute 'shell) (rename-buffer \"" Name "\"))"
+            }
+            command = Emacs " --eval " parameterize(el)
+
+            if (!system(command))
+                exit # child process finished, exit
+            else
+                Option["interactive"] = 1 # skip
+        } else
+            Option["interactive"] = 1 # skip
     }

     if (Option["play"]) {


Add missing FPAT from last commit

Add missing FPAT from last commit

Forgot to add the FPAT print out in the last commit.

--- bin/bawlk.awk
+++ bin/bawlk.awk
@@ -23,6 +23,7 @@
     print "    if(!action) action = \"validate\""
     print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
     print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
+    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
     print "}" RS

     print "# builtin helper functions"


add title, predicate prefix, exceptions, notes

--- scripts/ntriple2dot.awk
+++ scripts/ntriple2dot.awk
@@ -1,17 +1,36 @@
 #! /usr/bin/awk -f

-# remove first and last chars
+#  Reduce the subject and object of RDF triples (ntriples format)
+# down to their @prefix (or literal object class)
+#  Reduce predicates to the specific identifier and
+# the ontology they are from.(curi form)
+#  Express the subject and object as nodes with a directed edge
+# labeled with the predicate in the graphviz dot format
+#  Include a tally of each combination of nodes and edge
+
+#  This may over generalize in some cases because I do not have
+# a handy way to differentiate uri for subjects and objects
+# which may belong to a "structural" ontology as opposed to
+# a data uri.
+
+#  Perhaps an improvement would be to to also
+# express subject and objects as individual curies (ala predicates)
+# iff their namespace is also used by a predicate.
+
+##########################################################
+# remove first and last chars from input
 function trim(str){
 	return substr(str,2,length(str)-2)
 }

 # remove final element in slashed path
+# this leaves the namespace of the removed identifier
 function stripid(uri){
 	l = match(uri,/.*\//)  # side effect is RLENGTH
 	if(l<=0)
 		return uri
 	else
-		return substr(uri,1,RLENGTH -1)
+		return substr(uri,1,RLENGTH-1)
 }

 # keep underscore, letters & numbers
@@ -36,9 +55,9 @@
 		return simplify(uri)
 }

-# get the final (fragment identifier) portion of a slashed path
+# get the final (incl fragment identifier) portion of a slashed path
 function final(uri){
-	split(trim(uri),b,"/")
+	split(uri, b, "/")
 	p = b[length(b)]
 	anchor = match(p, "#")
 	if(anchor > 0)
@@ -51,6 +70,8 @@
 	# which escape the general rules
 	# in everything
 	exception["http://www.w3.org/2002/07"]="OWL_"
+	exception["http://www.w3.org/1999/02"]="RDF"
+	exception["http://www.w3.org/2000/01"]="RDFS"
 	# in go,mgi,wormbase
 	exception["http://dx.doi.org"]="DOI"
 	# in mgi
@@ -59,6 +80,8 @@
 	exception["http://identifiers.org/wormbase"]="WormBase"
 	# just until skolemized bnodes get in the curie map?
 	exception["https://monarchinitiave.org/.well-known/genid"]="BNODE"
+	# till all non httpS: are purged
+	exception["http://monarchinitiave.org"]="MONARCH"
 }

 # main loop
@@ -73,20 +96,22 @@
 	### Subject (uri)
 	s = contract(stripid(trim($1)))
 	### Predicate (uri)
-	p = final($2)
+	q = final(trim($2))
+	p = contract(stripid(trim($2)))
 	### Object (like subject)
 	o = contract(stripid(trim($3)))
-	edgelist[s " -> " o " [label=\"" p ]++
+	edgelist[s " -> " o " [label=\"" p ":" q]++
 }
 ### case when the object is a literal
 (FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*" \.$/ {
 	### Subject (uri)
 	s = contract(stripid(trim($1)))
 	### Predicate (uri)
-	p = final($2)
+	q = final(trim($2))
+	p = contract(stripid(trim($2)))
 	### not a uri
 	o = "LITERAL"
-	edgelist[s " -> " o " [label=\"" p ]++
+	edgelist[s " -> " o " [label=\"" p":"q ]++
 	nodelist[o " [shape=record];"]++
 }

@@ -97,6 +122,12 @@
 	print "charset=\"utf-8\";"
 	for(edge in edgelist) print edge " (" edgelist[edge] ")\"];"
 	for(node in nodelist) print node
+	print "labelloc=\"t\";"
+	title = final(FILENAME)
+	now = systime()
+	# return date(1)-style output
+	datestamp = strftime("%Y%m%d", now)
+	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
 	print "}"
 }


updates table related meta to be file based rules

--- bin/bawlk.awk
+++ bin/bawlk.awk
@@ -50,6 +50,10 @@
     print "}" RS
     print "# awk rules based on user csv ruleset"
     print "NR == 1 { headers=\"" $3 "\"; if (!are_headers_valid(headers)) print \"invalid headers\" }"
+}
+
+$1 == "file" {
+    file_options[$2]=$3;
 }

 $1 == "field" {
@@ -140,7 +144,7 @@
     print "action == \"insert\" && NR == 1 {"
     print "    print \"SET client_encoding = 'UTF8';\""
     print "    gsub(\"Range\", \"rangeno\");"
-    print "    print \"COPY " options["table"] " (\" $0 \") FROM stdin;\""
+    print "    print \"COPY " file_options["table"] " (\" $0 \") FROM stdin;\""
     print "}"

     print "action == \"insert\" && NR > 1 {"
@@ -161,20 +165,20 @@
         if (c < l) fields=fields ","
     }

-    if(options["pkey"]) {
-        cols=options["pkey"]
+    if(file_options["pkey"]) {
+        cols=file_options["pkey"]
         gsub(":", ",", cols)
         fields=fields ", PRIMARY KEY (" cols ") "
     }
-    if(options["fkey"]) {
-        split(options["fkey"], ref, " ")
+    if(file_options["fkey"]) {
+        split(file_options["fkey"], ref, " ")
         ftable=ref[1]
         fcols=ref[2]
         gsub(":", ",", fcols)
         fields=fields ", FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
     }

-    print "     print \"CREATE TABLE IF NOT EXISTS " options["table"] " (" fields ");\""
+    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
     print "}"

     print "action == \"sanitize\" { print }" RS


dtx_check.gawk: Fix false-positives in DTX check

Previously, if ONSET happened exactly at the place where next SID FIRST
or UPDATE should be it was incorrectly detected as error. Similarly,
continuos FACCH was misinterpreted as error.

Change-Id: I43fdbceea0dbdb0833c3b1cf0fc3b825803ed30e

--- contrib/dtx_check.gawk
+++ contrib/dtx_check.gawk
@@ -40,7 +40,7 @@
 			CHK = "FAIL: " TYPE " followed by " $2 " instead of P2."
 			ERR++
 		}
-		if ("OK" == CHK) { # check inter-SID distances:
+		if ("OK" == CHK && "ONSET" != $2) { # check inter-SID distances:
 			if ("UPDATE" == TYPE) {
 				if (DELTA > U_MAX) {
 					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID UPDATE (@" FN ") too big " DELTA "ms > " U_MAX "ms."
@@ -62,8 +62,8 @@
 				}
 			}
 		}
-		if ("FACCH" == TYPE && "FIRST" != $2 && 1 == SILENCE) { # check FACCH handling
-			CHK = "FAIL: incorrect silence resume after FACCH."
+		if ("FACCH" == TYPE && "FIRST" != $2 && "FACCH" != $2 && 1 == SILENCE) { # check FACCH handling
+			CHK = "FAIL: incorrect silence resume with " $2 " after FACCH."
 			ERR++
 		}
 	}


Add support for commenting the Python constants. (dm)

git-svn-id: 30a5f035a20f1bc647618dbad7eea2a951b61b7c@2829 91a5dbb7-01b9-0310-9b5f-b28072856b6e

--- Bindings/Python/constants.awk
+++ Bindings/Python/constants.awk
@@ -16,7 +16,7 @@
 ###############################################################################

 function brlCommand(name, symbol, value, help) {
-  writeCommandDefinition(name, value)
+  writeCommandDefinition(name, value, help)
 }

 function brlBlock(name, symbol, value, help) {
@@ -24,7 +24,7 @@
   if (name == "PASSKEY") return

   if (value ~ /^0[xX][0-9a-fA-F]+00$/) {
-    writeCommandDefinition(name, tolower(value) "00")
+    writeCommandDefinition(name, tolower(value) "00", help)
   }
 }

@@ -44,29 +44,29 @@
   } else {
     return
   }
-  writePythonAssignment("KEY_FLG_" name, value)
+  writePythonAssignment("KEY_FLG_" name, value, help)
 }

 function brlDot(number, symbol, value, help) {
-  writePythonAssignment("DOT" number, value)
+  writePythonAssignment("DOT" number, value, help)
 }

 function apiConstant(name, symbol, value, help) {
-  writePythonAssignment(name, value)
+  writePythonAssignment(name, value, help)
 }

 function apiMask(name, symbol, value, help) {
-  writePythonAssignment("KEY_" name, "c_brlapi." symbol)
+  writePythonAssignment("KEY_" name, "c_brlapi." symbol, help)
 }

 function apiShift(name, symbol, value, help) {
   value = hexadecimalValue(value)
-  writePythonAssignment("KEY_" name, value)
+  writePythonAssignment("KEY_" name, value, help)
 }

 function apiType(name, symbol, value, help) {
   value = hexadecimalValue(value)
-  writePythonAssignment("KEY_TYPE_" name, value)
+  writePythonAssignment("KEY_TYPE_" name, value, help)
 }

 function apiKey(name, symbol, value, help) {
@@ -74,22 +74,24 @@

   if (name == "FUNCTION") {
     for (i=0; i<35; ++i) {
-      writeKeyDefinition("F" (i+1), "(" value " + " i ")")
+      key = "F" (i+1)
+      writeKeyDefinition(key, "(" value " + " i ")", "the " key " key")
     }
   } else {
-    writeKeyDefinition(name, value)
+    writeKeyDefinition(name, value, help)
   }
 }

-function writeCommandDefinition(name, value) {
-  writePythonAssignment("KEY_CMD_" name, value)
+function writeCommandDefinition(name, value, help) {
+  writePythonAssignment("KEY_CMD_" name, value, help)
 }

-function writeKeyDefinition(name, value) {
-  writePythonAssignment("KEY_SYM_" name, value)
+function writeKeyDefinition(name, value, help) {
+  writePythonAssignment("KEY_SYM_" name, value, help)
 }

-function writePythonAssignment(name, value) {
+function writePythonAssignment(name, value, help) {
+  if (length(help) > 0) print "## " help
   print name " = " value
 }


build.awk: update readme generator

build.awk: update readme generator

--- build.awk
+++ build.awk
@@ -50,7 +50,7 @@
     return system("pandoc -s -f markdown-smart -t man " ManMarkdown " -o " Man)
 }

-function readme(    code, col, cols, content, group, i, j, num, language, r, rows, text) {
+function readme(    code, col, cols, content, group, i, iso, j, num, r, rows, text) {
     text = readFrom(ReadmeTemplate)

     content = getOutput("gawk -f translate.awk -- -no-ansi -h")
@@ -82,9 +82,11 @@
         r = r "| "
         for (j = 0; j < 3; j++)
             if (cols[j][i]) {
-                split(getName(cols[j][i]), group, " ")
-                language = length(group) == 1 ? group[1] "_language" : join(group, "_")
-                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
+                split(getISO(cols[j][i]), group, "-")
+                iso = group[1]
+
+                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/ISO_639:" iso \
+                    ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
             }
         r = r RS
     }


Fix bug. Strings are 1 based

Fix bug. Strings are 1 based

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@17659 4f837ed2-42f5-46e7-a7a5-fa17313484d4

--- scripts/tdb/statuses_clockss.awk
+++ scripts/tdb/statuses_clockss.awk
@@ -8,7 +8,7 @@

 {
 # add a loop to add line only if first 4 char or last 4 char of year are gt or eq to contract
-if ((substr($4,0,4) >= $3) || ((length($4) == 9) && (substr($4,6,4) >= $3))) {
+if ((substr($4,1,4) >= $3) || ((length($4) == 9) && (substr($4,6,4) >= $3))) {
     nn = split($2,na,/\./)
     lp2 = na[nn]
   if (!(($1,lp2,$3,$4) in b)) {


Added commandline args options -n and -c to oppress output of barcode in the readname and comment, respectively

--- extract_barcodes/extract_barcodes.awk
+++ extract_barcodes/extract_barcodes.awk
@@ -1,11 +1,39 @@
+BEGIN{
+    attach_in_readname = 1;
+    attach_in_comment = 1;
+
+    for (i = 1; i < ARGC; i++) {
+        if (ARGV[i] == "-n")
+            attach_in_readname = 0
+        else if (ARGV[i] == "-c")
+            attach_in_comment = 0
+        else if (ARGV[i] ~ /^-./) {
+            e = sprintf("%s: unrecognized option -- %c",
+                    ARGV[0], substr(ARGV[i], 2, 1))
+            print e > "/dev/stderr"
+        } else
+            break
+        delete ARGV[i]
+    }
+
+}
+
 {
     if (NR % 8 == 1 && NR > 1) {
-        barcode = substr(read_pair[2], 1, 16)
-        printf read_pair[1]" BX:Z:"barcode"\n"
+        barcode = substr(read_pair[2], 1, 16);
+        if (attach_in_readname) {
+            initial = barcode"-1_";
+        }
+
+        if (attach_in_comment) {
+            final = " BX:Z:"barcode"-1";
+        }
+
+        printf initial read_pair[1] final"\n"
         printf substr(read_pair[2], 17)"\n"
         printf read_pair[3]"\n"
         printf substr(read_pair[4], 17)"\n"
-        printf read_pair[5]" BX:Z:"barcode"\n"
+        printf initial read_pair[5] final"\n"
         printf read_pair[6]"\n"
         printf read_pair[7]"\n"
         printf read_pair[0]"\n"


LanguageData: update section T

LanguageData: update section T


Put local symbol suppression rule into most recent (e.g. last) version
block.

Approved by:	re (kensmith)


Utils: fix base64() on systems where echo does not accept option -n

--- include/Utils.awk
+++ include/Utils.awk
@@ -210,8 +210,8 @@
 # Base64 encode a string.
 # NOTE: can only be ran once for each text! Build a cache.
 function base64(text,    command, temp) {
-    command = "base64"
-    ("echo -n " parameterize(text) PIPE command) | getline temp
+    command = "echo -n " parameterize(text) PIPE "base64"
+    ("bash -c " parameterize(command, "\"")) | getline temp
     return temp
 }


Add a status for source AUs.

Add a status for source AUs.

--- scripts/tdb/statuses_clockss.awk
+++ scripts/tdb/statuses_clockss.awk
@@ -94,7 +94,8 @@
   s[13] = "superseded"
   s[14] = "zapped"
   s[15] = "doNotProcess"
-  sn = 16
+  s[16] = "readySource"
+  sn = 17

   sc[0] = "expe"
   sc[1] = "exis"
@@ -112,7 +113,8 @@
   sc[13] = "supe"
   sc[14] = "zapp"
   sc[15] = "doNP"
-  scn = 16
+  sc[16] = "rdSc"
+  scn = 17

   #print out header
   printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tP\tTotal"


dheader and hheader were the same function, really, so rename one of them to header, kill the other and use both with a different arg...

--- sys/tools/usbdevs2h.awk
+++ sys/tools/usbdevs2h.awk
@@ -37,44 +37,24 @@
 	exit 1;
 }

-function dheader(dfile)
+function header(file)
 {
 	if (os == "NetBSD")
-		printf("/*\t\$NetBSD\$\t*/\n\n") > dfile
+		printf("/*\t\$NetBSD\$\t*/\n\n") > file
 	else if (os == "FreeBSD")
-		printf("/* \$FreeBSD\$ */\n\n") > dfile
+		printf("/* \$FreeBSD\$ */\n\n") > file
 	else if (os == "OpenBSD")
-		printf("/*\t\$OpenBSD\$\t*/\n\n") > dfile
+		printf("/*\t\$OpenBSD\$\t*/\n\n") > file
 	else
-		printf("/* ??? */\n\n") > dfile
-	printf("/*\n") > dfile
+		printf("/* ??? */\n\n") > file
+	printf("/*\n") > file
 	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
-	    > dfile
-	printf(" *\n") > dfile
-	printf(" * generated from:\n") > dfile
-	printf(" *\t%s\n", VERSION) > dfile
-	printf(" */\n") > dfile
-}
-
-function hheader(hfile)
-{
-	if (os == "NetBSD")
-		printf("/*\t\$NetBSD\$\t*/\n\n") > hfile
-	else if (os == "FreeBSD")
-		printf("/* \$FreeBSD\$ */\n\n") > hfile
-	else if (os == "OpenBSD")
-		printf("/*\t\$OpenBSD\$\t*/\n\n") > hfile
-	else
-		printf("/* ??? */\n\n") > hfile
-	printf("/*\n") > hfile
-	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
-	    > hfile
-	printf(" *\n") > hfile
-	printf(" * generated from:\n") > hfile
-	printf(" *\t%s\n", VERSION) > hfile
-	printf(" */\n") > hfile
-}
-
+	    > file
+	printf(" *\n") > file
+	printf(" * generated from:\n") > file
+	printf(" *\t%s\n", VERSION) > file
+	printf(" */\n") > file
+}

 function vendor(hfile)
 {
@@ -261,12 +241,10 @@
 	if (line == 1) {
 		VERSION = $0
 		gsub("\\$", "", VERSION)
-
 		if (dfile)
-			dheader(dfile)
-
-		if (hfile)
-			hheader(hfile)
+			header(dfile)
+		if (hfile)
+			header(hfile)
 		continue;
 	}
 	if ($1 == "vendor") {


Update usage comment to match reality.

Update usage comment to match reality.

--- sys/tools/vnode_if.awk
+++ sys/tools/vnode_if.awk
@@ -34,8 +34,9 @@
 #
 # Script to produce VFS front-end sugar.
 #
-# usage: vnode_if.awk <srcfile> [-c | -h]
+# usage: vnode_if.awk <srcfile> [-c | -h | -p | -q]
 #	(where <srcfile> is currently /sys/kern/vnode_if.src)
+#	The source file must have a .src extension
 #

 function usage()


Vastly simplify smt output munger.

Vastly simplify smt output munger.

This is going to be stripping out CVC3's annoying "success" lines too.

--- smt_parse.awk
+++ smt_parse.awk
@@ -1,13 +1,5 @@
 #!/usr/bin/gawk -f
-BEGIN{
-	bees="";
+# Simplified approach: only pick out lines with round definitions
+/^\(\(round.*\)\)$/ {
+	print $0;
 }
-
-/^[ ]+\(define-fun.*round.*Int$/{
-	bees = $2;
-}
-
-/^[ ]+[0-9]+\)$/{
-	match ($0, /^[ ]+([0-9]+)\)/, arr)
-	print bees " " arr[1];
-}


Languages: add Konkani

Languages: add Konkani

--- include/Languages.awk
+++ include/Languages.awk
@@ -794,6 +794,19 @@
     Locale["rw"]["iso"]                = "kin"
     Locale["rw"]["glotto"]             = "kiny1244"
     Locale["rw"]["script"]             = "Latn"
+
+    # Konkani (Goan Konkani)
+    Locale["gom"]["name"]              = "Konkani"
+    Locale["gom"]["endonym"]           = "कोंकणी"
+    #Locale["gom"]["translations-of"]
+    #Locale["gom"]["definitions-of"]
+    #Locale["gom"]["synonyms"]
+    #Locale["gom"]["examples"]
+    #Locale["gom"]["see-also"]
+    Locale["gom"]["family"]            = "Indo-European"
+    Locale["gom"]["iso"]               = "gom"
+    Locale["gom"]["glotto"]            = "goan1235"
+    Locale["gom"]["script"]            = "Deva"

     #50 Korean
     Locale["ko"]["name"]               = "Korean"


Translate: localize variable i in preprocessByDump() (fixed a regression bug that affects Google Translate engine)

--- include/Translate.awk
+++ include/Translate.awk
@@ -87,7 +87,7 @@
 }

 # Pre-process string (using hexdump to URL-encode everything).
-function preprocessByDump(text,    arr, len, temp) {
+function preprocessByDump(text,    arr, i, len, temp) {
     len = dumpX(text, arr)
     temp = ""
     for (i = 1; i <= len; i++)


Replace split with loop for POSIX compatibility

split: null string for third arg is a gawk extension

--- levenshtein.awk
+++ levenshtein.awk
@@ -12,8 +12,12 @@
     if (len1 == 0) return len2
     if (len2 == 0) return len1
     if (str1 == str2) return 0
-    split(str1, chars1, "")
-    split(str2, chars2, "")
+    for(i = 1; i <= len1; i++) {
+        chars1[i] = substr(str1, i, 1)
+    }
+	for(j = 1; j <= len2; j++) {
+        chars2[j] = substr(str2, j, 1)
+    }
     for (i = 0; i <= len1; i++) {
         matrix[i, 0] = i
     }


Better variable names.

Better variable names.

--- gfwlist2dnsmasq.awk
+++ gfwlist2dnsmasq.awk
@@ -27,11 +27,11 @@
 	# Expand the lines like (aa|bb|cc) into multiple records
 	pos = match(line, /\([a-zA-Z0-9\.\|]+\)/)
 	if (pos != 0) {
-		bracket = substr(line, RSTART + 1, RLENGTH - 2)
-		n = split(bracket, a, "|")
+		in_bracket = substr(line, RSTART + 1, RLENGTH - 2)
+		n = split(in_bracket, arr, "|")
 		for (i = 1; i <= n; i++) {
 			expanded_line = line
-			sub(/\([a-zA-Z0-9\.\|]+\)/, a[i], expanded_line)
+			sub(/\([a-zA-Z0-9\.\|]+\)/, arr[i], expanded_line)
 			extract_regex(expanded_line, original_line)
 		}
 	} else {


Regress because of errors.

Regress because of errors.

--- scripts/tdb/statuses_gln.awk
+++ scripts/tdb/statuses_gln.awk
@@ -4,11 +4,11 @@
 BEGIN {
   FS="\t"
   pn = 0
-    str = "date +%m-%d-%Y";
-    str | getline date;
+    #str = "date +%m-%d-%Y";
+    #str | getline date;
     #close str;
 }
-$3>date
+#$3>date

 {
   # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2005 && journal year (or journal end year) is lt or eq to the current year we are testing]
@@ -110,7 +110,7 @@
     }

   printf "\n"
-  printf "%s", date
-  printf "\n"
+  #printf "%s", date
+  #printf "\n"
 }


Translators/BingTranslator: fix dictionary lookup

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -2,7 +2,7 @@
 # BingTranslator.awk                                               #
 ####################################################################
 #
-# Last Updated: 24 Jul 2019
+# Last Updated: 25 Jul 2019
 BEGIN { provides("bing") }

 function bingInit() {
@@ -11,7 +11,7 @@
     HttpPort = 80
 }

-# [OBSOLETE] Retrieve the Cookie and set IG.
+# [OBSOLETE?] Retrieve the Cookie and set IG.
 function bingSetIG(    content, cookie, group, header, isBody,
                        url, status, location) {
     url = HttpPathPrefix "/translator"
@@ -138,7 +138,7 @@
     return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
 }

-# Dictionary API (via HTTP GET).
+# [OBSOLETE?] Old dictionary API (via HTTP GET).
 function bingRequestUrl(text, sl, tl, hl) {
     return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
         "from=" sl "&to=" tl "&text=" preprocess(text)
@@ -148,8 +148,8 @@
 function bingPostRequestUrl(text, sl, tl, hl, type) {
     if (type == "lookup")
         return HttpPathPrefix "/tlookupv3"
-    else if (type == "transliterate")
-        return HttpPathPrefix "/ttransliteratev3"
+    #else if (type == "transliterate")
+    #    return HttpPathPrefix "/ttransliteratev3"
     else # type == "translate"
         return HttpPathPrefix "/ttranslatev3"
 }
@@ -160,9 +160,9 @@

 function bingPostRequestBody(text, sl, tl, hl, type) {
     if (type == "lookup")
-        return "&text=" quote(text) "&fromLang=" sl "&to=" tl  # FIXME!
-    else if (type == "transliterate")
-        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"  # FIXME!
+        return "&text=" quote(text) "&from=" sl "&to=" tl
+    #else if (type == "transliterate")
+    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
     else # type == "translate"
         return "&text=" quote(text) "&fromLang=" sl "&to=" tl
 }
@@ -172,11 +172,14 @@
                        isVerbose, toSpeech, returnPlaylist, returnIl,
                        ####
                        r,
-                       content, tokens, ast,
+                       content, tokens, ast, dicContent, dicTokens, dicAst,
                        _sl, _tl, _hl, il, isPhonetic,
-                       translation,
-                       wShowOriginal, wShowTranslation, wShowLanguages,
-                       group, temp) {
+                       translation, phonetics, oPhonetics,
+                       wordClasses, words, wordBackTranslations,
+                       wShowOriginal, wShowOriginalPhonetics,
+                       wShowTranslation, wShowTranslationPhonetics,
+                       wShowLanguages, wShowDictionary,
+                       i, j, k, group, temp, saveSortedIn) {
     isPhonetic = match(tl, /^@/)
     tl = substr(tl, 1 + isPhonetic)

@@ -275,7 +278,7 @@
             split(_sl, group, "-")
             #content = postResponse(text, group[1], group[1], _hl, "transliterate")
             #oPhonetics = unparameterize(content)
-            # TODO
+            # FIXME!
             if (oPhonetics == text) oPhonetics = ""
         }

@@ -320,19 +323,60 @@
             r = r prettify("languages", group[3])
         }

-        if (wShowDictionary && false) { # FIXME!
+        if (wShowDictionary) {
             # Dictionary API
-            # Note: source language must be identified
-            dicContent = getResponse(text, _sl, _tl, _hl)
-            tokenize(dicTokens, dicContent)
-            parseJson(dicAst, dicTokens) # FIXME: inefficient parser
-
-            if (anything(dicAst)) {
+            dicContent = postResponse(text, il, _tl, _hl, "lookup")
+            if (dicContent != "") {
+                tokenize(dicTokens, dicContent)
+                parseJson(dicAst, dicTokens)
+
+                l(dicContent, "content", 1, 1)
+                l(dicTokens, "tokens", 1, 0, 1)
+                l(dicAst, "ast")
+
+                saveSortedIn = PROCINFO["sorted_in"]
+                PROCINFO["sorted_in"] = "compareByIndexFields"
+                for (i in dicAst) {
+                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
+                              "posTag$", group))
+                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
+                }
+                for (i in dicAst) {
+                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
+                              "displayTarget$", group))
+                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
+                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
+                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
+                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
+                }
+                PROCINFO["sorted_in"] = saveSortedIn
+
                 # Display: dictionary entries
                 if (r) r = r RS
                 r = r m("-- display dictionary entries")
-
-                # TODO
+                for (i = 0; i < length(words); i++) {
+                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))
+
+                    for (j in words[wordClasses[i]]) {
+                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))
+
+                        if (isRTL(il))
+                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
+                        else {
+                            explanation = prettify("dictionary-explanation-item",
+                                                   wordBackTranslations[wordClasses[i]][j][0])
+                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
+                                explanation = explanation prettify("dictionary-explanation", ", ") \
+                                    prettify("dictionary-explanation-item",
+                                             wordBackTranslations[wordClasses[i]][j][k])
+                        }
+
+                        if (isRTL(il))
+                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
+                        else
+                            r = r RS ins(2, explanation)
+                    }
+                }
             }
         }
     }


Translators/BingTranslator: support original phonetics

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -237,6 +237,7 @@
         split(_tl, group, "-")
         content = postResponse(translation, group[1], group[1], _hl, "transliterate")
         phonetics = unparameterize(content)
+        if (phonetics == translation) phonetics = ""
     }

     # Generate output
@@ -255,13 +256,25 @@
         wShowLanguages = Option["show-languages"]
         wShowDictionary = Option["show-dictionary"]

+        # Transliteration (original)
+        wShowOriginalPhonetics = Option["show-original-phonetics"]
+        if (wShowOriginalPhonetics) {
+            split(il, group, "-")
+            content = postResponse(text, group[1], group[1], _hl, "transliterate")
+            oPhonetics = unparameterize(content)
+            if (oPhonetics == text) oPhonetics = ""
+        }
+
+        if (!oPhonetics) wShowOriginalPhonetics = 0
         if (!phonetics) wShowTranslationPhonetics = 0

         if (wShowOriginal) {
-            # Display: original text
+            # Display: original text & phonetics
             if (r) r = r RS RS
             r = r m("-- display original text")
             r = r prettify("original", s(text, il))
+            if (wShowOriginalPhonetics)
+                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), il))
         }

         if (wShowTranslation) {


port.wm: make it actually read it's input

--- src/port.wm.awk
+++ src/port.wm.awk
@@ -7,26 +7,26 @@
     teslawm_fifo = ENVIRON["TESLAWM_fifo"]
 }

-get("ctx.xprop.wm_class_name") == "Sonata" {
-    set("ctx.xprop.wm_name", HOME "/mnt/ssh/rfigura@wirsing/home/rfigura/pub/audio/" get("ctx.xprop.wm_name"))
+get("xprop.wm_class_name") == "Sonata" {
+    set("xprop.wm_name", HOME "/mnt/ssh/rfigura@wirsing/home/rfigura/pub/audio/" get("xprop.wm_name"))
 }

-get("ctx.xwin") && teslawm_fifo {
+get("xwin") && teslawm_fifo {
     # todo: improve teslawm to accept a window id in commands:
-    label("close"); menu("echo close " get("ctx.xwin") " > " Q(teslawm_fifo))
-    label("movemouse"); menu("echo movemouse " get("ctx.xwin") " > " Q(teslawm_fifo))
-    label("resizemouse"); menu("echo resizemouse " get("ctx.xwin") " > " Q(teslawm_fifo))
+    label("close"); menu("echo close " get("xwin") " > " Q(teslawm_fifo))
+    label("movemouse"); menu("echo movemouse " get("xwin") " > " Q(teslawm_fifo))
+    label("resizemouse"); menu("echo resizemouse " get("xwin") " > " Q(teslawm_fifo))
 }
-get("ctx.xwin") {
-    menu("xkill -id " get("ctx.xwin"))
-    menu(pager("xwininfo -id " get("ctx.xwin")))
-    menu(pager("xprop -id " get("ctx.xwin")))
-    label("wm_name..."); menu(nest(get("ctx.xprop.wm_name"), "open"))
-#    label("teslawm rule (sel)"); menu("@xsel " get("ctx.xprop.wm_class_name") ":" get("ctx.xprop.wm_class") ":" get("ctx.xprop.wm_name"))
+get("xwin") {
+    menu("xkill -id " get("xwin"))
+    menu(pager("xwininfo -id " get("xwin")))
+    menu(pager("xprop -id " get("xwin")))
+    label("wm_name..."); menu(nest(get("xprop.wm_name"), "open"))
+#    label("teslawm rule (sel)"); menu("@xsel " get("xprop.wm_class_name") ":" get("xprop.wm_class") ":" get("xprop.wm_name"))
 }

 {
-    set("pid", get("ctx.xprop.net_wm_pid"))
+    set("pid", get("xprop.net_wm_pid"))
 }
 @include "port.pid.awk"
 @include "port.default.awk"


ahrf.awk: Clean the snippet which was written yesterday


Add processing for movies

Add processing for movies


Add (1,1) seasons, episodes to movies

Add (1,1) seasons, episodes to movies

--- getBritBoxProgramsFrom-webscraper.awk
+++ getBritBoxProgramsFrom-webscraper.awk
@@ -21,6 +21,7 @@
     showTitle = $4
     $7  == "" ? Year = $5 : Year = $7
     NumSeasons = $6
+    NumEpisodes = ""
     Duration = $8
     Rating = $9
     Description = $10
@@ -45,13 +46,20 @@
     # Indicate different types of programs
     URL ~ /^\/us\/movie\// ? showtype = "M" : showtype = "S"

+    # Fill in missing seasons & episodes for movies with 1,1
+    if (showtype == "M") {
+        NumSeasons = 1
+        NumEpisodes = 1
+    }
+
     # Extract sortkey from URL
     nflds = split (URL,fld,"_")
     if (URL ~ /_[[:digit:]]*$/) {
         sortkey = sprintf ("%s%05d", showtype, fld[nflds])
         savedLine = sprintf \
-            ("%s (1) %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t\t%s\t%s\t%s\t%s",\
-             showTitle, sortkey, URL, showTitle, NumSeasons, HMS, Year, Rating, Description)
+            ("%s (1) %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t%s\t%s\t%s\t%s\t%s",\
+             showTitle, sortkey, URL, showTitle, NumSeasons, NumEpisodes, HMS, Year,\
+             Rating, Description)

         # Make sure line doesn't start with a single quote so it sorts correctly in Open Office
         sub (/^'/,"",savedLine)


add|sub rsp alter the indent

add|sub rsp alter the indent

--- tools/push_pop_indent.awk
+++ tools/push_pop_indent.awk
@@ -1,7 +1,7 @@
 #!/bin/sh

 exec awk '
-/pop/ {
+/pop|leave/ {
 	if(--indent < 0)
 		print "AWK: indent less than zero (" indent ")" #| "cat >&2"
 }
@@ -17,4 +17,11 @@
 /push/ {
 	indent++
 }
+
+/(add|sub) rsp/ {
+	n = $3 / 8
+	if($1 == "sub")
+		n = -n
+	indent -= n
+}
 '


Incorporated some changes to the documentation script.

--- comdown.awk
+++ comdown.awk
@@ -62,14 +62,14 @@
 # without any warranty.

 BEGIN {
-
+
 	# Configuration options
 	if(Title=="") Title = "Documentation";
 	if(Theme=="") Theme = 1;
     #TopLinks = 1;
 	#classic_underscore = 1;
     if(MaxWidth=="") MaxWidth="1080px";
-
+
     Mode = "none"; ToC = ""; ToCLevel = 1;
     CSS = init_css(Theme);
     for(i = 0; i < 128; i++)
@@ -178,7 +178,7 @@
         Out = fix_links(Out);
         Out = fix_abbrs(Out);
         Out = make_toc(Out);
-
+
         print trim(Out);
         if(footnotes) {
             footnotes = fix_links(footnotes);
@@ -237,7 +237,7 @@
             if(Buf) res = tag("p", scrub(Buf));
             Buf = scrub(trim(substr(st, RSTART+RLENGTH)));
             push("blockquote");
-        } else if(match(st, /^[[:space:]]*([*+-]|[[:digit:]]+\.)/)) {
+        } else if(match(st, /^[[:space:]]*([*+-]|[[:digit:]]+\.)[[:space:]]/)) {
             if(Buf) res = tag("p", scrub(Buf));
             Buf="";
             match(st, /^[[:space:]]*/);
@@ -311,7 +311,7 @@
     prev = st;
     return res;
 }
-function scrub(st,    mp, ms, me, r, p, tg, a) {
+function scrub(st,    mp, ms, me, r, p, tg, a) {
 	sub(/  $/,"
\n",st);
 	gsub(/(  |[[:space:]]+\\)\n/,"
\n",st);
 	while(match(st, /(__?|\*\*?|~~|`+|[&><\\])/)) {
@@ -320,8 +320,8 @@
 		ms = substr(st, RSTART-1,1);
 		me = substr(st, RSTART+RLENGTH, 1);
 		p = RSTART+RLENGTH;
-
-		if(!classic_underscore && match(mp,/_+/)) {
+
+		if(!classic_underscore && match(mp,/_+/)) {
 			if(match(ms,/[[:alnum:]]/) && match(me,/[[:alnum:]]/)) {
 				tg = substr(st, 1, index(st, mp));
 				r = r tg;
@@ -329,11 +329,11 @@
 				continue;
 			}
 		}
-		st = substr(st, p);
+		st = substr(st, p);
 		r = r a;
 		ms = "";
-
-		if(mp == "\\") {
+
+		if(mp == "\\") {
 			if(match(st, /^!?\[/)) {
 				r = r "\\" substr(st, RSTART, RLENGTH);
 				st = substr(st, 2);
@@ -350,12 +350,12 @@
 				r = r mp;
 				continue;
 			}
-			p = index(st, mp);
+			p = index(st, mp);
 			while(p && match(substr(st, p-1, 1),/[\\[:space:]]/)) {
 				ms = ms substr(st, 1, p-1) mp;
 				st = substr(st, p + length(mp));
 				p = index(st, mp);
-			}
+			}
 			if(!p) {
 				r = r mp ms;
 				continue;
@@ -373,7 +373,7 @@
 				ms = ms substr(st, 1, p-1) mp;
 				st = substr(st, p + length(mp));
 				p = index(st, mp);
-			}
+			}
 			if(!p) {
 				r = r mp ms;
 				continue;
@@ -407,7 +407,7 @@
 		} else if(mp == ">") {
 			r = r ">";
 		} else if(mp == "<") {
-
+
 			p = index(st, ">");
 			if(!p) {
 				r = r "<";
@@ -419,7 +419,7 @@
 				tg = substr(tg,1,RLENGTH-1);
 			} else
 				a = "";
-
+
 			if(match(tolower(tg), "^/?(a|abbr|div|span|blockquote|pre|img|code|p|em|strong|sup|sub|del|ins|s|u|b|i|br|hr|ul|ol|li|table|thead|tfoot|tbody|tr|th|td|caption|column|col|colgroup|figure|figcaption|dl|dd|dt|mark|cite|q|var|samp|small)$")) {
 				r = r "<" tg a ">";
 			} else if(match(tg, "^[[:alpha:]]+://[[:graph:]]+$")) {
@@ -432,8 +432,8 @@
 				r = r "<";
 				continue;
 			}
-
-			st = substr(st, p + 1);
+
+			st = substr(st, p + 1);
 		} else if(mp == "&") {
 			if(match(st, /^[#[:alnum:]]+;/)) {
 				r = r "&" substr(st, 1, RLENGTH);
@@ -442,7 +442,7 @@
 				r = r "&";
 			}
 		}
-	}
+	}
     return r st;
 }

@@ -473,7 +473,7 @@
     p = match(st, /!\[toc[-+]?\]/);
     while(p) {
 		++n;
-		dis = index(substr(st,RSTART,RLENGTH),"+");
+		dis = index(substr(st,RSTART,RLENGTH),"+");
 		t = "
\n" (dis?"[-]":"[+]") " Contents\n" \
 			"
\n
" ToC "
\n
\n
";
         r = r substr(st,1,RSTART-1);
@@ -616,7 +616,7 @@
 }
 function init_css(Theme,             css,ss,hr,c1,c2,c3,c4,c5,bg1,bg2,bg3,bg4,ff,i) {
     if(Theme == "0") return "";
-
+
     css["body"] = "color:%color1%;font-family:%font-family%;line-height:1.5em;" \
                 "padding:1em 2em;width:80%;max-width:%maxwidth%;margin:0 auto;min-height:100%;float:none;";
     css["h1"] = "color:%color1%;border-bottom:1px solid %background1%;padding:0.3em 0.1em;";
@@ -656,7 +656,7 @@
     css["a.footnote-back"] = "text-decoration:initial;font-size:x-small;";
     css[".fade"] = "color:%color5%;";
     css[".highlight"] = "color:%color2%;background-color:%color5%;";
-
+
     if(Theme==2){
         c1="#303F9F";c2="#0449CC";c3="#2162FA";c4="#4B80FB";c5="#EDF2FF";
         ff="\"Trebuchet MS\", Helvetica, sans-serif";
@@ -680,7 +680,7 @@
         c1="#092859";c2="#1351b5";c3="#d47034";c4="#DC7435";c5="#F6F8FF";
     }
 	if(!ff) ff = "sans-serif"
-
+
     for(i = 0; i<=255; i++)_hex[sprintf("%02X",i)]=i;
     bg1 = bright(c1,0.5);
     bg2 = bright(c2,0.5);


FIxup jamaicainn & newworlds bonus seasonNumber

--- getAcornFrom-seriesPages.awk
+++ getAcornFrom-seriesPages.awk
@@ -226,12 +226,18 @@
         showType = "S"
     # Default episodeType to "E"
     episodeType = "E"
-    # Default bonus & parttwo episodeType to "X"
-    if (episodeURL ~ /bonus\/|[0-9]{1,2}parttwo\//)
+    # Default bonus, parttwo, and christmasspecials episodeType to "X"
+    if (episodeURL ~ /bonus\/|[0-9]{1,2}parttwo\/|christmasspecial\//)
         episodeType = "X"
-    # Default christmasspecial & seriesfinale episodeType to "X" & don't increment seasonNumber
-    if (episodeURL ~ /christmasspecial\/|\/seriesfinale\//) {
-        episodeType = "X"
+    # jamaicainn & newworlds bonus seasonNumber should be 1
+    if (episodeURL ~ /\/jamaicainn\/bonus\/|\/newworlds\/bonus\//) {
+        split (episodeURL, part, "/")
+        printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
+               part[4], part[5], seasonNumber) >> ERROR_FILE
+        seasonNumber = 1
+    }
+    # Plain christmasspecial, seriesfinale don't increment seasonNumber
+    if (episodeURL ~ /\/christmasspecial\/|\/seriesfinale\//) {
         split (episodeURL, part, "/")
         printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
                part[4], part[5], seasonNumber) >> ERROR_FILE


Semicolons in xcodebuild.awk

Semicolons in xcodebuild.awk

--- xcodebuild.awk
+++ xcodebuild.awk
@@ -1,19 +1,19 @@
 {
-    print
-    fflush(stdout)
+    print;
+    fflush(stdout);
 }

 /[0-9]+: (error|warning):/ {
-    errors = errors $0 "\n"
+    errors = errors $0 "\n";
 }

 /(TEST|BUILD) FAILED/ {
     if (length(errors) > 0) {
-        print "\n*** All errors:\n" errors
+        print "\n*** All errors:\n" errors;
     }

-    fflush(stdout)
+    fflush(stdout);

     # SYS_SOFTWARE
-    exit 70
+    exit 70;
 }


Docs were failing with this escape.

Docs were failing with this escape.

Not sure why this started failing all of a sudden. Code has been the same for two years. Maybe awk upgrade somewhere?? CTR

--- docs/preprocessor/awk/language-variants.awk
+++ docs/preprocessor/awk/language-variants.awk
@@ -39,7 +39,7 @@
       case "python":
         gsub(/^gremlin>/, ">>>")
         gsub(/^==>/, "")
-        $0 = gensub(/processTraversal\("""(.*)"""\, jython, groovy)/, "\\1", 1)
+        $0 = gensub(/processTraversal\("""(.*)""", jython, groovy)/, "\\1", 1)
         $0 = gensub("g\\.V\\(([^\\)]+)", "g.V(Bindings\\.of('id',\\1)", "g")
         print gensub("g\\.V\\(Bindings.of\\('id',(Bindings.of\\([^\\)]+\\))\\)\\)", "g.V(\\1)", "g")
         break


ahrf.awk: Replace %s by %d for h{1,2,3,4,5,6}


Translators/BingTranslator: new API, fix #123

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -2,81 +2,93 @@
 # BingTranslator.awk                                               #
 ####################################################################
 #
-# Last Updated: 11 Mar 2016
-# http://ssl.microsofttranslator.com/dynamic/226010/js/LandingPage.js
+# Last Updated: 18 May 2016
 BEGIN { provides("bing") }

-function genRTTAppId(    content, group, header, isBody) {
+function bingInit() {
     HttpProtocol = "http://"
-    HttpHost = "ssl.microsofttranslator.com"
+    HttpHost = "www.bing.com"
     HttpPort = 80
-    LandingPage = "/dynamic/226010/js/LandingPage.js"
-
-    if (Option["proxy"]) {
-        match(Option["proxy"], /^(http:\/*)?([^\/]*):([^\/:]*)/, HttpProxySpec)
-        HttpService = "/inet/tcp/0/" HttpProxySpec[2] "/" HttpProxySpec[3]
-        HttpPathPrefix = HttpProtocol HttpHost
-    } else {
-        HttpService = "/inet/tcp/0/" HttpHost "/" HttpPort
-        HttpPathPrefix = ""
-    }
-
-    header = "GET " LandingPage " HTTP/1.1\n"                           \
+}
+
+# Retrieve the Cookie needed.
+function bingSetCookie(    cookie, group, header, url) {
+    url = HttpPathPrefix "/translator"
+
+    header = "GET " url " HTTP/1.1\n"                                   \
         "Host: " HttpHost "\n"                                          \
         "Connection: close\n"
     if (Option["user-agent"])
         header = header "User-Agent: " Option["user-agent"] "\n"

+    cookie = NULLSTR
+    print header |& HttpService
+    while ((HttpService |& getline) > 0 && length($0) > 1) {
+        match($0, /Set-Cookie: ([^;]*);/, group)
+        if (group[1]) {
+            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
+        }
+        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
+    }
+    close(HttpService)
+    Cookie = cookie
+}
+
+# TTS -- FIXME!
+function bingTTSUrl(text, tl,    narrator) {
+    narrator = Option["narrator"] ~ /^[AFaf]/ ? "female" : "male"
+    # FIXME:
+    # 1. use digraphia code (en-US) as Alpha-2 code alone doen't work
+    # 2. how to pass cookies to an external player?
+    return HttpProtocol HttpHost "/translator/api/language/Speak?"      \
+        "locale=" tl "&text=" preprocess(text)                          \
+        "&gender=" narrator "&media=audio/mp3"
+}
+
+function bingWebTranslateUrl(uri, sl, tl, hl) {
+    return "http://www.microsofttranslator.com/bv.aspx?"        \
+        "from=" sl "&to=" tl "&a=" uri
+}
+
+# Send an HTTP POST request and get response from Bing Translator.
+function bingPost(text, sl, tl, hl,
+                  ####
+                  content, contentLength, group,
+                  header, isBody, reqBody, url) {
+    reqBody = "[{" parameterize("text") ":" parameterize(text) "}]"
+    contentLength = dump(reqBody, group)
+
+    url = HttpPathPrefix "/translator/api/Translate/TranslateArray?"    \
+        "from=" sl "&to=" tl
+
+    header = "POST " url " HTTP/1.1\n"                  \
+        "Host: " HttpHost "\n"                          \
+        "Connection: close\n"                           \
+        "Content-Length: " contentLength "\n"           \
+        "Content-Type: application/json\n"     # must!
+    if (Option["user-agent"])
+        header = header "User-Agent: " Option["user-agent"] "\n"
+    if (Cookie)
+        header = header "Cookie: " Cookie "\n" # must!
+
     content = NULLSTR; isBody = 0
-    print header |& HttpService
+    print (header "\n" reqBody) |& HttpService
     while ((HttpService |& getline) > 0) {
         if (isBody)
             content = content ? content "\n" $0 : $0
         else if (length($0) <= 1)
             isBody = 1
-        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
+        l(sprintf("%4s bytes > %s", length($0), $0))
     }
     close(HttpService)

-    match(content, /rttAppId:"([^"]+)"/, group)
-    if (group[1]) {
-        RTTAppId = group[1]
-    } else {
-        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
-        exit 1
-    }
-}
-
-function bingInit() {
-    genRTTAppId() # generate a one-time key
-
-    HttpProtocol = "http://"
-    HttpHost = "api.microsofttranslator.com"
-    HttpPort = 80
-}
-
+    return assert(content, "[ERROR] Null response.")
+}
+
+# Dictionary API (via HTTP GET).
 function bingRequestUrl(text, sl, tl, hl) {
-    # Quick hack: Bing doesn't have an "auto" language code
-    if (sl == "auto") sl = NULLSTR
-
-    return HttpPathPrefix "/v2/ajax.svc/TranslateArray2?"               \
-        "appId="  preprocess(parameterize(RTTAppId, "\""))              \
-        "&from="  preprocess(parameterize(sl, "\""))                    \
-        "&to="    preprocess(parameterize(tl, "\""))                    \
-        "&texts=" preprocess("[" parameterize(text, "\"") "]")
-}
-
-function bingTTSUrl(text, tl,    narrator) {
-    narrator = Option["narrator"] ~ /^[AFaf]/ ? "female" : "male"
-
-    return HttpProtocol HttpHost "/v2/http.svc/speak?" "appId=" RTTAppId \
-        "&language=" tl "&text=" preprocess(text)                       \
-        "&format=audio/mp3" "&options=MinSize|" narrator
-}
-
-function bingWebTranslateUrl(uri, sl, tl, hl) {
-    return "http://www.microsofttranslator.com/bv.aspx?"        \
-        "from=" sl "&to=" tl "&a=" uri
+    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
+        "from=" sl "&to=" tl "&text=" preprocess(text)
 }

 # Get the translation of a string.
@@ -101,19 +113,19 @@
     _tl = getCode(tl); if (!_tl) _tl = tl
     _hl = getCode(hl); if (!_hl) _hl = hl

-    content = getResponse(text, _sl, _tl, _hl)
-    # Strip the content and get a valid JSON string
-    match(content, /(\[.*\])$/, group)
-    if (!group[0]) {
-        # Display the error message
-        match(content, /"(.*)"$/, group)
-        gsub(/\\u000d/, "\r", group[1])
-        gsub(/\\u000a/, "\n", group[1])
-        e("[ERROR] " group[1])
-        ExitCode = 1
-        return
-    }
-    content = group[1]
+    # Hot-patches for Bing's own translator language codes
+    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
+    if (_sl == "auto")  _sl = "-"
+    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
+    if (_sl == "zh-CN") _sl = "zh-CHS"
+    if (_sl == "zh-TW") _sl = "zh-CHT"
+    if (_tl == "bs")    _tl = "bs-Latn"
+    if (_tl == "zh-CN") _tl = "zh-CHS"
+    if (_tl == "zh-TW") _tl = "zh-CHT"
+
+    bingSetCookie() # must!
+
+    content = bingPost(text, _sl, _tl, _hl)
     tokenize(tokens, content)
     parseJson(ast, tokens)

@@ -124,11 +136,16 @@
         e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
         ExitCode = 1
         return
-    }
-
-    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "TranslatedText"])
-
-    returnIl[0] = il = unparameterize(ast[0 SUBSEP 0 SUBSEP "From"])
+    } else if (ast[0 SUBSEP "Message"]) {
+        e("[ERROR] " unparameterize(ast[0 SUBSEP "Message"]))
+        e("[ERROR] " unparameterize(ast[0 SUBSEP "Details" SUBSEP 0]))
+        ExitCode = 1
+        return
+    }
+
+    translation = unparameterize(ast[0 SUBSEP "items" SUBSEP 0 SUBSEP "text"])
+
+    returnIl[0] = il = unparameterize(ast[0 SUBSEP "from"])
     if (Option["verbose"] < 0)
         return getList(il)

@@ -139,9 +156,11 @@

     } else {
         # Verbose mode
+
         wShowOriginal = Option["show-original"]
         wShowTranslation = Option["show-translation"]
         wShowLanguages = Option["show-languages"]
+        wShowDictionary = Option["show-dictionary"]

         if (wShowOriginal) {
             # Display: original text
@@ -176,6 +195,22 @@
                 r = r prettify("languages-tl", getName(tl))
             r = r prettify("languages", group[3])
         }
+
+        if (wShowDictionary && false) { # FIXME!
+            # Dictionary API
+            # Note: source language must be identified
+            dicContent = getResponse(text, il, _tl, _hl)
+            tokenize(dicTokens, dicContent)
+            parseJson(dicAst, dicTokens) # FIXME: inefficient parser
+
+            if (anything(dicAst)) {
+                # Display: dictionary entries
+                if (r) r = r RS
+                r = r m("-- display dictionary entries")
+
+                # TODO
+            }
+        }
     }

     if (toSpeech) {


Translators/BingTranslator: change to new API (fix #218)

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -2,7 +2,7 @@
 # BingTranslator.awk                                               #
 ####################################################################
 #
-# Last Updated: 18 May 2016
+# Last Updated: 17 Mar 2018
 BEGIN { provides("bing") }

 function bingInit() {
@@ -11,27 +11,39 @@
     HttpPort = 80
 }

-# Retrieve the Cookie needed.
-function bingSetCookie(    cookie, group, header, url) {
+# Retrieve the Cookie and set IG.
+function bingSetCookie(    content, cookie, group, header, isBody, url) {
     url = HttpPathPrefix "/translator"

-    header = "GET " url " HTTP/1.1\r\n"                                   \
-        "Host: " HttpHost "\r\n"                                          \
+    header = "GET " url " HTTP/1.1\r\n"                                 \
+        "Host: " HttpHost "\r\n"                                        \
         "Connection: close\r\n"
     if (Option["user-agent"])
         header = header "User-Agent: " Option["user-agent"] "\r\n"

     cookie = NULLSTR
     print header |& HttpService
-    while ((HttpService |& getline) > 0 && length($0) > 1) {
+    while ((HttpService |& getline) > 0) {
         match($0, /Set-Cookie: ([^;]*);/, group)
         if (group[1]) {
             cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
         }
+        if (isBody)
+            content = content ? content "\r\n" $0 : $0
+        else if (length($0) <= 1)
+            isBody = 1
         l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
     }
     close(HttpService)
+
     Cookie = cookie
+    match(content, /IG:"([^"]+)"/, group)
+    if (group[1]) {
+        IG = group[1]
+    } else {
+        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
+        exit 1
+    }
 }

 function bingTTSUrl(text, tl,
@@ -104,48 +116,36 @@
         "from=" sl "&to=" tl "&a=" uri
 }

-# Send an HTTP POST request and get response from Bing Translator.
-function bingPost(text, sl, tl, hl,
-                  ####
-                  content, contentLength, group,
-                  header, isBody, reqBody, url) {
-    reqBody = "[{" parameterize("text") ":" parameterize(text, "\"") "}]"
-    if (DumpContentengths[reqBody])
-        contentLength = DumpContentengths[reqBody]
-    else
-        contentLength = DumpContentengths[reqBody] = dump(reqBody, group)
-
-    url = HttpPathPrefix "/translator/api/Translate/TranslateArray?"    \
-        "from=" sl "&to=" tl
-
-    header = "POST " url " HTTP/1.1\r\n"                  \
-        "Host: " HttpHost "\r\n"                          \
-        "Connection: close\r\n"                           \
-        "Content-Length: " contentLength "\r\n"           \
-        "Content-Type: application/json\r\n"     # must!
-    if (Option["user-agent"])
-        header = header "User-Agent: " Option["user-agent"] "\r\n"
-    if (Cookie)
-        header = header "Cookie: " Cookie "\r\n" # must!
-
-    content = NULLSTR; isBody = 0
-    print (header "\r\n" reqBody) |& HttpService
-    while ((HttpService |& getline) > 0) {
-        if (isBody)
-            content = content ? content "\r\n" $0 : $0
-        else if (length($0) <= 1)
-            isBody = 1
-        l(sprintf("%4s bytes > %s", length($0), $0))
-    }
-    close(HttpService)
-
-    return assert(content, "[ERROR] Null response.")
-}
-
 # Dictionary API (via HTTP GET).
 function bingRequestUrl(text, sl, tl, hl) {
     return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
         "from=" sl "&to=" tl "&text=" preprocess(text)
+}
+
+function bingPostRequestUrl(text, sl, tl, hl, type) {
+    if (type == "translate")
+        return HttpPathPrefix "/ttranslate?&IG=" IG "&IID=translator.5032.2"
+    else if (type == "translationlookup")
+        return HttpPathPrefix "/ttranslationlookup?&IG=" IG "&IID=translator.5032.2"
+    else if (type == "transliterate")
+        return HttpPathPrefix "/ttransliterate?&IG=" IG "&IID=translator.5032.1"
+    else # type == "detect"
+        return HttpPathPrefix "/tdetect?&IG=" IG "&IID=translator.5032.2"
+}
+
+function bingPostRequestContentType(text, sl, tl, hl, type) {
+    return "application/x-www-form-urlencoded"
+}
+
+function bingPostRequestBody(text, sl, tl, hl, type) {
+    if (type == "translate")
+        return "&text=" quote(text) "&from=" sl "&to=" tl
+    else if (type == "translationlookup")
+        return "&text=" quote(text) "&from=" sl "&to=" tl
+    else if (type == "transliterate")
+        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
+    else # type == "detect"
+        return "&text=" quote(text)
 }

 # Get the translation of a string.
@@ -170,9 +170,24 @@
     _tl = getCode(tl); if (!_tl) _tl = tl
     _hl = getCode(hl); if (!_hl) _hl = hl

+    bingSetCookie() # set IG
+
+    # Language identification
+    il = postResponse(text, _sl, _tl, _hl, "detect")
+    if (!il) {
+        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
+        ExitCode = 1
+        return
+    }
+    returnIl[0] = il
+    if (Option["verbose"] < -1)
+        return il
+    if (Option["verbose"] < 0)
+        return getList(il)
+
     # Hot-patches for Bing's own translator language codes
     # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
-    if (_sl == "auto")  _sl = "-"
+    if (_sl == "auto")  _sl = il
     if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
     if (_sl == "zh-CN") _sl = "zh-CHS"
     if (_sl == "zh-TW") _sl = "zh-CHT"
@@ -180,9 +195,8 @@
     if (_tl == "zh-CN") _tl = "zh-CHS"
     if (_tl == "zh-TW") _tl = "zh-CHT"

-    bingSetCookie() # must!
-
-    content = bingPost(text, _sl, _tl, _hl)
+    # Translation
+    content = postResponse(text, _sl, _tl, _hl, "translate")
     if (Option["dump"])
         return content
     tokenize(tokens, content)
@@ -195,20 +209,13 @@
         e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
         ExitCode = 1
         return
-    } else if (ast[0 SUBSEP "Message"]) {
-        e("[ERROR] " unparameterize(ast[0 SUBSEP "Message"]))
-        e("[ERROR] " unparameterize(ast[0 SUBSEP "Details" SUBSEP 0]))
+    } else if (ast[0 SUBSEP "statusCode"] != "200") {
+        e("[ERROR] statusCode: " ast[0 SUBSEP "statusCode"])
         ExitCode = 1
         return
     }

-    translation = unparameterize(ast[0 SUBSEP "items" SUBSEP 0 SUBSEP "text"])
-
-    returnIl[0] = il = unparameterize(ast[0 SUBSEP "from"])
-    if (Option["verbose"] < -1)
-        return il
-    else if (Option["verbose"] < 0)
-        return getList(il)
+    translation = unparameterize(ast[0 SUBSEP "translationResponse"])

     # Generate output
     if (!isVerbose) {


Move to 2018

Move to 2018

--- scripts/tdb/statuses_gln_auids.awk
+++ scripts/tdb/statuses_gln_auids.awk
@@ -14,7 +14,7 @@

 {
   # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2010 && journal year (or journal end year) is lt or eq to the current year we are testing]
-  current_year = 2017
+  current_year = 2018
   end_year = 0
   incontract = 0
   test_year = ""


SpecialCase "A Midsummer Night's Dream"

SpecialCase "A Midsummer Night's Dream"

--- getBBoxCatalogFrom-sitemap.awk
+++ getBBoxCatalogFrom-sitemap.awk
@@ -173,6 +173,15 @@
         # print "==> EntityId = " EntityId
     }

+    if (title == "A Midsummer Night's Dream") {
+        if (EntityId == "_26179")
+            title = "A Midsummer Night's Dream (1981)"
+        else if (EntityId == "_15999")
+            title = "A Midsummer Night's Dream (2016)"
+        # print "==> title = " title
+        # print "==> EntityId = " EntityId
+    }
+
     if (contentId == "p07gnw9f") {
         EntityId = "_23842"
         # print "==> EntityId = " EntityId


Added a shebang to `wrap.awk`

Added a shebang to `wrap.awk`

--- wrap.awk
+++ wrap.awk
@@ -1,3 +1,5 @@
+#! /usr/bin/awk -f
+#
 # Like the Un*x `fmt` command, but for markdown.
 #
 # https://github.com/wernsey/d.awk


A better lexicon extraction script

A better lexicon extraction script

--- lexicon.awk
+++ lexicon.awk
@@ -4,7 +4,6 @@
 }
 {
     for (i = 1; i <= split($3, words, ", "); i++) {
-        gsub("_", " ", words[i]);
-        print words[i] | "sort -us";
+        print words[i] | "sort --parallel=$(nproc) -S1G -us";
     }
 }


Fix awk script to work with cairo generated postscript

--- rtest/strps.awk
+++ rtest/strps.awk
@@ -1,3 +1,3 @@
 BEGIN{doit = 0}
  { if (doit) print $0 }
- $0 == "%%EndSetup" { doit = 1 }
+ $0 = /%%End.*Setup/ { doit = 1 }


include straight http too

include straight http too

--- scripts/ntriple2dot.awk
+++ scripts/ntriple2dot.awk
@@ -138,6 +138,7 @@
 	prefix["http://www.genome.jp/kegg/pathway/map/map"]="KEGG-img"
 	# in EOM
 	prefix["https://elementsofmorphology.nih.gov/images/terms/"]="EOM_IMG"
+	prefix["http://elementsofmorphology.nih.gov/index.cgi?tid="]="EOM"  # w/o httpS

 	# playing with the idea of a LOCAL identifier
     # not the same as a bnode in that tools won't rewrite identifier


Languages: add Sanskrit

Languages: add Sanskrit

--- include/Languages.awk
+++ include/Languages.awk
@@ -1306,6 +1306,19 @@
     Locale["sm"]["iso"]                = "smo"
     Locale["sm"]["glotto"]             = "samo1305"
     Locale["sm"]["script"]             = "Latn"
+
+    # Sanskrit
+    Locale["sa"]["name"]               = "Sanskrit"
+    Locale["sa"]["endonym"]            = "संस्कृतम्"
+    #Locale["sa"]["translations-of"]
+    #Locale["sa"]["definitions-of"]
+    #Locale["sa"]["synonyms"]
+    #Locale["sa"]["examples"]
+    #Locale["sa"]["see-also"]
+    Locale["sa"]["family"]             = "Indo-European"
+    Locale["sa"]["iso"]                = "san"
+    Locale["sa"]["glotto"]             = "sans1269"
+    Locale["sa"]["script"]             = "Deva"

     #78 Scots Gaelic
     Locale["gd"]["name"]               = "Scots Gaelic"


Translate: don't assert in postResponse()

--- include/Translate.awk
+++ include/Translate.awk
@@ -122,7 +122,7 @@
     }
     close(HttpService)

-    if (status == "301" && location)
+    if ((status == "301" || status == "302") && location)
         content = curl(location)

     return assert(content, "[ERROR] Null response.")
@@ -132,7 +132,7 @@
 function postResponse(text, sl, tl, hl, type,
                       ####
                       content, contentLength, contentType, group,
-                      header, isBody, reqBody, url) {
+                      header, isBody, reqBody, url, status, location) {
     url = _PostRequestUrl(text, sl, tl, hl, type)
     contentType = _PostRequestContentType(text, sl, tl, hl, type)
     reqBody = _PostRequestBody(text, sl, tl, hl, type)
@@ -161,11 +161,20 @@
             content = content ? content "\r\n" $0 : $0
         else if (length($0) <= 1)
             isBody = 1
+        else { # interesting fields in header
+            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
+            if (RSTART) status = group[1]
+            match($0, /^Location: (.*)/, group)
+            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
+        }
         l(sprintf("%4s bytes > %s", length($0), $0))
     }
     close(HttpService)

-    return assert(content, "[ERROR] Null response.")
+    if ((status == "301" || status == "302") && location)
+        content = curlPost(location, reqBody)
+
+    return content
 }

 # Print a string (to output file or terminal pager).


Translators/GoogleTranslate: genderedTrans must be local so that it gets reinitialized on every call

--- include/Translators/GoogleTranslate.awk
+++ include/Translators/GoogleTranslate.awk
@@ -90,7 +90,7 @@
                          wShowPromptMessage, wShowLanguages,
                          wShowOriginalDictionary, wShowDictionary,
                          wShowAlternatives,
-                         hasWordClasses, hasAltTranslations,
+                         genderedTrans, hasWordClasses, hasAltTranslations,
                          i, j, k, group, temp, saveSortedIn) {
     isPhonetic = match(tl, /^@/)
     tl = substr(tl, 1 + isPhonetic)


Preserve the spaces in comments
Fix: if the first word in a comment line, was directly connected to the "#"
comment character the first word was skipped

git-svn-id: 392d0e5ce652024192c571322d40dcc23d6d83b0@153 8a65f31f-a56f-4bd6-98e8-2755f64920ec

--- scripts/callflow.awk
+++ scripts/callflow.awk
@@ -45,7 +45,7 @@
   printf " .pkt-text { color: red; font-family: Trebuchet MS,Helvetica, sans-serif;\n";
   printf "            font-size: 8pt; stroke: none; fill: black;}\n";
   printf " .comment-text { color: black; font-family: Trebuchet MS,Helvetica, sans-serif;\n";
-  printf "            font-size: 9pt; font-style: italic; stroke: none; fill: green;}\n";
+  printf "            font-size: 9pt; stroke: none; fill: green;}\n";
   printf " .host-text { color: black; font-family: Trebuchet MS,Helvetica,sans-serif;\n";
   printf "             font-size: 10pt; stroke:none; fill:blue;}\n";
   printf " .title-text { color: black; font-family: Trebuchet MS,Helvetica,sans-serif;\n";
@@ -115,34 +115,24 @@
   y = NR;
   y = y * yLineSpace + ystart;

-  if($0 ~ "^#") {
-    output = "";
-    split($0, A, " ")
+  if ($0 ~ "^#") {

-    for (i=2; i <= length(A); i++) {
-
-      if (A[i] == "!")
-        break;
-      output = output " " A[i];
-    }
-
-    link = "";
-    if (A[i] == "!") {
-
-      i++;
-      for(;i <= length(A); i++) {
-        link = link " " A[i];
-      }
-    }
-
-    if (link != "") {
-
-      printf("<text x=\"%d\" y=\"%d\" class=\"link\">%s</text>\n", 50, y, output);
-      printf("    <area href=\"%s\" coords=\"%d,%d,%d,%d\"/>\n", link, 50, y-yLineSpace+2, w, y+1) >> "imagemap";
+    # The "!" is link identifier
+    LINK = index($0, "!")
+    if (LINK == 0) {
+      # There is no link, everything behind the first "#" is the comment
+      sub("#", "", $0)
+      printf("<text x=\"%d\" y=\"%d\" class=\"comment-text\" xml:space=\"preserve\">%s</text>\n", leftMargin + 5, y, $0);

     } else {
+      # Make the comment a hyperlink
+      # - Everything before the "!" is the link text
+      # - Everything behind the "!" is the hyperlink
+      output = substr($0, 2, LINK - 2)
+      link = substr($0, LINK + 1)

-      printf("<text x=\"%d\" y=\"%d\" class=\"comment-text\">%s</text>\n", 50, y, output);
+      printf("<text x=\"%d\" y=\"%d\" class=\"link\" xml:space=\"preserve\">%s</text>\n", leftMargin + 5, y, output);
+      printf("    <area href=\"%s\" coords=\"%d,%d,%d,%d\"/>\n", link, leftMargin + 5, y-yLineSpace+2, w, y+1) >> "imagemap";
     }

   } else {
@@ -168,11 +158,11 @@
     l2 = sprintf("%s:%s", $7,$8);
     for (i=0; i<numHosts; i++) {

-      if(l1 ~ hosts[i]) {
+      if (l1 ~ hosts[i]) {
         x1 = strtonum(lookup[hosts[i]]);
       }

-      if(l2 ~ hosts[i]) {
+      if (l2 ~ hosts[i]) {
         x2 = strtonum(lookup[hosts[i]]);
       }
     }


Slightly increase amount of bandwidth of resampling filter for
feeder_rate_quality=3. This have the benefit of reducing aliasing
artifacts due to alias masking.

Spectrogram analysis:

 o Old preset (100:36:0.90)
	http://people.freebsd.org/~ariff/z_comparison/z_q3_old.png

 o New preset (100:36:0.92):
	http://people.freebsd.org/~ariff/z_comparison/z_q3_new.png

Approved by:	re (kib)


layers.awk: handle comments properly in layers.txt

Signed-off-by: Denys Dmytriyenko <d29de71aea38aad3a87d486929cb0aad173ae612@ti.com>
Signed-off-by: Koen Kooi <42e3d96910f922bdd7dd1805fe3f8aff634f391d@dominion.thruhere.net>

--- scripts/layers.awk
+++ scripts/layers.awk
@@ -1,9 +1,6 @@
-
 BEGIN {
 	FS =",";
 	print "Configured layers:\n";
  }

-$1 != "# Name" { print "layer repository name: " $1 "\nlayer uri: " $2 "\nlayer branch/revision: " $3 "/" $4 ; system("${OE_BASE}/scripts/layerman " $1 " "  $2 " " $3 " " $4 " " command " " commandarg);}
-
-
+$1 !~ "^#" { print "layer repository name: " $1 "\nlayer uri: " $2 "\nlayer branch/revision: " $3 "/" $4 ; system("${OE_BASE}/scripts/layerman " $1 " "  $2 " " $3 " " $4 " " command " " commandarg);}


INTEGRATION: CWS changefileheader (1.2.478); FILE MERGED 2008/03/28 15:55:32 rt 1.2.478.1: #i87441# Change license header to LPGL v3.

--- solenv/bin/unxmap-to-macosx-explist.awk
+++ solenv/bin/unxmap-to-macosx-explist.awk
@@ -1,35 +1,31 @@
 #*************************************************************************
 #
-#   OpenOffice.org - a multi-platform office productivity suite
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# Copyright 2008 by Sun Microsystems, Inc.
 #
-#   $RCSfile: unxmap-to-macosx-explist.awk,v $
+# OpenOffice.org - a multi-platform office productivity suite
 #
-#   $Revision: 1.2 $
+# $RCSfile: unxmap-to-macosx-explist.awk,v $
 #
-#   last change: $Author: rt $ $Date: 2007-01-31 08:42:05 $
+# $Revision: 1.3 $
 #
-#   The Contents of this file are made available subject to
-#   the terms of GNU Lesser General Public License Version 2.1.
+# This file is part of OpenOffice.org.
 #
+# OpenOffice.org is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3
+# only, as published by the Free Software Foundation.
 #
-#     GNU Lesser General Public License Version 2.1
-#     =============================================
-#     Copyright 2005 by Sun Microsystems, Inc.
-#     901 San Antonio Road, Palo Alto, CA 94303, USA
+# OpenOffice.org is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License version 3 for more details
+# (a copy is included in the LICENSE file that accompanied this code).
 #
-#     This library is free software; you can redistribute it and/or
-#     modify it under the terms of the GNU Lesser General Public
-#     License version 2.1, as published by the Free Software Foundation.
-#
-#     This library is distributed in the hope that it will be useful,
-#     but WITHOUT ANY WARRANTY; without even the implied warranty of
-#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#     Lesser General Public License for more details.
-#
-#     You should have received a copy of the GNU Lesser General Public
-#     License along with this library; if not, write to the Free Software
-#     Foundation, Inc., 59 Temple Place, Suite 330, Boston,
-#     MA  02111-1307  USA
+# You should have received a copy of the GNU Lesser General Public License
+# version 3 along with OpenOffice.org.  If not, see
+# <http://www.openoffice.org/license.html>
+# for a copy of the LGPLv3 License.
 #
 #*************************************************************************


Minor change so that my testbox at home is marked as a pc300.


More tinkering with the street finder

More tinkering with the street finder

--- lib/addr_utils.awk
+++ lib/addr_utils.awk
@@ -105,14 +105,13 @@
 	f_st = 0

 	# find the street:
-	# street is the 1st element of ary that begins with an integer and has > 1 one word
-	# so, 200 by itself is not a street, but 200 Broadway is
+	# street is the last element in ary[1:nf-1] that begins with an integer and has more
+	# than one word so, 200 by itself is not a street, but 200 Broadway is
 	f_st = 0
 	for(i = 1; i < nf - 1; i++){
 		nf2 = split(ary[i], ary2, /  */)
 		if(ary2[1] ~ /^[1-9][0-9]*$/ && nf2 > 1){
 			f_st = i
-			break
 		}
 	}
 	if(f_st == 0){


Fix parsing on TCP header with options

Fix parsing on TCP header with options

--- parsedump.awk
+++ parsedump.awk
@@ -3,8 +3,11 @@
 {
   if(buffer != 0)
   {
-    key = substr(buffer, 0, keylen);
+    key = substr(buffer, 0, keyLen);
     buffer = 0;
+    keyPos = -1;
+    keyLenPos = -1;
+    extraLenPos = -1;
   }
 }

@@ -18,19 +21,42 @@
   else if($1 ~ /0x[0-9]+:/)
   {
     pos = (substr($1,0,6) + 0);
+
+    # capturing the begin of the tcp payload
     if(pos == 32)
     {
-      keylen = ("0x"$7) + 0;
-      keypos = ("0x"substr($8,0,2)) + 64;
-      offset = keypos % 16;
+      # 4 * tcp header size + IP header size
+      tcpBegin = ("0x"substr($2,0,1) + 0) * 4 + 20;
+      keyLenPos = tcpBegin + 2;
+      keyLenOffset = keyLenPos % 16;
+      extraLenPos = tcpBegin + 4;
+      extraLenOffset = extraLenPos % 16;
     }
-    else if(pos == keypos - offset)
-      buffer = substr($NF, offset+1)
-    else if(buffer != 0 && pos > keypos)
+
+    if(pos >= 32)
     {
-      buffer = buffer $NF;
-      if(length(buffer) >= keylen)
-        flushKey();
+      # get the key length
+      if(pos == keyLenPos - keyLenOffset)
+      {
+        keyLen = ("0x" $(2 + keyLenOffset / 2)) + 0;
+      }
+      # get the extra length
+      if(pos == extraLenPos - extraLenOffset)
+      {
+        extraLen = ("0x"substr($(2 + extraLenOffset / 2),0,2)) + 0;
+        keyPos = tcpBegin + 24 + extraLen;
+        keyOffset = keyPos % 16;
+      }
+      if(pos == keyPos - keyOffset)
+      {
+        buffer = substr($NF, keyOffset+1)
+      }
+      else if(buffer != 0)
+      {
+        buffer = buffer $NF;
+        if(length(buffer) >= keyLen)
+          flushKey();
+      }
     }
   }


sh instead

sh instead

--- tools/push_pop_indent.awk
+++ tools/push_pop_indent.awk
@@ -1,5 +1,6 @@
-#!/bin/awk -f
+#!/bin/sh

+exec awk '
 /pop/ {
 	if(--indent < 0)
 		print "AWK: indent less than zero (" indent ")" #| "cat >&2"
@@ -16,3 +17,4 @@
 /push/ {
 	indent++
 }
+'


space a bit uncollapsed hints

space a bit uncollapsed hints

--- scripts/hinter.awk
+++ scripts/hinter.awk
@@ -114,7 +114,7 @@
   } else {
     hint_format = "\033[1;33m[%s]\033[0m"
     highlight_format = "\033[1;33m%s\033[0m"
-    compound_format = highlight_format hint_format
+    compound_format = highlight_format " " hint_format
   }

   hint_lookup = ""


kube simpler now shows % used

kube simpler now shows % used

--- k8s/ihatekubeformating-simpler.awk
+++ k8s/ihatekubeformating-simpler.awk
@@ -1,48 +1,78 @@
-# functions 1024 based
-function normalize_Ti(used) {
-  if ( match(used, /([0-9]+)Ki/, value_split) )
-    return (sprintf("%.2f", int(value_split[1]) / 1024 / 1024 / 1024) "Ti")
-  else if ( match(used, /([0-9]+)Mi/, value_split) )
-    return (sprintf("%.2f", int(value_split[1]) / 1024 / 1024) "Ti")
-  else if ( match(used, /([0-9]+)Gi/, value_split) )
-    return (sprintf("%.2f", int(value_split[1]) / 1024) "Ti")
-  else if ( match(used, /([0-9]+)Ti/) )
-    return used
-  else
-    return (sprintf("%.2f", int(used) / 1024 / 1024 / 1024 / 1024) "Ti")
+function split_unit(used, output) {
+  if ( match(used, /([0-9]+)Ki/, value_split) ) {
+    output[0] = int(value_split[1])
+    output[1] = "Ki"
+    return 1
+  } else if ( match(used, /([0-9]+)Mi/, value_split) ) {
+    output[0] = int(value_split[1])
+    output[1] = "Mi"
+    return 1
+  } else if ( match(used, /([0-9]+)Gi/, value_split) ) {
+    output[0] = int(value_split[1])
+    output[1] = "Gi"
+    return 1
+  } else if ( match(used, /([0-9]+)Ti/, value_split) ) {
+    output[0] = int(value_split[1])
+    output[1] = "Ti"
+    return 1
+  } else if ( match(used, /([0-9]+)k/, value_split) ) {
+    output[0] = int(value_split[1])
+    output[1] = "k"
+    return 1
+  } else if ( match(used, /([0-9]+)m/, value_split) ) {
+    output[0] = int(value_split[1])
+    output[1] = "m"
+    return 1
+  } else if ( match(used, /([0-9]+)/, value_split) ) {
+		output[0] = int(value_split[1])
+    output[1] = "" # unknown unit
+    return 1
+  }
+  return 0
 }
-# functions 1000 based
-function normalize_k(used) {
-  if ( match(used, /([0-9]+)k/) )
-    return used
-  else if ( match(used, /([0-9]+)m/) )
-    return (sprintf("%.2f", int(used) / 1000 / 1000) "k")
-  else
-    return (sprintf("%.2f", int(used) / 1000) "k")
+function to_unit(a, a_unit, b, b_unit) {
+  if (a_unit == b_unit)
+    return a
+  if (b_unit == "Ti") {
+    if (a_unit == "Ki")
+      return a / 1024 / 1024 / 1024
+    if (a_unit == "Mi")
+      return a / 1024 / 1024
+    if (a_unit == "Gi")
+      return a / 1024
+    if (a_unit == "")
+      return a / 1024 / 1024 / 1024 / 1024
+    print "unknown a unit: " a  a_unit " for b " b b_unit
+    exit 1
+  } else if (b_unit == "Gi") {
+    if (a_unit == "Ki")
+      return a / 1024 / 1024
+    if (a_unit == "Mi")
+      return a / 1024
+    if (a_unit == "Gi")
+      return a
+    if (a_unit == "")
+      return a / 1024 / 1024 / 1024
+    print "unknown a unit: " a  a_unit " for b " b b_unit
+    exit 1
+  } else if (b_unit == "k") {
+    if (a_unit == "")
+      return a / 1000
+    print "unknown a unit: " a  a_unit " for b " b b_unit
+    exit 1
+  }
+  print "unknown b unit: " b b_unit " with a unit " a a_unit
+  exit 1
 }
-function normalize_no_unit(used) {
-  if ( match(used, /([0-9]+)m/) )
-    return sprintf("%.2f", int(used) / 1000)
-  else {
-    return used
+{
+  if (split_unit($2, used) && split_unit($3, hard)) {
+    value = to_unit(used[0], used[1], hard[0], hard[1])
+    #value = normalize_units(unit, $2, $3)
+    if (value != 0) {
+      percentage = value / int(hard[0]) * 100
+      printf "%-55s%-20s%-20s%s\n", $1, (sprintf("%.2f", value) hard[1]), $3, sprintf("%.2f%", percentage)
+    }
+  } else {
+    print $0
   }
 }
-
-# dynamic dispatch
-function normalize_units(used, hard) {
-  if ( match(hard, /[0-9]+Ti/) ) {
-    return normalize_Ti(used)
-  } else if ( match(hard, /[0-9]+k/) ) {
-    return normalize_k(used)
-  } else if ( match(hard, /[0-9]+/) ) {
-    return normalize_no_unit(used)
-  } else {
-    return used
-  }
-}
-{
-  value = normalize_units($2, $3)
-  if (value != 0) {
-    printf "%-55s%-20s%s\n", $1, value, $3
-  }
-}


BUILD: properly escape multiline XML strings in intlfix-xml.awk

--- autotools/intlfix-xml.awk
+++ autotools/intlfix-xml.awk
@@ -25,11 +25,13 @@
       }
       #print ""
     } else if (XMLEVENT == "COMMENT") {
-      gsub ("\\*/", "* /", XMLNAME)
+      gsub ("\\*/", "* /", XMLNAME)     # escape enclosed C-comment ends
       ostring = sprintf ("/* %s */\n", XMLNAME)
       printf ("__XML_LINE__%-3d: %s", OLINE, ostring); gsub (/[^\n]/, "", ostring); OLINE += length (ostring)
     } else if (XMLEVENT == "CHARDATA" && match (TAG, "^_[^_]")) {
-      sub ("^_", "", TAG); gsub ("[\\\"]", "\\" "\\&", XMLNAME)
+      gsub ("[\\\"]", "\\" "\\&", XMLNAME) # escape string quotes
+      gsub (/\n/, "\\n\"\n\"", XMLNAME)    # escape newlines
+      sub ("^_", "", TAG)
       ostring = sprintf ("%s /*<%s/>*/_(\"%s\");\n", MARKER, TAG, XMLNAME);
       printf ("__XML_LINE__%-3d: %s", OLINE, ostring); gsub (/[^\n]/, "", ostring); OLINE += length (ostring)
     } else if (XMLEVENT == "ENDELEM") {


HvD: Added a line to further reduce the number of duplicate lines being printed.


Translate: preserve line breaks in non-interactive brief mode, fix #164

--- include/Translate.awk
+++ include/Translate.awk
@@ -236,6 +236,11 @@
                     repl(line)
                 else
                     translate(line)
+            } else {
+                # Non-interactive brief mode: preserve line breaks
+                if (!Option["interactive"])
+                    if (!Option["verbose"])
+                        p(line)
             }
     } else
         e("[ERROR] File not found: " Option["input"])


Trivial change.

Trivial change.

--- linkparser/linkparser.awk
+++ linkparser/linkparser.awk
@@ -105,8 +105,6 @@

 function rel_install(source, rel_root, new_root) {
 	count++
-	if (pretend)
-		return 0

 	dir = dirname(source)
 	base = basename(source)
@@ -115,6 +113,12 @@

 	if (destdir)
 		new_root = destdir "/" new_root
+
+	if (pretend) {
+		print source > rel_root".source.list"
+		print rel_path"/"base > rel_root".relative.list"
+		return 0
+	}

 	system("mkdir -p " new_root "/" rel_path)
 	system("cp -L " source " " new_root "/" rel_path)


Clean up "No genre/country" message in MHz

--- getMHzFrom-seriesPages.awk
+++ getMHzFrom-seriesPages.awk
@@ -9,6 +9,8 @@
 #           -v ERROR_FILE=$ERROR_FILE -f getMHzFrom-seriesPages.awk
 #
 # INPUT:
+#       <meta name="apple-itunes-app" content="app-id=1096194231, \
+#           app-argument=https://mhzchoice.vhx.tv/agatha-christie-s-criminal-games">
 #       <title>A French Village - MHz Choice</title>
 #       <meta name="description" content="DRAMA | FRANCE | FRENCH WITH ENGLISH SUBTITLES | TV-MA^M
 #       This acclaimed drama is about the German...all in a day's work... its inhabitants.">
@@ -31,6 +33,13 @@
 #       $MARQUEE_FILE, $DESCRIPTION_FILE, $HEADER_FILE,
 #       list of Episode URLs
 #
+
+/meta name="apple-itunes-app" content=/,/>/ {
+    seriesURL = $0
+    sub (/.*app-argument=/,"",seriesURL)
+    sub (/">/,"",seriesURL)
+    sub (/mhzchoice.vhx.tv/,"watch.mhzchoice.com",seriesURL)
+}

 /title>/ {
     seriesTitle = $0
@@ -92,7 +101,7 @@
     print $0 >> DESCRIPTION_FILE
     # if we didn't find a header in this block, print a blank one
     if (headerPrinted == "no") {
-        print "==> No genre/country line: " seriesTitle >> ERROR_FILE
+        print "==> No genre/country line: " seriesTitle " " seriesURL >> ERROR_FILE
         print "\t\t\t" >> HEADER_FILE
     }
 }


publish-home would miss new logos - CTR

publish-home would miss new logos - CTR


Help: update

Help: update

--- include/Help.awk
+++ include/Help.awk
@@ -131,12 +131,15 @@
         ins(2, "Specify the output file.") RS                           \
         RS "Language preference options:" RS                            \
         ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
+            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
             ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
         ins(2, "Specify your home language.") RS                        \
         ins(1, ansi("bold", "-s ") ansi("underline", "CODE")            \
+            ", " ansi("bold", "-sl ") ansi("underline", "CODE")         \
             ", " ansi("bold", "-source ") ansi("underline", "CODE")) RS \
         ins(2, "Specify the source language.") RS                       \
         ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
+            ", " ansi("bold", "-tl ") ansi("underline", "CODE")         \
             ", " ansi("bold", "-target ") ansi("underline", "CODES")) RS \
         ins(2, "Specify the target language(s), joined by '+'.") RS     \
         RS "Other options:" RS                                          \


Main: ignore stderr when invoking `tput cols`, fix #172

--- include/Main.awk
+++ include/Main.awk
@@ -109,7 +109,7 @@

     # Initialize screen width if not set
     if (!Option["width"] && detectProgram("tput", "-V")) {
-        "tput cols" | getline temp
+        "tput cols" SUPERR | getline temp
         Option["width"] = temp ? temp - 2 : 64
     }


Added check for episode 00 to MHz

Added check for episode 00 to MHz

--- getMHzFrom-episodePages.awk
+++ getMHzFrom-episodePages.awk
@@ -102,6 +102,8 @@
 # Extract the episode description and print the composed info
 /<div class="transparent padding-top-medium"/,/<\/div>/ {
     if ($0 ~ /
 Episode number is 00: " episodeURL >> ERROR_FILE
         printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02dE%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
             SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeNumber, episodeTitle, \
             episodeDuration) >> EPISODE_INFO_FILE


Theme: Option["sgr-" name] needs not to be an array

--- include/Theme.awk
+++ include/Theme.awk
@@ -5,8 +5,11 @@
 function prettify(name, string,    i, temp) {
     temp = string
     if ("sgr-" name in Option)
-        for (i in Option["sgr-" name])
-            temp = ansi(Option["sgr-" name][i], temp)
+        if (isarray(Option["sgr-" name]))
+            for (i in Option["sgr-" name])
+                temp = ansi(Option["sgr-" name][i], temp)
+        else
+            temp = ansi(Option["sgr-" name], temp)
     return temp
 }

@@ -27,14 +30,14 @@
         # Default theme
         Option["sgr-original"][1] = "bold"
         Option["sgr-original"][2] = "negative"
-        Option["sgr-translation"][1] = "bold"
-        Option["sgr-prompt-message-original"][1] = "underline"
-        Option["sgr-original-dictionary-detailed-explanation"][1] = "bold"
+        Option["sgr-translation"] = "bold"
+        Option["sgr-prompt-message-original"] = "underline"
+        Option["sgr-original-dictionary-detailed-explanation"] = "bold"
         Option["sgr-original-dictionary-examples-original"][1] = "bold"
         Option["sgr-original-dictionary-examples-original"][2] = "negative"
-        Option["sgr-original-dictionary-see-also-item"][1] = "underline"
-        Option["sgr-dictionary-word"][1] = "bold"
-        Option["sgr-alternatives-original"][1] = "underline"
-        Option["sgr-alternatives-translation"][1] = "bold"
+        Option["sgr-original-dictionary-see-also-item"] = "underline"
+        Option["sgr-dictionary-word"] = "bold"
+        Option["sgr-alternatives-original"] = "underline"
+        Option["sgr-alternatives-translation"] = "bold"
     }
 }