Translators/BingTranslator: update (close #319, #406)

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -2,7 +2,7 @@
 # BingTranslator.awk                                               #
 ####################################################################
 #
-# Last Updated: 25 Jul 2019
+# Last Updated: 8 Jun 2022
 BEGIN { provides("bing") }

 function bingInit() {
@@ -11,9 +11,9 @@
     HttpPort = 80
 }

-# [OBSOLETE?] Retrieve the Cookie and set IG.
-function bingSetIG(    content, cookie, group, header, isBody,
-                       url, status, location) {
+# Set IG, IID, and BingTokenKey (a URL-encoded string containing token and key).
+function bingSetup(    ast, content, cookie, group, header, isBody, key,
+                       location, status, token, tokens, url) {
     url = HttpPathPrefix "/translator"

     header = "GET " url " HTTP/1.1\r\n"                                 \
@@ -46,13 +46,36 @@
     if ((status == "301" || status == "302") && location)
         content = curl(location)
     # FIXME: cookie
-
     Cookie = cookie
+
     match(content, /IG:"([^"]+)"/, group)
     if (group[1]) {
         IG = group[1]
+        l(IG, "IG")
     } else {
-        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
+        e("[ERROR] Failed to extract IG.")
+        exit 1
+    }
+
+    match(content, /data-iid="([^"]+)"/, group)
+    if (group[1]) {
+        IID = group[1]
+        l(IID, "IID")
+    } else {
+        e("[ERROR] Failed to extract IID.")
+        exit 1
+    }
+
+    match(content, /params_RichTranslateHelper = ([^;]+);/, group)
+    if (group[1]) {
+        tokenize(tokens, group[1])
+        parseJson(ast, tokens)
+        key = ast[0 SUBSEP 0]
+        token = unparameterize(ast[0 SUBSEP 1])
+        BingTokenKey = sprintf("&token=%s&key=%s", quote(token), quote(key))
+        l(BingTokenKey, "BingTokenKey")
+    } else {
+        e("[ERROR] Failed to extract token & key.")
         exit 1
     }
 }
@@ -151,7 +174,7 @@
     #else if (type == "transliterate")
     #    return HttpPathPrefix "/ttransliteratev3"
     else # type == "translate"
-        return HttpPathPrefix "/ttranslatev3"
+        return HttpPathPrefix "/ttranslatev3" sprintf("?IG=%s&IID=%s", IG, IID)
 }

 function bingPostRequestContentType(text, sl, tl, hl, type) {
@@ -168,7 +191,7 @@
     #else if (type == "transliterate")
     #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
     else # type == "translate"
-        return "&text=" quote(text) "&fromLang=" sl "&to=" tl
+        return "&text=" quote(text) "&fromLang=" sl "&to=" tl BingTokenKey
 }

 # Get the translation of a string.
@@ -199,7 +222,7 @@
     _tl = getCode(tl); if (!_tl) _tl = tl
     _hl = getCode(hl); if (!_hl) _hl = hl

-    #bingSetIG() # set IG (no longer needed)
+    bingSetup()

     # Hot-patches for Bing's own translator language codes
     # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>


Fixed more site errors

Fixed more site errors

--- getMHzFromSitemap.awk
+++ getMHzFromSitemap.awk
@@ -265,6 +265,14 @@
     # A Night in ... is missing episode numbers
     if (shortEpisodeURL ~ /season:1\/videos\/a-night-in/)
         snEpisodeNumber = 1
+    # Wallander episode 6
+    if (shortEpisodeURL ~ /\/wallander\/season:1\/videos\/mastermind/)
+        snEpisodeNumber = 6
+    # Bureau of Sexist Affairs confuses two episodes
+    if (shortEpisodeURL ~ /\/bureau-of-sexist-affairs\/season:1\/videos\/feminist-contradictions/)
+        snEpisodeNumber = 15
+    if (shortEpisodeURL ~ /\/bureau-of-sexist-affairs\/season:1\/videos\/violation-of-privacy/)
+        snEpisodeNumber = 16

     # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
     # Handle normal (Sn 1 Ep 1) with variations in spacing and capitalization


FIx bonus seasonNumbers that should be 1

cryptoftears, newworlds & newtonslaw bonus episodes belong in season 1

--- getAcornFrom-showPages.awk
+++ getAcornFrom-showPages.awk
@@ -176,6 +176,13 @@
     if (episodeURL ~ /_cs\//)
         episodeType = "T"
     #
+    # cryptoftears, newworlds & newtonslaw bonus seasonNumber should be 1
+    if (episodeURL ~ \
+        /\/cryptoftears\/bonus\/|\/newworlds\/bonus\/|\/newtonslaw\/bonus\// \
+        && seasonNumber != 1) {
+            printf ("==> Changed S%02d to S01: /%s\n", seasonNumber, shortEpisodeURL) >> ERRORS
+        seasonNumber = 1
+    }
     # Wrap up this episode
     # =HYPERLINK("https://acorn.tv/1900island/series1/week-one";"1900 Island, S01E01, Week One")
     episodeLink = sprintf ("=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)", episodeURL, showTitle,


Languages: mark as stable: Scots Gaelic

Languages: mark as stable: Scots Gaelic

--- include/Languages.awk
+++ include/Languages.awk
@@ -304,15 +304,14 @@
     Locale["sm"]["glotto"]             = "samo1305"
     Locale["sm"]["script"]             = "Latn"

-    #? Scottish Gaelic
-    Locale["gd"]["support"]            = "unstable"
-    Locale["gd"]["name"]               = "Scottish Gaelic"
+    # Scots Gaelic
+    Locale["gd"]["name"]               = "Scots Gaelic"
     Locale["gd"]["endonym"]            = "Gàidhlig"
-    #Locale["gd"]["translations-of"]
-    #Locale["gd"]["definitions-of"]
-    #Locale["gd"]["synonyms"]
-    #Locale["gd"]["examples"]
-    #Locale["gd"]["see-also"]
+    Locale["gd"]["translations-of"]    = "Eadar-theangachadh airson %s"
+    Locale["gd"]["definitions-of"]     = "Deifiniseanan airson %s"
+    Locale["gd"]["synonyms"]           = "Co-fhaclan"
+    Locale["gd"]["examples"]           = "Buill-eisimpleir"
+    Locale["gd"]["see-also"]           = "Faic na leanas cuideachd"
     Locale["gd"]["family"]             = "Indo-European"
     Locale["gd"]["iso"]                = "gla"
     Locale["gd"]["glotto"]             = "scot1245"


Languages: update Guarani (Google support)

--- include/Languages.awk
+++ include/Languages.awk
@@ -519,6 +519,19 @@
     Locale["el"]["glotto"]             = "mode1248"
     Locale["el"]["script"]             = "Grek"

+    # Guarani
+    Locale["gn"]["name"]               = "Guarani"
+    Locale["gn"]["endonym"]            = "Avañe'ẽ"
+    #Locale["gn"]["translations-of"]
+    #Locale["gn"]["definitions-of"]
+    #Locale["gn"]["synonyms"]
+    #Locale["gn"]["examples"]
+    #Locale["gn"]["see-also"]
+    Locale["gn"]["family"]            = "Tupian"
+    Locale["gn"]["iso"]               = "gug"
+    Locale["gn"]["glotto"]            = "para1311"
+    Locale["gn"]["script"]            = "Latn"
+
     #31 Gujarati
     Locale["gu"]["name"]               = "Gujarati"
     Locale["gu"]["endonym"]            = "ગુજરાતી"
@@ -1752,20 +1765,6 @@
     Locale["fo"]["iso"]                = "fao"
     Locale["fo"]["glotto"]             = "faro1244"
     Locale["fo"]["script"]             = "Latn"
-
-    #? Guarani
-    Locale["gn"]["support"]            = "unstable"
-    Locale["gn"]["name"]               = "Guarani"
-    Locale["gn"]["endonym"]            = "Avañe'ẽ"
-    #Locale["gn"]["translations-of"]
-    #Locale["gn"]["definitions-of"]
-    #Locale["gn"]["synonyms"]
-    #Locale["gn"]["examples"]
-    #Locale["gn"]["see-also"]
-    Locale["gn"]["family"]             = "Tupian"
-    Locale["gn"]["iso"]                = "grn"
-    Locale["gn"]["glotto"]             = "para1311"
-    Locale["gn"]["script"]             = "Latn"

     #? Hill Mari
     Locale["mrj"]["support"]           = "yandex-only"


support `most` as an alternative pager of `less` * fix Debian#762138 (https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=762138)

--- include/Main.awk
+++ include/Main.awk
@@ -83,7 +83,9 @@
             if (ENVIRON["TRANS_MANPAGE"])
                 system("echo -E \"${TRANS_MANPAGE}\" | " \
                        "groff -Wall -mtty-char -mandoc -Tutf8 -Dutf8 -rLL=${COLUMNS}n -rLT=${COLUMNS}n | " \
-                       "less -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\"")
+                       (system("most 2>/dev/null") ?
+                        "less -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\"" :
+                        "most -Cs"))
             else
                 print getHelp()
             exit


version 0.9.0

version 0.9.0

--- metainfo.awk
+++ metainfo.awk
@@ -1,7 +1,7 @@
 BEGIN {
     Name        = "Translate Shell"
     Description = "Google Translate to serve as a command-line tool"
-    Version     = "0.9-dev"
+    Version     = "0.9.0"
     Command     = "trans"
     EntryPoint  = "translate.awk"
 }


usage comments corrected

usage comments corrected

--- english/data_processing/code/dxfinfo.awk
+++ english/data_processing/code/dxfinfo.awk
@@ -2,9 +2,10 @@
 #
 # statistics about entities and layers in a DXF file
 # Zoltan Siki siki.zoltan@epito.bme.hu
-# usage: leltar.awk your_file.dxf
+# usage: dxfinfo.awk your_file.dxf
 #		or
-#        gawk -f leltar.awk your_file.dxf > statistics.txt
+#        gawk -f dxfinfo.awk your_file.dxf > statistics.txt
+#        gawk -f dxfinfo.awk your_file.dxf | sort > statistics.txt

 BEGIN { # initialize variables
 	last = ""; entity = ""; layer = "";


Languages: mark as stable: Xhosa

Languages: mark as stable: Xhosa

--- include/Languages.awk
+++ include/Languages.awk
@@ -440,15 +440,14 @@
     Locale["fy"]["glotto"]             = "west2354"
     Locale["fy"]["script"]             = "Latn"

-    #? Xhosa
-    Locale["xh"]["support"]            = "unstable"
+    # Xhosa
     Locale["xh"]["name"]               = "Xhosa"
     Locale["xh"]["endonym"]            = "isiXhosa"
-    #Locale["xh"]["translations-of"]
-    #Locale["xh"]["definitions-of"]
-    #Locale["xh"]["synonyms"]
-    #Locale["xh"]["examples"]
-    #Locale["xh"]["see-also"]
+    Locale["xh"]["translations-of"]    = "Iinguqulelo zika-%s"
+    Locale["xh"]["definitions-of"]     = "Iingcaciso zika-%s"
+    Locale["xh"]["synonyms"]           = "Izithethantonye"
+    Locale["xh"]["examples"]           = "Imizekelo"
+    Locale["xh"]["see-also"]           = "Kwakhona bona"
     Locale["xh"]["family"]             = "Atlantic-Congo"
     Locale["xh"]["iso"]                = "xho"
     Locale["xh"]["glotto"]             = "xhos1239"


Languages: mark sr-Latn as bing-only

Languages: mark sr-Latn as bing-only

--- include/Languages.awk
+++ include/Languages.awk
@@ -1077,6 +1077,7 @@
     Locale["sr-Cyrl"]["script"]        = "Cyrl"

     #79b Serbian, Latin alphabet
+    Locale["sr-Latn"]["support"]       = "bing-only"
     Locale["sr-Latn"]["name"]          = "Serbian (Latin)"
     Locale["sr-Latn"]["endonym"]       = "srpski"
     Locale["sr-Latn"]["translations-of"] = "Prevodi za „%s“"


moarawk

moarawk

--- generate_no_include_bodies_iwyu_mapping.awk
+++ generate_no_include_bodies_iwyu_mapping.awk
@@ -6,7 +6,12 @@
   body = $0
   header = body
   sub(/_body\.hpp$/, ".hpp", header)
-  print "{ include: [\"\\\"" body "\\\"\", private, \"\\\"" header "\\\"\", public] },"
+  if (header != "massless.hpp" &&
+      header != "massive.hpp" &&
+      header != "oblate.hpp" &&
+      header != "rotating.hpp") {
+    print "{ include: [\"\\\"" body "\\\"\", private, \"\\\"" header "\\\"\", public] },"
+  }
 }

 END {


Mark missing SIP header

Mark missing SIP header

git-svn-id: 392d0e5ce652024192c571322d40dcc23d6d83b0@169 8a65f31f-a56f-4bd6-98e8-2755f64920ec

--- scripts/long2html.awk
+++ scripts/long2html.awk
@@ -170,6 +170,7 @@
       if ($1 ~ "Date:" ) MARK = "sip"
       if ($1 ~ "Expires:" ) MARK = "sip"
       if ($1 ~ "From:" ) MARK = "sip"
+      if ($1 ~ "Max-Forwards:" ) MARK = "sip"
       if ($1 ~ "Min-SE:" ) MARK = "sip"
       if ($1 ~ "P-[A-Z][a-z].*:" ) MARK = "sip"
       if ($1 ~ "Privacy:" ) MARK = "sip"


Languages: ignore LANGUAGE (close #280)

Languages: ignore LANGUAGE (close #280)

--- include/Languages.awk
+++ include/Languages.awk
@@ -2115,10 +2115,6 @@

 # Initialize `UserLang`.
 function initUserLang(    lang, utf) {
-    if (lang = ENVIRON["LANGUAGE"]) {
-        if (!UserLocale) UserLocale = lang
-        utf = utf || tolower(lang) ~ /utf-?8$/
-    }
     if (lang = ENVIRON["LC_ALL"]) {
         if (!UserLocale) UserLocale = lang
         utf = utf || tolower(lang) ~ /utf-?8$/


Adding in flag for no wobble / just exact duplicates

--- SLURM/scripts/split_rmdups.awk
+++ SLURM/scripts/split_rmdups.awk
@@ -27,64 +27,74 @@
 # non-duplicate
 # Juicer version 1.5
 BEGIN{
-	tot=0;
-	name=0;
+    tot=0;
+    name=0;
 }
 {
-	if (tot >= 1000000) {
-		if (p1 != $1 || p2 != $2 || p4 != $4 || p5 != $5 || p8 != $8) {
-			sname = sprintf("%s_msplit%04d_", groupname, name);
-			sscriptname = sprintf("%s/.%s.slurm", debugdir, sname);
-			printf("#!/bin/bash -l\n#SBATCH -o %s/dup-split-%s.out\n#SBATCH -e %s/dup-split-%s.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;awk -f %s/scripts/dups.awk -v name=%s/%s %s/split%04d;\necho Reads:%s\ndate\n", debugdir, name, debugdir, name, queue, groupname, juicedir, dir, sname, dir, name, tot) > sscriptname;
-			sysstring = sprintf("sbatch %s", sscriptname);
-			system(sysstring);
-			outname = sprintf("%s/split%04d", dir, name);
-			close(outname);
-			close(sscriptname);
-
-			name++;
-			tot=0;
-		}
+    if (tot >= 1000000) {
+	if (p1 != $1 || p2 != $2 || p4 != $4 || p5 != $5 || p8 != $8) {
+	    sname = sprintf("%s_msplit%04d_", groupname, name);
+	    sscriptname = sprintf("%s/.%s.slurm", debugdir, sname);
+	    if (justexact) {
+		printf("#!/bin/bash -l\n#SBATCH -o %s/dup-split-%s.out\n#SBATCH -e %s/dup-split-%s.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;awk -f %s/scripts/dups.awk -v nowobble=1 -v name=%s/%s %s/split%04d;\necho Reads:%s\ndate\n", debugdir, name, debugdir, name, queue, groupname, juicedir, dir, sname, dir, name, tot) > sscriptname;
+	    }
+	    else {
+		printf("#!/bin/bash -l\n#SBATCH -o %s/dup-split-%s.out\n#SBATCH -e %s/dup-split-%s.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;awk -f %s/scripts/dups.awk -v name=%s/%s %s/split%04d;\necho Reads:%s\ndate\n", debugdir, name, debugdir, name, queue, groupname, juicedir, dir, sname, dir, name, tot) > sscriptname;
+	    }
+	    sysstring = sprintf("sbatch %s", sscriptname);
+	    system(sysstring);
+	    outname = sprintf("%s/split%04d", dir, name);
+	    close(outname);
+	    close(sscriptname);
+
+	    name++;
+	    tot=0;
 	}
-	outname = sprintf("%s/split%04d", dir, name);
-	print > outname;
-	p1=$1;p2=$2;p4=$4;p5=$5;p6=$6;p8=$8;
-	tot++;
+    }
+    outname = sprintf("%s/split%04d", dir, name);
+    print > outname;
+    p1=$1;p2=$2;p4=$4;p5=$5;p6=$6;p8=$8;
+    tot++;
 }
 END {
-	sname = sprintf("%s_msplit%04d_", groupname, name);
-	sscriptname = sprintf("%s/.%s.slurm", debugdir, sname);
+    sname = sprintf("%s_msplit%04d_", groupname, name);
+    sscriptname = sprintf("%s/.%s.slurm", debugdir, sname);
+    if (justexact) {
+	printf("#!/bin/bash -l\n#SBATCH -o %s/dup-split-%s.out\n#SBATCH -e %s/dup-split-%s.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;awk -f %s/scripts/dups.awk -v nowobble=1 -v name=%s/%s %s/split%04d;\necho Reads:%s\ndate\n", debugdir, name, debugdir, name, queue, groupname, juicedir, dir, sname, dir, name, tot) > sscriptname;
+    }
+    else {
 	printf("#!/bin/bash -l\n#SBATCH -o %s/dup-split-%s.out\n#SBATCH -e %s/dup-split-%s.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;awk -f %s/scripts/dups.awk -v name=%s/%s %s/split%04d;\necho Reads:%s\ndate\n", debugdir, name, debugdir, name, queue, groupname, juicedir, dir, sname, dir, name, tot) > sscriptname;
-	sysstring = sprintf("sbatch %s", sscriptname);
-	system(sysstring);
-	close(sscriptname);
+    }
+    sysstring = sprintf("sbatch %s", sscriptname);
+    system(sysstring);
+    close(sscriptname);

-	sscriptname = sprintf("%s/.%s_msplit.slurm", debugdir, groupname);
-	printf("#!/bin/bash -l\n#SBATCH -o %s/dup-merge.out\n#SBATCH -e %s/dup-merge.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\necho \"\"; cat %s/%s_msplit*_optdups.txt > %s/opt_dups.txt;cat %s/%s_msplit*_dups.txt > %s/dups.txt;cat %s/%s_msplit*_merged_nodups.txt > %s/merged_nodups.txt;\ndate\n", debugdir, debugdir, queue, groupname, dir, groupname, dir, dir, groupname, dir, dir, groupname, dir) > sscriptname;
-	sysstring = sprintf("sbatch %s", sscriptname);
-	system(sysstring);
-	close(sscriptname);
+    sscriptname = sprintf("%s/.%s_msplit.slurm", debugdir, groupname);
+    printf("#!/bin/bash -l\n#SBATCH -o %s/dup-merge.out\n#SBATCH -e %s/dup-merge.err\n#SBATCH --mem=50G\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\necho \"\"; cat %s/%s_msplit*_optdups.txt > %s/opt_dups.txt;cat %s/%s_msplit*_dups.txt > %s/dups.txt;cat %s/%s_msplit*_merged_nodups.txt > %s/merged_nodups.txt;\ndate\n", debugdir, debugdir, queue, groupname, dir, groupname, dir, dir, groupname, dir, dir, groupname, dir) > sscriptname;
+    sysstring = sprintf("sbatch %s", sscriptname);
+    system(sysstring);
+    close(sscriptname);

-	sscriptname = sprintf("%s/.%s_rmsplit.slurm", debugdir, groupname);
-	printf("#!/bin/bash -l\n#SBATCH -o %s/dup-rm.out\n#SBATCH -e %s/dup-rm.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\nrm %s/*_msplit*_optdups.txt; rm %s/*_msplit*_dups.txt; rm %s/*_msplit*_merged_nodups.txt;rm %s/split*;\ndate\n", debugdir, debugdir, queue, groupname, dir, dir, dir, dir) > sscriptname;
-	sysstring = sprintf("sbatch %s", sscriptname);
-	system(sysstring);
-	close(sscriptname);
+#	sscriptname = sprintf("%s/.%s_rmsplit.slurm", debugdir, groupname);
+#	printf("#!/bin/bash -l\n#SBATCH -o %s/dup-rm.out\n#SBATCH -e %s/dup-rm.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\nrm %s/*_msplit*_optdups.txt; rm %s/*_msplit*_dups.txt; rm %s/*_msplit*_merged_nodups.txt;rm %s/split*;\ndate\n", debugdir, debugdir, queue, groupname, dir, dir, dir, dir) > sscriptname;
+#	sysstring = sprintf("sbatch %s", sscriptname);
+#	system(sysstring);
+#	close(sscriptname);

-	sscriptname = sprintf("%s/.%s_finalize.slurm", debugdir, groupname);
-	printf("#!/bin/bash -l\n#SBATCH -o %s/dup-guard-trigger.out\n#SBATCH -e %s/dup-guard-trigger.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\necho %s %s %s %s;\nsqueue -u %s;\ndate\n", debugdir, debugdir, queue, groupname, topDir, site, genomeID, genomePath, user, user) > sscriptname;
-	sysstring = sprintf("sbatch %s", sscriptname);
-	system(sysstring);
-	(sysstring | getline maildupid);
-	var = gensub("[^0-9]", "", "g", maildupid);
-	sysstring = sprintf("scontrol update JobID=%s dependency=afterok:%s", guardjid, var);
-	system(sysstring);
-	close(sscriptname);
-
-	sscriptname = sprintf("%s/.%s_mail.slurm", debugdir, groupname);
-	printf("#!/bin/bash -l\n#SBATCH -o %s/dup-mail.out\n#SBATCH -e %s/dup-mail.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\necho %s %s %s %s | mail -r Juicer@rice.edu -s \"Juicer pipeline finished successfully @ Rice\" -t %s@rice.edu;\ndate\n", debugdir, debugdir, queue, groupname, topDir, site, genomeID, genomePath, user) > sscriptname;
-	sysstring = sprintf("sbatch %s", sscriptname);
-	system(sysstring);
-	close(sscriptname);
+    sscriptname = sprintf("%s/.%s_finalize.slurm", debugdir, groupname);
+    printf("#!/bin/bash -l\n#SBATCH -o %s/dup-guard-trigger.out\n#SBATCH -e %s/dup-guard-trigger.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\necho %s %s %s %s;\nsqueue -u %s;\ndate\n", debugdir, debugdir, queue, groupname, topDir, site, genomeID, genomePath, user, user) > sscriptname;
+    sysstring = sprintf("sbatch %s", sscriptname);
+    system(sysstring);
+    (sysstring | getline maildupid);
+    var = gensub("[^0-9]", "", "g", maildupid);
+    sysstring = sprintf("scontrol update JobID=%s dependency=afterok:%s", guardjid, var);
+    system(sysstring);
+    close(sscriptname);
+
+    sscriptname = sprintf("%s/.%s_mail.slurm", debugdir, groupname);
+    printf("#!/bin/bash -l\n#SBATCH -o %s/dup-mail.out\n#SBATCH -e %s/dup-mail.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\necho %s %s %s %s | mail -r Juicer@rice.edu -s \"Juicer pipeline finished successfully @ Rice\" -t %s@rice.edu;\ndate\n", debugdir, debugdir, queue, groupname, topDir, site, genomeID, genomePath, user) > sscriptname;
+    sysstring = sprintf("sbatch %s", sscriptname);
+    system(sysstring);
+    close(sscriptname);
 }


Languages: add Dogri

Languages: add Dogri

--- include/Languages.awk
+++ include/Languages.awk
@@ -344,6 +344,19 @@
     Locale["dv"]["glotto"]             = "dhiv1236"
     Locale["dv"]["script"]             = "Thaa"
     Locale["dv"]["rtl"]                = "true" # RTL language
+
+    # Dogri
+    Locale["doi"]["name"]              = "Dogri"
+    Locale["doi"]["endonym"]           = "डोगरी"
+    #Locale["doi"]["translations-of"]
+    #Locale["doi"]["definitions-of"]
+    #Locale["doi"]["synonyms"]
+    #Locale["doi"]["examples"]
+    #Locale["doi"]["see-also"]
+    Locale["doi"]["family"]            = "Indo-European"
+    Locale["doi"]["iso"]               = "doi"
+    Locale["doi"]["glotto"]            = "indo1311"
+    Locale["doi"]["script"]            = "Deva"

     #20 Dutch
     Locale["nl"]["name"]               = "Dutch"


Add ARGIND trick #chunks

Add ARGIND trick #chunks


pull the latest slidy script from the internets at presentation time

--- tools/slidy.awk
+++ tools/slidy.awk
@@ -1,5 +1,24 @@
 BEGIN {
     interesting = 1
+}
+
+skipScript && /<\/script>/ {
+  skipScript = 0
+  next
+}
+
+skipScript {
+  next
+}
+
+### TODO:
+#  <link rel="stylesheet" type="text/css" media="screen, projection, print" href="http://www.w3.org/Talks/Tools/Slidy2/styles/slidy.css" />
+
+
+/<script type="text\/javascript" charset="utf-8">/ {
+  print "  <script src='http://www.w3.org/Talks/Tools/Slidy2/scripts/slidy.js' charset='utf-8' type='text/javascript'></script>"
+  skipScript = 1
+  next
 }

 {


Remove quotes

Remove quotes


Languages: add hasDictionary(code)

Languages: add hasDictionary(code)

--- include/Languages.awk
+++ include/Languages.awk
@@ -196,6 +196,7 @@
     Locale["zh-CN"]["iso"]             = "zho-CN"
     Locale["zh-CN"]["glotto"]          = "mand1415"
     Locale["zh-CN"]["script"]          = "Hans"
+    Locale["zh-CN"]["dictionary"]      = "true" # has dictionary

     #14b Chinese (Mandarin), Traditional
     Locale["zh-TW"]["name"]            = "Chinese Traditional"
@@ -209,6 +210,7 @@
     Locale["zh-TW"]["iso"]             = "zho-TW"
     Locale["zh-TW"]["glotto"]          = "mand1415"
     Locale["zh-TW"]["script"]          = "Hant"
+    Locale["zh-TW"]["dictionary"]      = "true" # has dictionary

     #15 Croatian
     Locale["hr"]["name"]               = "Croatian"
@@ -261,6 +263,7 @@
     Locale["nl"]["iso"]                = "nld"
     Locale["nl"]["glotto"]             = "dutc1256"
     Locale["nl"]["script"]             = "Latn"
+    Locale["nl"]["dictionary"]         = "true" # has dictionary

     #19 English (Standard English)
     Locale["en"]["name"]               = "English"
@@ -274,6 +277,7 @@
     Locale["en"]["iso"]                = "eng"
     Locale["en"]["glotto"]             = "stan1293"
     Locale["en"]["script"]             = "Latn"
+    Locale["en"]["dictionary"]         = "true" # has dictionary

     #20 Esperanto
     Locale["eo"]["name"]               = "Esperanto"
@@ -339,6 +343,7 @@
     Locale["fr"]["iso"]                = "fra"
     Locale["fr"]["glotto"]             = "stan1290"
     Locale["fr"]["script"]             = "Latn"
+    Locale["fr"]["dictionary"]         = "true" # has dictionary

     #25 Galician
     Locale["gl"]["name"]               = "Galician"
@@ -378,6 +383,7 @@
     Locale["de"]["iso"]                = "deu"
     Locale["de"]["glotto"]             = "stan1295"
     Locale["de"]["script"]             = "Latn"
+    Locale["de"]["dictionary"]         = "true" # has dictionary

     #28 Greek (Modern Greek)
     Locale["el"]["name"]               = "Greek"
@@ -548,6 +554,7 @@
     Locale["it"]["iso"]                = "ita"
     Locale["it"]["glotto"]             = "ital1282"
     Locale["it"]["script"]             = "Latn"
+    Locale["it"]["dictionary"]         = "true" # has dictionary

     #41 Japanese
     Locale["ja"]["name"]               = "Japanese"
@@ -561,6 +568,7 @@
     Locale["ja"]["iso"]                = "jpn"
     Locale["ja"]["glotto"]             = "nucl1643"
     Locale["ja"]["script"]             = "Jpan"
+    Locale["ja"]["dictionary"]         = "true" # has dictionary

     #42 Javanese, Latin alphabet
     Locale["jv"]["name"]               = "Javanese"
@@ -626,6 +634,7 @@
     Locale["ko"]["iso"]                = "kor"
     Locale["ko"]["glotto"]             = "kore1280"
     Locale["ko"]["script"]             = "Kore"
+    Locale["ko"]["dictionary"]         = "true" # has dictionary

     #47 Lao
     Locale["lo"]["name"]               = "Lao"
@@ -861,6 +870,7 @@
     Locale["pt"]["iso"]                = "por"
     Locale["pt"]["glotto"]             = "port1283"
     Locale["pt"]["script"]             = "Latn"
+    Locale["pt"]["dictionary"]         = "true" # has dictionary

     #65 Punjabi, Gurmukhī alphabet
     Locale["pa"]["name"]               = "Punjabi"
@@ -900,6 +910,7 @@
     Locale["ru"]["iso"]                = "rus"
     Locale["ru"]["glotto"]             = "russ1263"
     Locale["ru"]["script"]             = "Cyrl"
+    Locale["ru"]["dictionary"]         = "true" # has dictionary

     #68 Serbian, Cyrillic alphabet
     Locale["sr"]["name"]               = "Serbian"
@@ -991,6 +1002,7 @@
     Locale["es"]["iso"]                = "spa"
     Locale["es"]["glotto"]             = "stan1288"
     Locale["es"]["script"]             = "Latn"
+    Locale["es"]["dictionary"]         = "true" # has dictionary

     #75 Sundanese, Latin alphabet
     Locale["su"]["name"]               = "Sundanese"
@@ -1306,6 +1318,11 @@
 # Return 1 if a language is R-to-L; otherwise return 0.
 function isRTL(code) {
     return Locale[getCode(code)]["rtl"] ? 1 : 0
+}
+
+# Return 1 if Google provides dictionary data for a language; otherwise return 0.
+function hasDictionary(code) {
+    return Locale[getCode(code)]["dictionary"] ? 1 : 0
 }

 # Return the name of script (writing system).


Languages: alternative language names

Languages: alternative language names

--- include/Languages.awk
+++ include/Languages.awk
@@ -119,8 +119,9 @@
     Locale["be"]["glotto"]             = "bela1254"
     Locale["be"]["script"]             = "Cyrl"

-    #9 Bengali
+    #9 Bengali / Bangla
     Locale["bn"]["name"]               = "Bengali"
+    Locale["bn"]["name2"]              = "Bangla"
     Locale["bn"]["endonym"]            = "বাংলা"
     Locale["bn"]["translations-of"]    = "%s এর অনুবাদ"
     Locale["bn"]["definitions-of"]     = "%s এর সংজ্ঞা"
@@ -333,6 +334,7 @@

     #24 Filipino / Tagalog
     Locale["tl"]["name"]               = "Filipino"
+    Locale["tl"]["name2"]              = "Tagalog"
     Locale["tl"]["endonym"]            = "Tagalog"
     Locale["tl"]["translations-of"]    = "Mga pagsasalin ng %s"
     Locale["tl"]["definitions-of"]     = "Mga kahulugan ng %s"
@@ -885,6 +887,7 @@

     #66 Myanmar / Burmese
     Locale["my"]["name"]               = "Myanmar"
+    Locale["my"]["name2"]              = "Burmese"
     Locale["my"]["endonym"]            = "မြန်မာစာ"
     Locale["my"]["translations-of"]    = "%s၏ ဘာသာပြန်ဆိုချက်များ"
     Locale["my"]["definitions-of"]     = "%s၏ အနက်ဖွင့်ဆိုချက်များ"
@@ -935,8 +938,9 @@
     Locale["or"]["glotto"]             = "macr1269"
     Locale["or"]["script"]             = "Orya"

-    #70 Pashto
+    #70 Pashto / Pushto
     Locale["ps"]["name"]               = "Pashto"
+    Locale["ps"]["name2"]              = "Pushto"
     Locale["ps"]["endonym"]            = "پښتو"
     Locale["ps"]["translations-of"]    = "د %sژباړې"
     Locale["ps"]["definitions-of"]     = "د%s تعریفونه"
@@ -1150,6 +1154,7 @@

     #85 Slovenian / Slovene
     Locale["sl"]["name"]               = "Slovenian"
+    Locale["sl"]["name2"]              = "Slovene"
     Locale["sl"]["endonym"]            = "Slovenščina"
     Locale["sl"]["translations-of"]    = "Prevodi za %s"
     Locale["sl"]["definitions-of"]     = "Razlage za %s"
@@ -1201,8 +1206,9 @@
     Locale["su"]["glotto"]             = "sund1252"
     Locale["su"]["script"]             = "Latn"

-    #89 Swahili
+    #89 Swahili / Kiswahili
     Locale["sw"]["name"]               = "Swahili"
+    Locale["sw"]["name2"]              = "Kiswahili"
     Locale["sw"]["endonym"]            = "Kiswahili"
     Locale["sw"]["translations-of"]    = "Tafsiri ya %s"
     Locale["sw"]["definitions-of"]     = "Ufafanuzi wa %s"
@@ -1227,8 +1233,9 @@
     Locale["sv"]["glotto"]             = "swed1254"
     Locale["sv"]["script"]             = "Latn"

-    #91 Tajik, Cyrillic alphabet
+    #91 Tajik / Tajiki, Cyrillic alphabet
     Locale["tg"]["name"]               = "Tajik"
+    Locale["tg"]["name2"]              = "Tajiki"
     Locale["tg"]["endonym"]            = "Тоҷикӣ"
     Locale["tg"]["translations-of"]    = "Тарҷумаҳои %s"
     Locale["tg"]["definitions-of"]     = "Таърифҳои %s"
@@ -1826,6 +1833,8 @@

         # Names and endonyms as aliases
         LocaleAlias[tolower(Locale[i]["name"])] = i
+        if ("name2" in Locale[i])
+            LocaleAlias[tolower(Locale[i]["name2"])] = i
         LocaleAlias[tolower(Locale[i]["endonym"])] = i
     }

@@ -1881,6 +1890,14 @@
 # Return the name of a language.
 function getName(code) {
     return Locale[getCode(code)]["name"]
+}
+
+# Return all the names of a language, separated by "/"
+function getNames(code) {
+    if ("name2" in Locale[getCode(code)])
+        return Locale[getCode(code)]["name"] " / " Locale[getCode(code)]["name2"]
+    else
+        return Locale[getCode(code)]["name"]
 }

 # Return the endonym of a language.
@@ -2006,7 +2023,7 @@
 }

 # Return detailed information of a language as a string.
-function getDetails(code,    group, iso, language, script) {
+function getDetails(code,    group, iso, script) {
     if (code == "auto" || !getCode(code)) {
         e("[ERROR] Language not found: " code "\n"                      \
           "        Run '-reference / -R' to see a list of available languages.")
@@ -2017,11 +2034,8 @@
     if (isRTL(code)) script = script " (R-to-L)"
     split(getISO(code), group, "-")
     iso = group[1]
-    split(getName(code), group, " ")
-    language = length(group) == 1 ? group[1] "_language" :
-        group[2] ~ /^\(.*\)$/ ? group[1] "_language" : join(group, "_")
     return ansi("bold", sprintf("%s\n", getDisplay(code)))              \
-        sprintf("%-22s%s\n", "Name", ansi("bold", getName(code)))       \
+        sprintf("%-22s%s\n", "Name", ansi("bold", getNames(code)))      \
         sprintf("%-22s%s\n", "Family", ansi("bold", getFamily(code)))   \
         sprintf("%-22s%s\n", "Writing system", ansi("bold", script))    \
         sprintf("%-22s%s\n", "Code", ansi("bold", getCode(code)))       \


Translators/YandexTranslate: support dictionary service

--- include/Translators/YandexTranslate.awk
+++ include/Translators/YandexTranslate.awk
@@ -35,6 +35,35 @@
     return HttpPathPrefix "/api/v1/tr.json/translate?"                  \
         "id=" SID "-0-0&srv=tr-text"                                    \
         "&text=" preprocess(text) "&lang=" (sl == "auto" ? tl : sl "-" tl)
+}
+
+function yandexGetDictionaryResponse(text, sl, tl, hl,    content, header, isBody, url) {
+    # Quick hack: Yandex doesn't support digraphia code (yet)
+    split(sl, group, "-"); sl = group[1]
+    split(tl, group, "-"); tl = group[1]
+
+    url = HttpPathPrefix "/dicservice.json/lookup?"                     \
+        "sid=" SID                                                      \
+        "&text=" preprocess(text) "&lang=" (sl == "auto" ? tl : sl "-" tl)
+
+    header = "GET " url " HTTP/1.1\n"           \
+        "Host: " "dictionary.yandex.net" "\n"   \
+        "Connection: close\n"
+    if (Option["user-agent"])
+        header = header "User-Agent: " Option["user-agent"] "\n"
+
+    content = NULLSTR; isBody = 0
+    print header |& HttpService
+    while ((HttpService |& getline) > 0) {
+        if (isBody)
+            content = content ? content "\n" $0 : $0
+        else if (length($0) <= 1)
+            isBody = 1
+        l(sprintf("%4s bytes > %s", length($0), $0))
+    }
+    close(HttpService)
+
+    return assert(content, "[ERROR] Null response.")
 }

 function yandexTTSUrl(text, tl) {
@@ -78,6 +107,8 @@
                          _sl, _tl, _hl, il,
                          translation,
                          wShowOriginal, wShowTranslation, wShowLanguages,
+                         wShowDictionary, dicContent, dicTokens, dicAst,
+                         i, syn, mean,
                          group, temp) {
     if (!getCode(tl)) {
         # Check if target language is supported
@@ -123,9 +154,11 @@

     } else {
         # Verbose mode
+
         wShowOriginal = Option["show-original"]
         wShowTranslation = Option["show-translation"]
         wShowLanguages = Option["show-languages"]
+        wShowDictionary = Option["show-dictionary"]

         if (wShowOriginal) {
             # Display: original text
@@ -160,6 +193,66 @@
                 r = r prettify("languages-tl", getName(tl))
             r = r prettify("languages", group[3])
         }
+
+        if (wShowDictionary) {
+            # Dictionary API
+            dicContent = yandexGetDictionaryResponse(text, _sl, _tl, _hl)
+            tokenize(dicTokens, dicContent)
+            parseJson(dicAst, dicTokens)
+
+            if (anything(dicAst)) {
+                # Display: dictionary entries
+                if (r) r = r RS
+                r = r m("-- display dictionary entries")
+
+                saveSortedIn = PROCINFO["sorted_in"]
+                PROCINFO["sorted_in"] = "@ind_num_asc"
+                for (i in dicAst) {
+                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
+                        "pos$") {
+                        r = r RS prettify("dictionary-word-class", s((literal(dicAst[i])), hl))
+                        syn = mean = ""
+                    }
+
+                    # TODO: ex, gen, ...
+
+                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
+                        "tr" SUBSEP "[[:digit:]]+" SUBSEP               \
+                        "mean" SUBSEP "[[:digit:]]+" SUBSEP "text") {
+                        if (mean) {
+                            mean = mean prettify("dictionary-explanation", ", ") \
+                                prettify("dictionary-explanation-item", s((literal(dicAst[i])), sl))
+                        } else {
+                            mean = prettify("dictionary-explanation-item", s((literal(dicAst[i])), sl))
+                        }
+                    }
+
+                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
+                        "tr" SUBSEP "[[:digit:]]+" SUBSEP               \
+                        "syn" SUBSEP "[[:digit:]]+" SUBSEP "text") {
+                        if (syn) {
+                            syn = syn prettify("dictionary-explanation", ", ") \
+                                prettify("dictionary-word", s((literal(dicAst[i])), il))
+                        } else {
+                            syn = prettify("dictionary-word", s((literal(dicAst[i])), il))
+                        }
+                    }
+
+                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
+                        "tr" SUBSEP "[[:digit:]]+" SUBSEP "text$") {
+                        text = prettify("dictionary-word", s((literal(dicAst[i])), il))
+                        if (syn) {
+                            r = r RS ins(1, text prettify("dictionary-explanation", ", ") syn)
+                        } else {
+                            r = r RS ins(1, text)
+                        }
+                        r = r RS ins(2, mean)
+                        syn = mean = ""
+                    }
+                }
+                PROCINFO["sorted_in"] = saveSortedIn
+            }
+        }
     }

     if (toSpeech) {


Convert duration in seconds to HH:MM:SS

Convert duration in seconds to HH:MM:SS

--- getBBoxCatalogFrom-sitemap.awk
+++ getBBoxCatalogFrom-sitemap.awk
@@ -111,8 +111,14 @@
 # <duration>3000</duration>
 /<duration>/ {
     split ($0,fld,"[<>]")
-    duration = fld[3]
-    # print "duration = " duration  > "/dev/stderr"
+    seconds = fld[3]
+    secs = seconds % 60
+    mins = int(seconds / 60 % 60)
+    hrs = int(seconds / 3600)
+    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
+    if (duration == "00:00:00")
+        duration = ""
+    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
 }

 # Grab originalDate


fix start_date value

fix start_date value


Change assignments.

Change assignments.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@31188 4f837ed2-42f5-46e7-a7a5-fa17313484d4


geo_utils.awk	-- added GU_get_style_defaults()

--- lib/geo_utils.awk
+++ lib/geo_utils.awk
@@ -6,6 +6,10 @@
 }
 function GU_r2d(r) {
 	return (r+0)*180.0/GU_pi()
+}
+function GU_get_style_defaults(sd) {
+	sd["marker-color"] = "#aae"
+	sd["marker-size"]  = "small"
 }
 function GU_is_righthanded(count, first, x, y,   area, p, x1, y1, x2, y2) {
 	area = 0


testuite report: minor refactoring

testuite report: minor refactoring

--- testsuite/makefiles/summarize.awk
+++ testsuite/makefiles/summarize.awk
@@ -199,11 +199,10 @@
                    ignored);
             printf("  %3d unexpected errors\n", unexped);
             printf("  %3d tests considered", nresults);
-            if (nresults == passed + skipped + ignored + failed + unexped){
-                printf ("\n");
-            }else{
-                printf (" (totals don't add up??)\n");
-            }
+            if (nresults != passed + skipped + ignored + failed + unexped){
+                printf (" (totals don't add up??)");
+            }
+            printf ("\n");
             if (reran != 0){
                 printf("  %3d test dir re-runs\n", reran);
             }


Main: restrict minimum screen width to 4 (close #269)

--- include/Main.awk
+++ include/Main.awk
@@ -117,7 +117,7 @@
         command = "tput cols" SUPERR
         command | getline temp
         close(command)
-        Option["width"] = temp ? temp - 2 : 64
+        Option["width"] = temp > 5 ? temp - 2 : 64  # minimum screen width: 4
     }

     # Disable ANSI escape codes if required


Hide some diagnostic information

Hide some diagnostic information

--- getAcornFrom-showPages.awk
+++ getAcornFrom-showPages.awk
@@ -167,7 +167,7 @@
 # Set showType to "P" for Prequel epiodes so they sort before any Season episodes
 /<h6>Prequel Movies: / {
     showType = "P"
-    printf ("==> Prequel to '%s': %s\n", showTitle, shortEpisodeURL) >> ERRORS
+    # printf ("==> Prequel to '%s': %s\n", showTitle, shortEpisodeURL) >> ERRORS
 }

 # Extract the episode duration
@@ -222,7 +222,7 @@
     # If episode is a Trailer, set episodeType to "T" - even though not all these have episodes
     if (episodeURL ~ /_cs\//) {
         episodeType = "T"
-        printf ("==> Trailer '%s': %s\n", showTitle, shortEpisodeURL) >> ERRORS
+        # printf ("==> Trailer '%s': %s\n", showTitle, shortEpisodeURL) >> ERRORS
     }
     #
     # cryptoftears, newworlds & newtonslaw bonus seasonNumber should be 1


Correction.

Correction.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@33795 4f837ed2-42f5-46e7-a7a5-fa17313484d4

--- scripts/tdb/statuses_gln.awk
+++ scripts/tdb/statuses_gln.awk
@@ -36,7 +36,7 @@
           r[pn] = "4"
         } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Rz")) {
           r[pn] = "5"
-        } else if (substr(p[pn],1,1) >= "Sa") {
+        } else if (substr(p[pn],1,1) >= "SA") {
           r[pn] = "8"
         }
       }


Improve scripts

Improve scripts

--- tools/aggregate.awk
+++ tools/aggregate.awk
@@ -3,9 +3,10 @@
     OFS=":";
 }

-/Timers/ {
-    next
-}
+/Timers/ { next }
+/Total/ { next }
+/whole_optimizations/ { next }
+/all_optimizations/ { next }

 {
     COUNTS[$1] += 1;
@@ -13,9 +14,16 @@
 }

 END {
+    GLOBAL_TOTAL = 0;
+    for(ID in COUNTS) {
+        GLOBAL_TOTAL += TOTALS[ID];
+    }
+
     for(ID in COUNTS) {
         if(TOTALS[ID] > 0){
-            print TOTALS[ID], ID;
+            print TOTALS[ID], ID, (100 * (TOTALS[ID] / GLOBAL_TOTAL));
         }
     }
+
+    print "Total", GLOBAL_TOTAL;
 }


Commons: add w(), e() and d()

Commons: add w(), e() and d()

--- lib/Commons.awk
+++ lib/Commons.awk
@@ -1,4 +1,40 @@
 #!/usr/bin/gawk -f
+
+# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
+# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
+function initAnsiCode() {
+    # Dumb terminal: no ANSI escape code whatsoever
+    if(ENVIRON["TERM"] == "dumb") return
+
+    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"
+
+    AnsiCode["bold"]          = "\33[1m"
+    AnsiCode["underline"]     = "\33[4m"
+    AnsiCode["negative"]      = "\33[7m"
+    AnsiCode["no bold"]       = "\33[21m"
+    AnsiCode["no underline"]  = "\33[24m"
+    AnsiCode["positive"]      = "\33[27m"
+
+    AnsiCode["black"]         = "\33[30m"
+    AnsiCode["red"]           = "\33[31m"
+    AnsiCode["green"]         = "\33[32m"
+    AnsiCode["yellow"]        = "\33[33m"
+    AnsiCode["blue"]          = "\33[34m"
+    AnsiCode["magenta"]       = "\33[35m"
+    AnsiCode["cyan"]          = "\33[36m"
+    AnsiCode["gray"]          = "\33[37m"
+
+    AnsiCode["default"]       = "\33[39m"
+
+    AnsiCode["dark gray"]     = "\33[90m"
+    AnsiCode["light red"]     = "\33[91m"
+    AnsiCode["light green"]   = "\33[92m"
+    AnsiCode["light yellow"]  = "\33[93m"
+    AnsiCode["light blue"]    = "\33[94m"
+    AnsiCode["light magenta"] = "\33[95m"
+    AnsiCode["light cyan"]    = "\33[96m"
+    AnsiCode["white"]         = "\33[97m"
+}

 # Initialize `UrlEncoding`.
 # See: <https://en.wikipedia.org/wiki/Percent-encoding>
@@ -39,53 +75,6 @@
     UrlEncoding["~"]  = "%7E"
 }

-# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
-# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
-function initAnsiCode() {
-    # Dumb terminal: no ANSI escape code whatsoever
-    if(ENVIRON["TERM"] == "dumb") return
-
-    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"
-
-    AnsiCode["bold"]          = "\33[1m"
-    AnsiCode["underline"]     = "\33[4m"
-    AnsiCode["negative"]      = "\33[7m"
-    AnsiCode["no bold"]       = "\33[21m"
-    AnsiCode["no underline"]  = "\33[24m"
-    AnsiCode["positive"]      = "\33[27m"
-
-    AnsiCode["black"]         = "\33[30m"
-    AnsiCode["red"]           = "\33[31m"
-    AnsiCode["green"]         = "\33[32m"
-    AnsiCode["yellow"]        = "\33[33m"
-    AnsiCode["blue"]          = "\33[34m"
-    AnsiCode["magenta"]       = "\33[35m"
-    AnsiCode["cyan"]          = "\33[36m"
-    AnsiCode["gray"]          = "\33[37m"
-
-    AnsiCode["default"]       = "\33[39m"
-
-    AnsiCode["dark gray"]     = "\33[90m"
-    AnsiCode["light red"]     = "\33[91m"
-    AnsiCode["light green"]   = "\33[92m"
-    AnsiCode["light yellow"]  = "\33[93m"
-    AnsiCode["light blue"]    = "\33[94m"
-    AnsiCode["light magenta"] = "\33[95m"
-    AnsiCode["light cyan"]    = "\33[96m"
-    AnsiCode["white"]         = "\33[97m"
-}
-
-# Naive assertion.
-function assert(x, message) {
-    if (!message)
-        message = "[ERROR] Assertion failed."
-
-    if (x)
-        return x
-    else
-        e(message)
-}
-
 # Return the real character represented by an escape sequence.
 # Example: escapeChar("n") returns "\n".
 # See: <https://en.wikipedia.org/wiki/Escape_character>
@@ -220,10 +209,25 @@
     return temp
 }

+# Print warning message.
+function w(text) {
+    print AnsiCode["yellow"] text AnsiCode[0] > "/dev/stderr"
+}
+
+# Print error message.
+function e(text) {
+    print AnsiCode["red"] AnsiCode["bold"] text AnsiCode[0] > "/dev/stderr"
+}
+
+# Print debugging message.
+function d(text) {
+    print AnsiCode["gray"] text AnsiCode[0] > "/dev/stderr"
+}
+
 # Debug an array.
 function da(array, formatString, sortedIn,
-                ####
-                i, j, saveSortedIn) {
+            ####
+            i, j, saveSortedIn) {
     # Default parameters
     if (!formatString)
         formatString = "_[%s]='%s'"
@@ -238,3 +242,14 @@
     }
     PROCINFO["sorted_in"] = saveSortedIn
 }
+
+# Naive assertion.
+function assert(x, message) {
+    if (!message)
+        message = "[ERROR] Assertion failed."
+
+    if (x)
+        return x
+    else
+        e(message)
+}


correct encoding

correct encoding

--- trunk/JLanguageTool/src/dev/tools/stats.awk
+++ trunk/JLanguageTool/src/dev/tools/stats.awk
@@ -1,6 +1,6 @@
 #Script to sort rule matches from LanguageTool
 #Usage: gawk -f stats.awk <file_created_by_LanguageTool>
-#(c) 2008, Marcin Mikowski
+#(c) 2008, Marcin Milkowski
 #Licensed on the terms of LGPL 3.0

 /[0-9]+\.\)/ {


 - Add the character "E" to the understood lock types.  This means
   the VOP requires an exclusive lock.

Sponsored by:	Isilon Systems, Inc.

--- sys/tools/vnode_if.awk
+++ sys/tools/vnode_if.awk
@@ -78,6 +78,8 @@
 			printc(ind"ASSERT_VOP_LOCKED(" star "a->a_"arg", \""uname"\");");
 		else if (lockdata[name, arg, pos] == "U")
 			printc(ind"ASSERT_VOP_UNLOCKED(" star "a->a_"arg", \""uname"\");");
+		else if (lockdata[name, arg, pos] == "E")
+			printc(ind"ASSERT_VOP_ELOCKED(" star "a->a_"arg", \""uname"\");");
 		else if (0) {
 			# XXX More checks!
 		}


Revert "Update min/max length to be inclusive"

This reverts commit 29ea83f7cd7440be605244a9ac75ecd09b903439.

--- bin/bawlk.awk
+++ bin/bawlk.awk
@@ -158,7 +158,7 @@
         msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

     } else if (rule_type ~ /^minLength|maxLength$/) {
-        comparator  = $2 == "maxLength" ? ">=" : "<="
+        comparator  = $2 == "maxLength" ? ">" : "<"
         limit       = params[2]
         term        = $2 == "maxLength" ? "less" : "greater"


Pass doxygen doc comments through to the output.

--- sys/tools/makeobjops.awk
+++ sys/tools/makeobjops.awk
@@ -153,6 +153,21 @@
 }

 #
+# Pass doc comments through to the C file
+#
+function handle_doc ()
+{
+	doc = ""
+	while (!/\*\//) {
+		doc = doc $0 "\n";
+		getline < src;
+		lineno++;
+	}
+	doc = doc $0 "\n";
+	return doc;
+}
+
+#
 #   Handle "CODE" and "HEADER" sections.
 #   Returns the code as-is.
 #
@@ -176,7 +191,7 @@
 #   Handle "METHOD" and "STATICMETHOD" sections.
 #

-function handle_method (static)
+function handle_method (static, doc)
 {
 	#
 	#   Get the return type and function name and delete that from
@@ -282,8 +297,10 @@
 		default = "kobj_error_method";

 	# the method description
+	printh("/** @brief Unique descriptor for the " umname "() method */");
 	printh("extern struct kobjop_desc " mname "_desc;");
 	# the method typedef
+	printh("/** @brief A function implementing the " umname "() method */");
 	prototype = "typedef " ret " " mname "_t(";
 	printh(format_line(prototype argument_list ");",
 	    line_width, length(prototype)));
@@ -298,6 +315,7 @@
 	printc("};\n");

 	# Print out the method itself
+	printh(doc);
 	if (0) {		# haven't chosen the format yet
 		printh("static __inline " ret " " umname "(" varname_list ")");
 		printh("\t" join(";\n\t", arguments, num_arguments) ";");
@@ -422,6 +440,7 @@
 	intname = "";
 	lineno = 0;
 	error = 0;		# to signal clean up and gerror setting
+	lastdoc = "";

 	while (!error && (getline < src) > 0) {
 		lineno++;
@@ -442,17 +461,23 @@

 		if (/^$/) {		# skip empty lines
 		}
-		else if (/^INTERFACE[ 	]+[^ 	;]*[ 	]*;?[ 	]*$/)
+		else if (/^\/\*\*/)
+			lastdoc = handle_doc();
+		else if (/^INTERFACE[ 	]+[^ 	;]*[ 	]*;?[ 	]*$/) {
+			printh(lastdoc);
+			lastdoc = "";
 			handle_interface();
-		else if (/^CODE[ 	]*{$/)
+		} else if (/^CODE[ 	]*{$/)
 			printc(handle_code());
 		else if (/^HEADER[	 ]*{$/)
 			printh(handle_code());
-		else if (/^METHOD/)
-			handle_method(0);
-		else if (/^STATICMETHOD/)
-			handle_method(1);
-		else {
+		else if (/^METHOD/) {
+			handle_method(0, lastdoc);
+			lastdoc = "";
+		} else if (/^STATICMETHOD/) {
+			handle_method(1, lastdoc);
+			lastdoc = "";
+		} else {
 			debug($0);
 			warnsrc("Invalid line encountered");
 			error = 1;


Languages: add Sindhi language

Languages: add Sindhi language

--- include/Languages.awk
+++ include/Languages.awk
@@ -261,6 +261,20 @@
     Locale["rm"]["iso"]                = "roh"
     Locale["rm"]["glotto"]             = "roma1326"
     Locale["rm"]["script"]             = "Latn"
+
+    #? Sindhi
+    Locale["sd"]["name"]               = "Sindhi"
+    Locale["sd"]["endonym"]            = "سنڌي"
+    #Locale["sd"]["translations-of"]
+    #Locale["sd"]["definitions-of"]
+    #Locale["sd"]["synonyms"]
+    #Locale["sd"]["examples"]
+    #Locale["sd"]["see-also"]
+    Locale["sd"]["family"]             = "Indo-European"
+    Locale["sd"]["iso"]                = "snd"
+    Locale["sd"]["glotto"]             = "sind1272"
+    Locale["sd"]["script"]             = "Arab"
+    Locale["sd"]["rtl"]                = "true" # RTL language

     #1 Afrikaans
     Locale["af"]["name"]               = "Afrikaans"


Distribution of permute function is fine.

--- library.awk
+++ library.awk
@@ -172,7 +172,6 @@

 # random permutation of k=n integers between 1 and n
-# the distribution of this isn't great, but it does cover the whole range of permutations
 # a random deck is: split(permute(52,52), deck, SUBSEP)
 function permute(n,  k,   i, r, p) {
     k = checkpos(k, n, "permute: second argument must be positive")


INTEGRATION: CWS locales203 (1.2.2); FILE MERGED 2006/04/20 16:31:23 er 1.2.2.1: #i61058# yet another place for MS-LANGIDs: http://msdn.microsoft.com/library/en-us/intl/nls_238z.asp

--- i18npool/source/isolang/lcid.awk
+++ i18npool/source/isolang/lcid.awk
@@ -1,41 +1,60 @@
 #!/usr/bin/awk -f
 #
-# Utility to compare MS-LANGID definitions with those defined in ../../inc/i18npool/lang.hxx
+# Utility to compare MS-LANGID definitions with those defined in ../../inc/i18npool/lang.h
 # Run in i18npool/source/isolang
 #
 # outputs new #define LANGUAGE_... 0x... and also some commented out substrings
 # that were matched in already existing defines.
 #
-# Expects input from the saved page of
+# ATTENTION! The sed filter in the command line examples below assures that a
+# '|' border is drawn by html2text in data tables, and nowhere else, on which
+# this awk script relies. This script also heavily relies on the column layout
+# encountered. Should MS decide to change their layout or their CSS names
+# ("data..."), this would probably break. Should html2text decide that the last
+# border="..." attribute encountered wins instead of the first, this may break
+# also.
+#
+# sed -e 's/|/,/g; s/<TABLE/<table/g; /<table/\!b; s/\(<table[^>]*\)\(border\|BORDER\)="[0-9]*"/\1/g; s/\(<table\)\([^>]*\(class\|CLASS\)="data\)/\1 border="1"\2/g'
+#
+# After html2text best if file cleaned up to _only_ contain the table entries,
+# but not necessary, entries are filtered. Check output.
+#
+# Expects input from the saved page of one of
 #
 # (1)
 # http://www.microsoft.com/globaldev/reference/lcid-all.mspx
 # filtered through ``html2text -nobs ...'', generated table:
-# blank,name,hex,dec fields:
-#    |Afrikaans_-_South_Africa____________|0436|1078_|
-# Best if file cleaned up to _only_ contain the table entries, but not
-# necessary, entries are filtered. Check output.
+# blank,name,hex,dec,blank fields:
+#    |Afrikaans_-_South_Africa___|0436___|1078___|
 #
 # complete command line:
-# lynx -dump -source http://www.microsoft.com/globaldev/reference/lcid-all.mspx | html2text -nobs | awk -f lcid.awk >outfile
+# lynx -dump -source http://www.microsoft.com/globaldev/reference/lcid-all.mspx | sed -e 's/|/,/g; s/<TABLE/<table/g; /<table/\!b; s/\(<table[^>]*\)\(border\|BORDER\)="[0-9]*"/\1/g; s/\(<table\)\([^>]*\(class\|CLASS\)="data\)/\1 border="1"\2/g' | html2text -nobs -width 234 | awk -f lcid.awk >outfile
 #
 #
 # (2)
 # http://www.microsoft.com/globaldev/reference/winxp/xp-lcid.mspx
 # filtered through ``html2text -nobs ...'', generated table:
-# blank,name,hex,dec,inputlocales,collection fields:
-#    |Afrikaans                           |0436|1078 |0436:       |Basic     |
-# Best if file cleaned up to _only_ contain the table entries, but not
-# necessary, entries are filtered. Check output.
+# blank,name,hex,dec,inputlocales,collection,blank fields:
+#    |Afrikaans   |0436   |1078   |0436:00000409,   |Basic   |
 #
 # complete command line:
-# lynx -dump -source http://www.microsoft.com/globaldev/reference/winxp/xp-lcid.mspx | html2text -nobs | awk -f lcid.awk >outfile
+# lynx -dump -source http://www.microsoft.com/globaldev/reference/winxp/xp-lcid.mspx | sed -e 's/|/,/g; s/<TABLE/<table/g; /<table/\!b; s/\(<table[^>]*\)\(border\|BORDER\)="[0-9]*"/\1/g; s/\(<table\)\([^>]*\(class\|CLASS\)="data\)/\1 border="1"\2/g' | html2text -nobs -width 234 | awk -f lcid.awk >outfile
+#
+#
+# (3)
+# http://msdn.microsoft.com/library/en-us/intl/nls_238z.asp
+# filtered through ``html2text -nobs ...'', generated table:
+# blank,hex,locale,name,blank  fields:
+#   |0x0436___|af-ZA___|Afrikaans_(South_Africa)___|
+#
+# complete command line:
+# lynx -dump -source http://msdn.microsoft.com/library/en-us/intl/nls_238z.asp | sed -e 's/|/,/g; s/<TABLE/<table/g; /<table/\!b; s/\(<table[^>]*\)\(border\|BORDER\)="[0-9]*"/\1/g; s/\(<table\)\([^>]*\(class\|CLASS\)="data\)/\1 border="1"\2/g' | html2text -nobs -width 234 | awk -f lcid.awk >outfile
 #
 # Author: Eike Rathke <erack@sun.com>, <er@openoffice.org>
 #

 BEGIN {
-    while ((getline < "../../inc/i18npool/lang.hxx") > 0)
+    while ((getline < "../../inc/i18npool/lang.h") > 0)
     {
         if ($0 ~ /^#define[ ]*LANGUAGE_[_A-Za-z0-9]*[ ]*0x[0-9a-fA-F]/)
         {
@@ -49,43 +68,76 @@
     filetype = 0
     lcid_all = 1
     xp_lcid  = 2
+    nls_238z = 3
+    filetypename[filetype] = "unknown"
+    filetypename[lcid_all] = "lcid_all"
+    filetypename[xp_lcid]  = "xp_lcid"
+    filetypename[nls_238z] = "nls_238z"
+    namefield[lcid_all] = 2
+    namefield[xp_lcid]  = 2
+    namefield[nls_238z] = 4
+    hexfield[lcid_all]  = 3
+    hexfield[xp_lcid]   = 3
+    hexfield[nls_238z]  = 2
+    locfield[lcid_all]  = 0
+    locfield[xp_lcid]   = 0
+    locfield[nls_238z]  = 3
 }

 (NF < 5) { next }

 !filetype {
     if (NF == 5)
-        filetype = lcid_all
+    {
+        if ($2 ~ /^0x/)
+            filetype = nls_238z
+        else if ($2 ~ /^Afrikaans/)
+            filetype = lcid_all
+    }
     else if (NF == 7)
         filetype = xp_lcid
-    else
+    if (!filetype)
         next
+    name = namefield[filetype]
+    hex = hexfield[filetype]
+    loc = locfield[filetype]
 }

-($3 !~ /^[0-9a-fA-F][0-9a-fA-F]*$/) { filtered[$3] = $0; next }
+{
+    gsub( /^[^:]*:/, "", $name)
+    gsub( /\..*/, "", $name)
+    gsub( /(^[ _]+)|([ _]+$)/, "", $hex)
+    gsub( /(^[ _]+)|([ _]+$)/, "", $name)
+    if (loc)
+        gsub( /(^[ _]+)|([ _]+$)/, "", $loc)
+}
+
+($hex ~ /^0x/) { $hex = substr( $hex, 3) }
+
+# if only 464 instead of 0464, make it match lang.h
+(length($hex) < 4) { $hex = "0" $hex }
+
+($hex !~ /^[0-9a-fA-F][0-9a-fA-F]*$/) { filtered[$hex] = $0; next }

 # all[HEX]=string
-{ all[toupper($3)] = $2 }
+{ all[toupper($hex)] = $name }
+
+(loc) { comment[toupper($hex)] = "  /* " $loc " */" }

 # new hex: newlang[HEX]=string
-!(toupper($3) in lang) { newlang[toupper($3)] = $2 }
+!(toupper($hex) in lang) { newlang[toupper($hex)] = $name }

 END {
     if (!filetype)
     {
-        print "No file type (lcid-all|xp-lcid) recognized." >>"/dev/stderr"
+        print "No file type recognized." >>"/dev/stderr"
         exit(1)
     }
-    else if (filetype == lcid_all)
-        print "// assuming lcid-all file"
-    else if (filetype == xp_lcid)
-        print "// assuming xp-lcid file"
-    else
-        print "// unknown file type"
+    print "// assuming " filetypename[filetype] " file"
     # every new language
     for (x in newlang)
     {
-        printf( "xxxxxxx LANGUAGE_%-26s 0x%s\n", newlang[x], x)
+        printf( "xxxxxxx LANGUAGE_%-26s 0x%s%s\n", newlang[x], x, comment[x])
         n = split(newlang[x],arr,/[^A-Za-z0-9]/)
         def = ""
         for (i=1; i<=n; ++i)


Utils: fix curl() by parameterize(url)

Utils: fix curl() by parameterize(url)

--- include/Utils.awk
+++ include/Utils.awk
@@ -148,7 +148,7 @@
         w("[WARNING] curl is not found.")
         return NULLSTR
     } else {
-        command = Curl " --location --silent " url
+        command = Curl " --location --silent " parameterize(url)
         content = NULLSTR
         while ((command |& getline line) > 0)
             content = (content ? content "\n" : NULLSTR) line


mpers.awk: add support for DWARF files generated by clang

* mpers.awk: Define default_pointer_size.  Initialize byte_size
for DW_TAG_pointer_type to default_pointer_size.  Set parent
for DW_TAG_member only.  Handle DW_AT_count as an alternative
to DW_AT_upper_bound.

--- mpers.awk
+++ mpers.awk
@@ -1,7 +1,7 @@
 #!/bin/gawk
 #
 # Copyright (c) 2015 Elvira Khabirova <lineprinter0@gmail.com>
-# Copyright (c) 2015 Dmitry V. Levin <ldv@altlinux.org>
+# Copyright (c) 2015-2016 Dmitry V. Levin <ldv@altlinux.org>
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -173,6 +173,8 @@
 	return leave(what_idx, "")
 }
 BEGIN {
+	match(ARCH_FLAG, /[[:digit:]]+/, temparray)
+	default_pointer_size = temparray[0] / 8
 	print "#include <inttypes.h>"
 }
 /^<[[:xdigit:]]+>/ {
@@ -181,9 +183,6 @@
 	idx = "0x" matches[2]
 	array[idx]["idx"] = idx
 	parent[level] = idx
-	if (level > 1) {
-		array[idx]["parent"] = parent[level-1]
-	}
 }
 /^DW_AT_data_member_location/ {
 	if (!match($0, /\(DW_OP_plus_uconst:[[:space:]]+([[:digit:]]+)\)/, temparray))
@@ -211,10 +210,18 @@
 	match($0, /[[:digit:]]+/, temparray)
 	array[parent[level-1]]["upper_bound"] = temparray[0] + 1
 }
+/^DW_AT_count/ {
+	match($0, /[[:digit:]]+/, temparray)
+	array[parent[level-1]]["upper_bound"] = temparray[0]
+}
 /^Abbrev Number:[^(]+\(DW_TAG_/ {
 	if (match($0, /typedef|union_type|structure_type|pointer_type\
 |enumeration_type|array_type|base_type|member/, temparray)) {
 		array[idx]["special"] = temparray[0]
+		if ("pointer_type" == temparray[0])
+			array[idx]["byte_size"] = default_pointer_size
+		if (level > 1 && "member" == temparray[0])
+			array[idx]["parent"] = parent[level-1]
 	}
 }
 END {


Updated timeCalculator to work with the more complex prints.

--- timeCalculator.awk
+++ timeCalculator.awk
@@ -1,13 +1,12 @@
-#!/usr/bin/awk -f
-
-$1 ~ "^Time edu.berkeley.cs." { a[$1] += $2 }
-$1 ~ "^Time spark.timeseries." { a[$1] += $2 }
+#![$2] += $NF }
+$0 ~ "^Time edu.berkeley.cs.amplab.carat." { a[$2] += $NF }
+$0 ~ "^Time spark.timeseries." { a[$2] += $NF }

 END {
         for (k in a){
                 name=k
-                gsub("Time edu.berkeley.cs.amplab.carat", "", name)
-                gsub("Time spark.timeseries.", "", name)
+                gsub("edu.berkeley.cs.amplab.carat.", "", name)
+                gsub("spark.timeseries.", "", name)
                 print a[k], name
         }
 }


Clarify printlist.awk

Clarify printlist.awk

--- printList.awk
+++ printList.awk
@@ -7,17 +7,18 @@
 # No processing on header and other lines unrelated to shows
 /=HYPERLINK/ {
     split ($1,str,"\"")
-    description = str[4] "\n"
+    title = str[4] "\n"
     #
+    duration = ""
     if ($2 && $2 !~ /^00:00/) {
         split ($2,fld,":")
         if (fld[1] != "00" ) {
             sub (/^0/,"",fld[1])
-            description = description fld[1] "h "
+            duration = fld[1] "h "
         }
         sub (/^0/,"",fld[2])
-        description = description fld[2] "m\n"
+        duration = duration fld[2] "m\n"
     }
-    description = description $3 "\n"
-    print description
+    description = $3 "\n"
+    print title duration description
 }


Probably going to rewrite in Python.  Stupid AWK and its lack of... sane language features.

--- tests/multi_app_eval/ma_process_results.awk
+++ tests/multi_app_eval/ma_process_results.awk
@@ -1,8 +1,8 @@
 BEGIN {
-    split("", vanilla_avg_fg_response_times);
-    split("", vanilla_avg_bg_total_throughputs);
-    split("", intnw_avg_fg_response_time);
-    split("", intnw_avg_bg_total_throughputs);
+    split("", vanilla_fg_response_times);
+    split("", vanilla_bg_total_throughputs);
+    split("", intnw_fg_response_time);
+    split("", intnw_bg_total_throughputs);

     split("", vanilla_avg_fg_chunksizes);
     split("", vanilla_avg_bg_chunksizes);
@@ -17,9 +17,15 @@
     run_num = $4;

     # initialize data structures
+    split("", vanilla_fg_sender_nums);
+    split("", vanilla_bg_sender_nums);
+    split("", intnw_fg_sender_nums);
+    split("", intnw_bg_sender_nums);

-    total_send_times = 0;
+    total_send_time = 0;
     send_count = 0;
+    sender_num = 0;
+    sender_count = 0;
 }

 # Worker PID 12345 - intnw foreground sender results
@@ -29,30 +35,56 @@
     label = $6; # 'foreground' or 'background'

     #if it's not the first time...
-    if (current_type != "") {
+    if (current_type != "" && sender_num == 0 || sender_num == pid) {
         # add to running average - depends on whether I'm FG or BG
         # TODO: pick up here.  write down what figures we need
         #       and then figure out how to compute them.
+        # XXX: the array management and other code is really ugly.
+        #  Might be better overall to rewrite in Python;
+        #  the simplicity of AWK's text parsing doesn't appear
+        #  to be worth the headaches.
         if (current_type == "vanilla") {
             if (current_label == "foreground") {
-                #vanilla_avg_fg_response_times[
+                vanilla_fg_response_times[run_num] += (total_send_time / send_count);
             } else if (current_label == "background") {
-
+                vanilla_bg_throughputs[run_num] += ((send_count * current_chunksize) / total_send_time);
             }
         } else if (current_type == "intnw") {
-            if (current_label == "foreground") {
-
+            if (current_label == "foreground") {
+                intnw_fg_response_times[run_num] += (total_send_time / send_count);
             } else if (current_label == "background") {
-
+                intnw_bg_throughputs[run_num] += ((send_count * current_chunksize) / total_send_time);
             }
         }
     }
+
+    if (sender_num != pid) {
+        if (current_type == "vanilla") {
+            if (current_label == "foreground") {
+                vanilla_fg_response_times[run_num] /= sender_count;
+            } else if (current_label == "background") {
+                vanilla_bg_response_times[run_num] /= sender_count;
+            }
+        } else if (current_type == "intnw") {
+            if (current_label == "foreground") {
+                intnw_fg_response_times[run_num] /= sender_count;
+            } else if (current_label == "background") {
+                intnw_bg_response_times[run_num] /= sender_count;
+            }
+        }
+
+        sender_count = 0;
+    } else {
+        sender_count += 1;
+    }
+    sender_num = pid;
+    total_send_time = 0;
+    send_count = 0;

     # initialize data structures
     current_type = type;
     current_label = label;

-
     if (type != "vanilla" && type != "intnw") {
         printf("SAW UNKNOWN TYPE %s\n", type);
     }
@@ -72,8 +104,8 @@
     timestamp = $1;
     response_time = $2;

-
-
+    total_send_time += response_time;
+    send_count += 1;
 }

 # print everything as it is processed
@@ -81,6 +113,22 @@
     print;
 }

+function print_summary(results_array)
+{
+    cur_run = 0;
+    cur_sender = 0;
+    for (run in results_array) {
+        split(run, indices, SUBSEP);
+        if (
+            printf("Run %d:\n", );
+            cur_run = indices[i];
+            sender_num = indices[i+1];
+
+        }
+    }
+
+}
+
 END {
     # print summary


Accounting: better datanodes filtering

Accounting: better datanodes filtering

--- files/accounting/hdfs.awk
+++ files/accounting/hdfs.awk
@@ -57,7 +57,7 @@
 #/^Cache Remaining:.*/		{cache_free=$3}

 /^$/ {
-	if (name != "(none)" && ip !~ /^10\./) {
+	if (name != "(none)" && state != 2) {
 		print "INSERT INTO hdfs (id_measure, hostname, state, full, disk, disk_free, disk_used, block_under, block_corrupt, block_missing) VALUES (last_insert_id(), " dbstr(name) ", " dbi(state) ", " dbi(full) ", IFNULL(" dbi(disk) ", " dbi(disk_free) " + " dbi(disk_used) "), " dbi(disk_free) ", " dbi(disk_used) ", " dbi(block_under) ", " dbi(block_corrupt) ", " dbi(block_missing) ");";
 	}
 	reset()


Shorten header; eliminate useless pubDate

--- getBBoxCatalogFrom-sitemap.awk
+++ getBBoxCatalogFrom-sitemap.awk
@@ -2,9 +2,9 @@

 BEGIN {
     # Print header
-    printf ("contentType\tcontentId\tpubDate\ttitle\tEntityId\tgenre\trating\t")
-    printf ("showType\tduration\tdateType\toriginalDate\tshowContentId\tseasonContentId\t")
-    printf ("seasonNumber\tepisodeNumber\tdescription\n")
+    printf ("Content Type\tContent ID\tTitle\tEntity ID\tGenre\tRating\t")
+    printf ("Show Type\tDuration\tDate Type\tOriginal Date\tShow ID\tSeason ID\t")
+    printf ("Sn #\tEp #\tDescription\n")
 }

 # Don't process credits
@@ -14,6 +14,12 @@

 # Don't use Canadian ratings
 /<rating systemCode="ca-tv">/ {
+    next
+}
+
+# Don't use pubDate - it's too random to be useful
+# <pubDate>2020-06-05T21:51:00Z</pubDate>
+/<pubDate>/ {
     next
 }

@@ -32,7 +38,6 @@
     # print "contentType = " contentType
     # print "contentId = " contentId
     # Make sure no fields will be carried over due to missing keys
-    pubDate = ""
     title = ""
     EntityId = ""
     genre = ""
@@ -46,16 +51,6 @@
     seasonNumber = ""
     episodeNumber = ""
     description = ""
-}
-
-# Grab pubDate
-# <pubDate>2020-06-05T21:51:00Z</pubDate>
-/<pubDate>/ {
-    split ($0,fld,"[<>]")
-    pubDate = fld[3]
-    sub (/T/," ",pubDate)
-    sub (/Z/,"",pubDate)
-    # print "pubDate = " pubDate
 }

 # Grab title
@@ -171,8 +166,8 @@
         EntityId = "_23842"
         # print "EntityId = " EntityId
     }
-    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", contentType, contentId, \
-            pubDate, title, EntityId, genre, rating, showType, duration, dateType, originalDate, \
+    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", contentType, contentId, \
+            title, EntityId, genre, rating, showType, duration, dateType, originalDate, \
             showContentId, seasonContentId, seasonNumber, episodeNumber, description)
 }


increase specificity

increase specificity

--- scripts/ntriple2dot.awk
+++ scripts/ntriple2dot.awk
@@ -94,7 +94,7 @@

 function deoboify(curie,  a){
 	split(curie, a , ":")
-	if(match(a[2], a[1]) > 0){
+	if(match(a[2], a[1]"_") > 0){
 		sub("_", ":", a[2]);
 		curie=a[2]
 	}


changes values rule to be pattern to match w/ dp

--- bin/bawlk.awk
+++ bin/bawlk.awk
@@ -80,11 +80,11 @@

         test=field " " comparator " " limit
         msg=field " in line \" NR \" should be " term " than " limit " and was \" " field " \" "
-    } else if (rule_type == "values") {
-        values=params[2]
+    } else if (rule_type == "pattern") {
+        pattern=params[2]

-        test=field " != \"\" && " field " !~ /^(" values ")$/"
-        msg=field " in line \" NR \" should be one of " values " and was \" " field " \" "
+        test=field " != \"\" && " field " !~ " pattern
+        msg=field " in line \" NR \" should match the following pattern " pattern " and was \" " field " \" "
     }

     if (!options["mode"]) options["mode"] = "text"


let stuff trickle through, clean up and close last slide when open

--- tools/hierarchy.awk
+++ tools/hierarchy.awk
@@ -21,10 +21,8 @@
 }

 /
/ {
-    #print "
"
     heading = 1
     hunt = 1
-    #print
     next
 }

@@ -35,29 +33,25 @@
 }

 /
/ && heading == 0 {
-    #print "
"
-    #print "
"
-    #print "
" hier[1] "
"
     heading = 2
     hunt = 1
-    #print
     next
 }

 /
/ && heading == 0 {
-    #print "
"
-    #print "
"
-    #print "
" hier[1] "
"
-    #print "
" hier[2] "
"
     heading = 3
     hunt = 1
-    #print
     next
 }

-/
/ && heading == 3 {
+/
/ && heading == 0 {
     heading = 4
     hunt = 1
-    #print
     next
+}
+
+{ print }
+
+END {
+    if (slide) print "
"
 }


Translators/BingTranslator: show error if statusCode is 400

--- include/Translators/BingTranslator.awk
+++ include/Translators/BingTranslator.awk
@@ -276,6 +276,12 @@
         return
     }

+    if (ast[0 SUBSEP "statusCode"] == "400") {
+        e("[ERROR] " ucfirst(Option["engine"]) " does not support the specified language(s)")
+        ExitCode = 1
+        return
+    }
+
     translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

     returnIl[0] = il = _sl == "auto-detect" ?


Add Javadoc comments to the constants. (dm)

git-svn-id: 30a5f035a20f1bc647618dbad7eea2a951b61b7c@2613 91a5dbb7-01b9-0310-9b5f-b28072856b6e

--- Bindings/Java/constants.awk
+++ Bindings/Java/constants.awk
@@ -24,7 +24,7 @@
 }

 function brlCommand(name, symbol, value, help) {
-  writeCommandDefinition(name, value)
+  writeCommandDefinition(name, value, help)
 }

 function brlBlock(name, symbol, value, help) {
@@ -32,7 +32,7 @@
   if (name == "PASSKEY") return

   if (value ~ /^0[xX][0-9a-fA-F]+00$/) {
-    writeCommandDefinition(name, toupper(value) "00")
+    writeCommandDefinition(name, toupper(value) "00", help)
   }
 }

@@ -52,11 +52,11 @@
   } else {
     return
   }
-  writeJavaConstant("KEY_FLG_" name, value)
+  writeJavaConstant("KEY_FLG_" name, value, help)
 }

 function brlDot(number, symbol, value, help) {
-  writeJavaConstant("BRL_DOT" number, value)
+  writeJavaConstant("BRL_DOT" number, value, help)
 }

 function apiType(name, symbol, value, help) {
@@ -75,16 +75,21 @@
   }
 }

-function writeCommandDefinition(name, value) {
-  writeJavaConstant("KEY_CMD_" name, "(KEY_TYPE_CMD | " value ")")
+function writeCommandDefinition(name, value, help) {
+  writeJavaConstant("KEY_CMD_" name, "(KEY_TYPE_CMD | " value ")", help)
 }

 function writeKeyDefinition(name, value) {
   writeJavaConstant("KEY_SYM_" name, "(KEY_TYPE_SYM | " value ")")
 }

-function writeJavaConstant(name, value) {
+function writeJavaConstant(name, value, help) {
+  writeJavadocComment(help)
   print "  public static final int " name " = " value ";"
+}
+
+function writeJavadocComment(text) {
+  print "  /** " text " */"
 }

 function hexadecimalValue(value) {


makeDistortosConfiguration.awk: add missing support for hex numbers

--- scripts/makeDistortosConfiguration.awk
+++ scripts/makeDistortosConfiguration.awk
@@ -8,7 +8,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
 # distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #
-# date: 2015-08-21
+# date: 2015-08-25
 #

 BEGIN {
@@ -16,7 +16,7 @@
 	booleanTrueConfigPattern = "^" configPattern "=y$"
 	booleanFalseConfigPattern = "^" configPattern "=n$"
 	notSetConfigPattern = "^# " configPattern " is not set$"
-	numericConfigPattern = "^" configPattern "=[[:digit:]]+$"
+	numericConfigPattern = "^" configPattern "=(([[:digit:]]+)|(0x[[:xdigit:]]+))$"
 	stringConfigPattern = "^" configPattern "=\"([^\"\\\\]|\\\\.)*\"$"

 	print "/**"


indentation: fix awful mixture of tabs and spaces

--- scripts/hinter.awk
+++ scripts/hinter.awk
@@ -124,14 +124,14 @@
   line = $0;
   pos = 0;
   col_pos = 0;
-	col_pos_correction = 0;
+  col_pos_correction = 0;

-	output_line = line;
+  output_line = line;

   while (match(line, finger_patterns)) {
     n_matches += 1;

-		hint = HINTS[n_matches - 1]
+    hint = HINTS[n_matches - 1]
     pos += RSTART;

     col_pos = pos;
@@ -148,7 +148,7 @@

     line_pos = NR;

-		pre_match = substr(output_line, 0, col_pos - 1);
+    pre_match = substr(output_line, 0, col_pos - 1);

     if (COMPACT_HINTS) {
       hint_match = sprintf(compound_format, hint, line_match);
@@ -156,7 +156,7 @@
       hint_match = sprintf(compound_format, line_match, hint);
     }

-		post_match = substr(output_line, col_pos + RLENGTH, length(line) - 1);
+    post_match = substr(output_line, col_pos + RLENGTH, length(line) - 1);

     output_line = pre_match hint_match post_match;


INTEGRATION: CWS localefallback (1.1.386); FILE MERGED 2007/07/05 11:19:59 er 1.1.386.1: beautify output

--- i18npool/source/localedata/data/list-locales.awk
+++ i18npool/source/localedata/data/list-locales.awk
@@ -4,10 +4,14 @@
 #// Author: Eike Rathke <erack@sun.com>

 BEGIN {
+    file = ""
+    count = 0
+}
+
+function init_locale() {
     lcinfo = 0
     inlang = 0
     incoun = 0
-    file = ""
     language = ""
     country = ""
 }
@@ -15,6 +19,8 @@
 FILENAME != file {
     printEntry()
     file = FILENAME
+    ++count
+    init_locale()
 }

 {
@@ -51,9 +57,19 @@

 END {
     printEntry()
+    print "\n" count " locales"
 }

 function printEntry() {
     if ( file )
-        printf( "%s: %s %s\n", file, language, country )
+    {
+        tmp = file
+        gsub( /.*\//, "", tmp )
+        gsub( /\.xml/, "", tmp )
+        split( tmp, iso, /_/ )
+        if ( iso[2] )
+            printf( "%3s_%2s: %s - %s\n", iso[1], iso[2], language, country )
+        else
+            printf( "%3s %2s: %s   %s\n", iso[1], iso[2], language, country )
+    }
 }


-Remove count from dsl lastPass script

-Remove count from dsl lastPass script

-Correct regex for comment


translate.awk: include "include/GenTK"

translate.awk: include "include/GenTK"

--- translate.awk
+++ translate.awk
@@ -4,6 +4,8 @@

 @include "include/Commons"
 @include "include/Utils"
+
+@include "include/GenTK"

 @include "include/Languages"
 @include "include/Help"


Parser: add parseJson()

Parser: add parseJson()

--- include/Parser.awk
+++ include/Parser.awk
@@ -245,7 +245,53 @@
     }
 }

-#
+# JSON parser.
+function parseJson(returnAST, tokens,
+                   arrayStartTokens, arrayEndTokens,
+                   objectStartTokens, objectEndTokens,
+                   commas, colons,
+                   ####
+                   flag, i, j, key, name, p, stack, token) {
+    # Default parameters
+    if (!arrayStartTokens[0])  arrayStartTokens[0]  = "["
+    if (!arrayEndTokens[0])    arrayEndTokens[0]    = "]"
+    if (!objectStartTokens[0]) objectStartTokens[0] = "{"
+    if (!objectEndTokens[0])   objectEndTokens[0]   = "}"
+    if (!commas[0])            commas[0]            = ","
+    if (!colons[0])            colons[0]            = ":"
+
+    stack[p = 0] = 0
+    flag = 0 # ready to read key
+    for (i = 0; i < length(tokens); i++) {
+        token = tokens[i]
+
+        if (belongsTo(token, arrayStartTokens))
+            stack[++p] = 0
+        else if (belongsTo(token, objectStartTokens))
+            stack[++p] = NULLSTR
+        else if (belongsTo(token, objectEndTokens) ||
+                 belongsTo(token, arrayEndTokens))
+            --p
+        else if (belongsTo(token, commas)) {
+            if (isnum(stack[p])) # array
+                stack[p]++ # increase index
+        } else if (belongsTo(token, colons)) {
+            flag = 1 # ready to read value
+        } else if (isnum(stack[p]) || flag) {
+            # Read a value
+            key = stack[0]
+            for (j = 1; j <= p; j++)
+                key = key SUBSEP stack[j]
+            returnAST[key] = token
+            flag = 0 # ready to read key
+        } else {
+            # Read a key
+            stack[p] = unparameterize(token)
+        }
+    }
+}
+
+# S-expr parser.
 function parseList(returnAST, tokens,
                    leftBrackets,
                    rightBrackets,


Commons: add initUrlEncoding() and quote()

--- lib/Commons.awk
+++ lib/Commons.awk
@@ -1,4 +1,43 @@
 #!/usr/bin/gawk -f
+
+# Initialize `UrlEncoding`.
+# See: <https://en.wikipedia.org/wiki/Percent-encoding>
+function initUrlEncoding() {
+    UrlEncoding["\n"] = "%0A"
+    UrlEncoding[" "]  = "%20"
+    UrlEncoding["!"]  = "%21"
+    UrlEncoding["\""] = "%22"
+    UrlEncoding["#"]  = "%23"
+    UrlEncoding["$"]  = "%24"
+    UrlEncoding["%"]  = "%25"
+    UrlEncoding["&"]  = "%26"
+    UrlEncoding["'"]  = "%27"
+    UrlEncoding["("]  = "%28"
+    UrlEncoding[")"]  = "%29"
+    UrlEncoding["*"]  = "%2A"
+    UrlEncoding["+"]  = "%2B"
+    UrlEncoding[","]  = "%2C"
+    UrlEncoding["-"]  = "%2D"
+    UrlEncoding["."]  = "%2E"
+    UrlEncoding["/"]  = "%2F"
+    UrlEncoding[":"]  = "%3A"
+    UrlEncoding[";"]  = "%3B"
+    UrlEncoding["<"]  = "%3C"
+    UrlEncoding["="]  = "%3D"
+    UrlEncoding[">"]  = "%3E"
+    UrlEncoding["?"]  = "%3F"
+    UrlEncoding["@"]  = "%40"
+    UrlEncoding["["]  = "%5B"
+    UrlEncoding["\\"] = "%5C"
+    UrlEncoding["]"]  = "%5D"
+    UrlEncoding["^"]  = "%5E"
+    UrlEncoding["_"]  = "%5F"
+    UrlEncoding["`"]  = "%60"
+    UrlEncoding["{"]  = "%7B"
+    UrlEncoding["|"]  = "%7C"
+    UrlEncoding["}"]  = "%7D"
+    UrlEncoding["~"]  = "%7E"
+}

 # Naive assertion.
 function assert(x, message) {
@@ -59,6 +98,15 @@
     return string
 }

+# Return the URL-encoded string.
+function quote(string,    i, r, s) {
+    r = ""
+    split(string, s, "")
+    for (i = 1; i <= length(s); i++)
+        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
+    return r
+}
+
 # Replicate a string.
 function replicate(string, len,
                    ####


In Competition

In Competition


INTEGRATION: CWS locales23 (1.4.10); FILE MERGED 2007/05/09 11:05:36 er 1.4.10.2: getAllFormats1 is optional 2007/04/27 17:11:15 er 1.4.10.1: #i74008# add optional second LC_FORMAT section (LC_FORMAT_1 element) to locale data to be able to distinguish inheritance

--- i18npool/source/localedata/data/linkermapfile-check.awk
+++ i18npool/source/localedata/data/linkermapfile-check.awk
@@ -15,7 +15,9 @@
     nPublics = 0
     sPublic[nPublics++] = "getAllCalendars_"
     sPublic[nPublics++] = "getAllCurrencies_"
-    sPublic[nPublics++] = "getAllFormats_"
+    sPublic[nPublics++] = "getAllFormats0_"
+    bOptional[nPublics] = 1     # getAllFormats1 most times not present
+    sPublic[nPublics++] = "getAllFormats1_"
     sPublic[nPublics++] = "getBreakIteratorRules_"
     sPublic[nPublics++] = "getCollationOptions_"
     sPublic[nPublics++] = "getCollatorImplementation_"
@@ -53,6 +55,11 @@
             {
                 if ( sSymbol[j,symbol] )
                     bFound = 1
+            }
+            if ( !bFound && bOptional[i] )
+            {
+                print symbol " not present but optional"
+                bFound = 1
             }
             if ( !bFound )
             {


account for abstract vertices

account for abstract vertices

--- bin/gspan2vertex_dict.awk
+++ bin/gspan2vertex_dict.awk
@@ -12,6 +12,11 @@
 /^v/ && !/\^/ && !/#/ {
 	print seqid, vertex_id++, $3, $4-1;
 }
-# non-viewpoint  vertices only increment nucleotide position
+# relational and abstract vertices increment vertex id and have no position
+/^v/ && (/\^/ || /#/) {
+	#print seqid, vertex_id++, $3, -1
+	vertex_id++
+}
+# non-viewpoint vertices are silent
 /^V/ && !/\^/ && !/#/ {
 }


switch to new Google Translate API * fix #45

--- include/Translate.awk
+++ include/Translate.awk
@@ -51,9 +51,10 @@

 # Send an HTTP request and get response from Google Translate.
 function getResponse(text, sl, tl, hl,    content, url) {
-    url = HttpPathPrefix "/translate_a/t?client=t"              \
-        "&ie=UTF-8&oe=UTF-8"                                    \
-        "&text=" preprocess(text) "&sl=" sl "&tl=" tl "&hl=" hl
+    url = HttpPathPrefix "/translate_a/single?client=t"                 \
+        "&ie=UTF-8&oe=UTF-8"                                            \
+        "&dt=bd&dt=ex&dt=ld&dt=md&dt=qca&dt=rw&dt=rm&dt=ss&dt=t&dt=at"  \
+        "&q=" preprocess(text) "&sl=" sl "&tl=" tl "&hl=" hl

     print "GET " url " HTTP/1.1\n"             \
           "Host: " HttpHost "\n"               \
@@ -108,7 +109,7 @@

     # Debug mode
     if (Option["debug"]) {
-        d(sprintf("content='%s'", content))
+        d(content)
         da(tokens, "tokens[%s]='%s'")
         da(ast, "ast[%s]='%s'")
     }


TestUtils: perform curl-related tests only if NETWORK_ACCESS is set to yes (#221)

--- test/TestUtils.awk
+++ test/TestUtils.awk
@@ -30,22 +30,24 @@
         assertTrue(newerVersion("2", "1.9.9999"))
     }

-    T("curl()", 1)
-    {
-        delete tokens; delete ast
-        tokenize(tokens, curl("https://httpbin.org/get"))
-        parseJson(ast, tokens)
-        assertEqual(unparameterize(ast[0 SUBSEP "url"]),
-                    "https://httpbin.org/get")
-    }
+    if (yn(ENVIRON["NETWORK_ACCESS"])) { # if network access enabled
+        T("curl()", 1)
+        {
+            delete tokens; delete ast
+            tokenize(tokens, curl("https://httpbin.org/get"))
+            parseJson(ast, tokens)
+            assertEqual(unparameterize(ast[0 SUBSEP "url"]),
+                        "https://httpbin.org/get")
+        }

-    T("curlPost()", 1)
-    {
-        delete tokens; delete ast
-        tokenize(tokens, curlPost("https://httpbin.org/post", "fizz=buzz"))
-        parseJson(ast, tokens)
-        assertEqual(unparameterize(ast[0 SUBSEP "url"]),
-                    "https://httpbin.org/post")
+        T("curlPost()", 1)
+        {
+            delete tokens; delete ast
+            tokenize(tokens, curlPost("https://httpbin.org/post", "fizz=buzz"))
+            parseJson(ast, tokens)
+            assertEqual(unparameterize(ast[0 SUBSEP "url"]),
+                        "https://httpbin.org/post")
+        }
     }

     T("dump()", 3)


Handle movies rented on Amazon Prime Video

--- generateLinkFrom-text.awk
+++ generateLinkFrom-text.awk
@@ -24,6 +24,10 @@
     sub (/\/ref=stream.*/,"")
 }

+/ref=pd_/ {
+    sub (/\/ref=pd_.*/,"")
+}
+
 /^https:/ {
     link = $0
     if ((getline title) > 0) {


Fix import modue for Tests due to the module rename (#33)

See https://github.com/apple/swift-protobuf/commit/6b1a0b0a270f31b93c3d4255d94f8d8f2f9e1ae0

--- CollectTests.awk
+++ CollectTests.awk
@@ -32,7 +32,7 @@
     TESTCASES=""
     TESTCASE_separator=""
     printf("import XCTest\n")
-    printf("@testable import ProtobufTests\n")
+    printf("@testable import SwiftProtobufTests\n")
     printf("\n")
     printf("private func run_test(test:() -> ()) throws {\n")
     printf("    test()\n")


Update filter_en_uk.awk

Update filter_en_uk.awk

Typos

--- filter_en_uk.awk
+++ filter_en_uk.awk
@@ -5,7 +5,7 @@
     for(x=2;x<=NF;x++){
         do{
             # I found a keys being kept pressed or just released, I print it as it is
-            # This works because (probably) I don't want to keep pressed keys that need to be translated (symbols)
+            # This works because (probably) I don't want to keep pressed keys that need to be translated such as symbols
             if(length($x)>1&&(substr($x,1,1)=="+"||substr($x,1,1)=="-")){macroLine=macroLine $x; break;}
             # An upper-case key, I need to press shift to do it. Using the !=toupper I avoid triggering things like ","==tolower(",") and ","==toupper(",")
             if($x!=tolower($x)){macroLine=macroLine "+lshift," "+" tolower($x) ",-" tolower($x) ",-lshift"; break;}
@@ -28,7 +28,7 @@
             if($x=="("){ macroLine=macroLine "+lctrl,+lshift,+u,+0,-0,+0,-0,+2,-2,+8,-8,-u,-lshift,-lctrl"; break;}
             if($x==")"){ macroLine=macroLine "+lctrl,+lshift,+u,+0,-0,+0,-0,+2,-2,+9,-9,-u,-lshift,-lctrl"; break;}
             if($x=="\""){ macroLine=macroLine "+lctrl,+lshift,+u,+2,-2,+0,-0,+1,-1,+c,-c,-u,-lshift,-lctrl"; break;}
-            # !"!""£$
+
             # A normal key, press and release it, catches all the remaining cases
             macroLine=macroLine "+" $x ",-" $x
         }


awk: fix ignored escaped 'u' warning.

awk: fix ignored escaped 'u' warning.

--- awk/reader.awk
+++ awk/reader.awk
@@ -1,10 +1,10 @@
 function reader_read_string(token,    v, r)
 {
 	token = substr(token, 1, length(token) - 1)
-	gsub(/\\\\/, "\u029e", token)
+	gsub(/\\\\/, "\xf7", token)
 	gsub(/\\"/, "\"", token)
 	gsub(/\\n/, "\n", token)
-	gsub("\u029e", "\\", token)
+	gsub("\xf7", "\\", token)
 	return token
 }


ahrf.awk: allow grave as the first paragraph symbol


change behavior of option -dictionary

change behavior of option -dictionary

--- include/Main.awk
+++ include/Main.awk
@@ -186,6 +186,8 @@
         match(ARGV[pos], /^--?d(i(c(t(i(o(n(a(ry?)?)?)?)?)?)?)?)?$/)
         if (RSTART) {
             Option["show-original-dictionary"] = 1
+            Option["show-dictionary"] = 0
+            Option["show-alternatives"] = 0
             continue
         }


output change to formatted printf

output change to formatted printf

--- english/data_processing/code/minmax.awk
+++ english/data_processing/code/minmax.awk
@@ -8,5 +8,4 @@
     if ($col > ma) { ma = $col; }
   }
 }
-END { print mi, ma; }
-
+END { printf("%.3f %.3f", mi, ma); }


hmm: Sort target list during output.

hmm: Sort target list during output.

Change-Id: Ida1e8ebddfb0f47120160fcf45e630ccfd7e0ffa
GitOrigin-RevId: ecf5325329ec42aaa01c803620e82c37043800a6

--- helpmemake.awk
+++ helpmemake.awk
@@ -74,7 +74,16 @@
   print "Targets available are:";
   print "";
   column = 0;
+
+  # Sort the target list keys into a normal array so we can sort 'em.
   for (target in docs) {
+    targets[++j] = target
+  }
+
+  asort(targets)
+
+  for (target in targets) {
+    target = targets[target]
     printf("%s ", target);
     column += length(target) + 1;
     if (column > 75) {


Commons: fix append() and join()

Commons: fix append() and join()

--- lib/Commons.awk
+++ lib/Commons.awk
@@ -48,11 +48,6 @@
     return string
 }

-# Append an element into an array (zero-based).
-function append(array, element) {
-    array[length(array)] = element
-}
-
 # Return 1 if the array contains anything; otherwise return 0.
 function anything(array,
                   ####
@@ -60,6 +55,11 @@
     for (i in array)
         if (array[i]) return 1
     return 0
+}
+
+# Append an element into an array (zero-based).
+function append(array, element) {
+    array[anything(array) ? length(array) : 0] = element
 }

 # Return an element if it belongs to the array;
@@ -108,7 +108,7 @@
     saveSortedIn = PROCINFO["sorted_in"]
     PROCINFO["sorted_in"] = sortedIn
     for (i in array)
-        if (preserveNull || array[i])
+        if (preserveNull || array[i] != "")
             temp = j++ ? temp separator array[i] : array[i]
     PROCINFO["sorted_in"] = saveSortedIn


extract the episode description

extract the episode description

--- getAcornFrom-showPages.awk
+++ getAcornFrom-showPages.awk
@@ -123,6 +123,22 @@
     sub (/.*acorn\.tv/,"acorn.tv",shortEpisodeURL)
     split (episodeURL, part, "/")
     shortSeasonURL = "acorn.tv/" part[4] "/" part[5]
+    # extract the episode description
+    cmd = "curl -s " episodeURL " | grep '<meta itemprop=\"description\"' | tail -1"
+    while ((cmd | getline desc ) > 0) {
+        split (desc,fld,"\"")
+        episodeDescription = fld[4]
+        # fix sloppy input spacing
+        gsub (/ \./,".",episodeDescription)
+        gsub (/  */," ",episodeDescription)
+        sub (/^ */,"",episodeDescription)
+        sub (/ *$/,"",episodeDescription)
+        # fix funky HTML characters
+        gsub (/&/,"\\&",episodeDescription)
+        gsub (/"/,"\"\"",episodeDescription)
+        gsub (/'/,"'",episodeDescription)
+    }
+    close cmd
     next
 }


Complete quirk handling script by matching the in-kernel table format. Use a union to properly assign values.  Collapse some unnecessary variables in the revision rules.

--- sys/tools/acpi_quirks2h.awk
+++ sys/tools/acpi_quirks2h.awk
@@ -55,7 +55,7 @@
 #
 $1 == "name:" {
 	ENTRY_NAME = $2;
-	printf("const struct acpi_table %s[] = {\n", ENTRY_NAME) > OUTPUT;
+	printf("const struct acpi_q_rule %s[] = {\n", ENTRY_NAME) > OUTPUT;
 }

 #
@@ -77,7 +77,7 @@
 	M = match (REMAINDER, /\"[^\"]*\"/);
 	OEM_TABLE_ID = substr(REMAINDER, M, RLENGTH);

-	printf("\t{ ACPI_TABLE_%s, OEM, %s, %s },\n",
+	printf("\t{ ACPI_TABLE_%s, OEM, {%s}, {%s} },\n",
 	    TABLE, OEM_ID, OEM_TABLE_ID) > OUTPUT;
 }

@@ -91,43 +91,38 @@
 	M = match ($0, /\"[^\"]*\"/);
 	CREATOR = substr($0, M, RLENGTH);

-	printf("\t{ ACPI_TABLE_%s, CREATOR, %s },\n", TABLE, CREATOR) > OUTPUT;
+	printf("\t{ ACPI_TABLE_%s, CREATOR, {%s} },\n",
+	    TABLE, CREATOR) > OUTPUT;
 }

 #
 # OEM REVISION field
 #
 $1 == "oem_rev:" {
-	TYPE = $2;
+	TABLE = $2;
 	SIGN = $3;
 	VALUE = $4;

 	# Parse operand
 	OPERAND = trans_sign(SIGN);

-	# Parse OEM revision ID
-	REV = "OEM_REV";
-
-	printf("\t{ ACPI_TABLE_%s, %s, %s, %s },\n",
-	    TYPE, REV, OPERAND, VALUE) > OUTPUT;
+	printf("\t{ ACPI_TABLE_%s, OEM_REV, {.op = %s}, {.rev = %s} },\n",
+	    TABLE, OPERAND, VALUE) > OUTPUT;
 }

 #
 # CREATOR REVISION field
 #
 $1 == "creator_rev:" {
-	TYPE = $2;
+	TABLE = $2;
 	SIGN = $3;
 	VALUE = $4;

 	# Parse operand
 	OPERAND = trans_sign(SIGN);

-	# Parse creator revision ID
-	REV = "CREATOR_REV";
-
-	printf("\t{ ACPI_TABLE_%s, %s, %s, %s },\n",
-	    TYPE, REV, OPERAND, VALUE) > OUTPUT;
+	printf("\t{ ACPI_TABLE_%s, CREATOR_REV, {.op = %s}, {.rev = %s} },\n",
+	    TABLE, OPERAND, VALUE) > OUTPUT;
 }

 #
@@ -149,7 +144,7 @@
 #
 END {
 	# Header
-	printf("const struct acpi_blacklist acpi_quirks_table[] = {\n") \
+	printf("const struct acpi_q_entry acpi_quirks_table[] = {\n") \
 	    > OUTPUT;

 	# Array of all quirks


globals.awk not needed

globals.awk not needed


Commons: add reverse(string)

Commons: add reverse(string)

--- include/Commons.awk
+++ include/Commons.awk
@@ -106,6 +106,16 @@
     temp = NULLSTR
     for (i = 0; i < len; i++)
         temp = temp string
+    return temp
+}
+
+# Reverse a string.
+function reverse(string,
+                 ####
+                 i, temp) {
+    temp = NULLSTR
+    for (i = length(string); i > 0; i--)
+        temp = temp substr(string, i, 1);
     return temp
 }


Commons: add SUPERR

Commons: add SUPERR

--- include/Commons.awk
+++ include/Commons.awk
@@ -6,9 +6,11 @@

 # Initialize constants.
 function initConst() {
-    STDIN = "/dev/stdin"
+    STDIN  = "/dev/stdin"
     STDOUT = "/dev/stdout"
     STDERR = "/dev/stderr"
+
+    SUPERR = " 2> /dev/null" # suppress errors
 }

 # Initialize `UrlEncoding`.
@@ -229,7 +231,8 @@
     return (tolower(string) ~ /^[0fn]/) ? 0 : 1
 }

-# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
+# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
+# (ANSI X3.64 Standard Control Sequences)
 # See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
 function initAnsiCode() {
     # Dumb terminal: no ANSI escape code whatsoever


build.awk: fix wrong sha1sum

build.awk: fix wrong sha1sum

--- build.awk
+++ build.awk
@@ -234,10 +234,6 @@
 }

 function release(    content, group, sha1, size, temp, text) {
-    ("sha1sum " Trans) | getline temp
-    split(temp, group)
-    sha1 = group[1]
-
     d("Updating registry ...")
     # Update registry
     text = readFrom(MainRegistryTemplate)
@@ -261,6 +257,9 @@

     d("Updating gh-pages/translate-shell.rb ...")
     # Update gh-pages/translate-shell.rb
+    ("sha1sum " PagesPath "translate-shell.tar.gz") | getline temp
+    split(temp, group)
+    sha1 = group[1]
     text = readFrom(HomebrewFormula ".temp")
     gsub(/\$sha1\$/, sha1, text)
     gsub(/\$Version\$/, Version, text)
@@ -268,6 +267,9 @@

     d("Updating gh-pages/index.md ...")
     # Update gh-pages/index.md
+    ("sha1sum " Trans) | getline temp
+    split(temp, group)
+    sha1 = group[1]
     content = readFrom(Readme)
     text = readFrom(Index ".temp")
     gsub(/\$sha1\$/, sha1, text)


istio: Add nslookup timeout during container injection

Aborts the DNS lookup if still unsuccessful when timeout elapses

Fixes #6230

Signed-off-by: ifeanyi <4a1d650dde98cdd0188a3f30e06c6f7a5cf63bd7@yahoo.com>

--- examples/kubernetes-istio/cilium-kube-inject.awk
+++ examples/kubernetes-istio/cilium-kube-inject.awk
@@ -11,14 +11,14 @@

 { print }

-# Add an init container to delay the start of the application containers for
-# 10 seconds, to reduce the chance of dropping early traffic.
+# Add an init container to delay the start of the application containers,
+# to reduce the chance of dropping early traffic.
 /initContainers:/ {
 	indent = $0 ; gsub(/[^ ].*/, "", indent)
 	print indent "- name: sleep"
 	print indent "  image: busybox:1.28.4"
 	print indent "  imagePullPolicy: IfNotPresent"
-	print indent "  command: ['sh', '-c', 'until nslookup kube-dns.kube-system.svc.cluster.local; do sleep 1; done']"
+	print indent "  command: ['sh', '-c', 'max=120; i=0; until nslookup kube-dns.kube-system.svc.cluster.local; do i=$((i + 1)); if [ $i -eq $max ]; then echo timed-out; exit 1; else sleep 1; fi done ']"
 }

 # Mount the Cilium state directory to give Cilium's Envoy filters access to the


Languages: update Kurdish dialects (Google support)

--- include/Languages.awk
+++ include/Languages.awk
@@ -420,10 +420,10 @@
     #Locale["ee"]["synonyms"]
     #Locale["ee"]["examples"]
     #Locale["ee"]["see-also"]
-    Locale["ee"]["family"]            = "Atlantic-Congo"
-    Locale["ee"]["iso"]               = "ewe"
-    Locale["ee"]["glotto"]            = "ewee1241"
-    Locale["ee"]["script"]            = "Latn"
+    Locale["ee"]["family"]             = "Atlantic-Congo"
+    Locale["ee"]["iso"]                = "ewe"
+    Locale["ee"]["glotto"]             = "ewee1241"
+    Locale["ee"]["script"]             = "Latn"

     #24 Filipino / Tagalog
     Locale["tl"]["name"]               = "Filipino"
@@ -835,10 +835,10 @@
     Locale["kri"]["glotto"]            = "krio1253"
     Locale["kri"]["script"]            = "Latn"

-    #51 Kurdish (Central Kurdish / Sorani), Latin alphabet (Arabic alphabet for Bing)
-    Locale["ku"]["name"]               = "Kurdish (Central)"
-    Locale["ku"]["name2"]              = "Sorani"
-    Locale["ku"]["endonym"]            = "Soranî"
+    #51 Kurdish (Northern) / Kurmanji
+    Locale["ku"]["name"]               = "Kurdish (Northern)"
+    Locale["ku"]["name2"]              = "Kurmanji"
+    Locale["ku"]["endonym"]            = "Kurmancî"
     Locale["ku"]["endonym2"]           = "Kurdî"
     #Locale["ku"]["translations-of"]
     #Locale["ku"]["definitions-of"]
@@ -846,9 +846,25 @@
     #Locale["ku"]["examples"]
     #Locale["ku"]["see-also"]
     Locale["ku"]["family"]             = "Indo-European"
-    Locale["ku"]["iso"]                = "kur"
-    Locale["ku"]["glotto"]             = "kurd1259"
+    Locale["ku"]["iso"]                = "kmr"
+    Locale["ku"]["glotto"]             = "nort2641"
     Locale["ku"]["script"]             = "Latn"
+
+    # Kurdish (Central) / Sorani
+    Locale["ckb"]["name"]              = "Kurdish (Central)"
+    Locale["ckb"]["name2"]             = "Sorani"
+    Locale["ckb"]["endonym"]           = "سۆرانی"
+    Locale["ckb"]["endonym2"]          = "کوردیی ناوەندی"
+    #Locale["ckb"]["translations-of"]
+    #Locale["ckb"]["definitions-of"]
+    #Locale["ckb"]["synonyms"]
+    #Locale["ckb"]["examples"]
+    #Locale["ckb"]["see-also"]
+    Locale["ckb"]["family"]            = "Indo-European"
+    Locale["ckb"]["iso"]               = "ckb"
+    Locale["ckb"]["glotto"]            = "cent1972"
+    Locale["ckb"]["script"]            = "Arab"
+    Locale["ckb"]["rtl"]               = "true" # RTL language

     #52 Kyrgyz
     Locale["ky"]["name"]               = "Kyrgyz"
@@ -1724,16 +1740,6 @@
     Locale["tlh-Qaak"]["iso"]          = "tlh"
     #Locale["tlh-Qaak"]["glotto"]
     Locale["tlh-Qaak"]["script"]       = "Piqd"
-
-    #* Kurdish (Northern Kurdish / Kurmanji), Latin alphabet
-    Locale["kmr"]["support"]           = "bing-only"
-    Locale["kmr"]["name"]              = "Kurdish (Northern)"
-    Locale["kmr"]["name2"]             = "Kurmanji"
-    Locale["kmr"]["endonym"]           = "Kurmancî"
-    Locale["kmr"]["family"]            = "Indo-European"
-    Locale["kmr"]["iso"]               = "kmr"
-    Locale["kmr"]["glotto"]            = "nort2641"
-    Locale["kmr"]["script"]            = "Latn"

     #? Bashkir
     Locale["ba"]["support"]            = "yandex-only"


Translate: reduce empty line

Translate: reduce empty line

--- include/Translate.awk
+++ include/Translate.awk
@@ -250,6 +250,8 @@
         }
         hasWordClasses = isarray(wordClasses) && anything(wordClasses)
         hasAltTranslations = isarray(altTranslations[0]) && anything(altTranslations[0])
+        if (!hasWordClasses && !hasAltTranslations)
+            wShowPromptMessage = wShowLanguages = 0
         if (!hasWordClasses) wShowDictionary = 0
         if (hasWordClasses || !hasAltTranslations) wShowAlternatives = 0


HvD: Extended the script a little bit so that duplicate ID lines will not be repeated in util_version.F. If the right property bits are set this should remove all duplication as all the ID lines in a single file SHOULD be the exactly the same.

--- src/util/ids.awk
+++ src/util/ids.awk
@@ -5,6 +5,7 @@
 # a CVS ID    -> produce a write statement with the ID ("Exp" at the end)
 # a SVN ID    -> produce a write statement with the ID (no "Exp" at the end)
 # module name -> produce a write statement for the module name
+# prev_str    -> skip to eliminate duplicates
 # other       -> produce nothing
 #
 # $Id$
@@ -17,6 +18,7 @@
 		printf("      write(6,*)\n");
 		printf("      write(6,*) ' Software version information'\n");
 		printf("      write(6,*) ' ----------------------------'\n");
+		prev_str = underline;
 }

 /^module/			{
@@ -26,12 +28,17 @@
 				 printf("      write(6,*) ' %s'\n",substr(underline,1,len));
 				}

+0 != match($0,prev_str)		{
+				  $0="";
+				}
+
 /\$\I\d: [^\n]*Exp \$/		{
 				 i = index($0, "$Id: ") + 5;
 				 j = index($0, "Exp $") - 1;
 				 n = j - i + 1;
 				 if (n > (72 - 20)) n = 72 - 20;
 				 printf("      write(6,*) ' %s'\n",substr($0,i,n));
+				 prev_str = substr($0,i,n);
 				}

 /\$\I\d: [^\n]* \$/		{
@@ -39,6 +46,7 @@
 				 n = index(substr($0,i), " $") - 1;
 				 if (n > (72 - 20)) n = 72 - 20;
 				 printf("      write(6,*) ' %s'\n",substr($0,i,n));
+				 prev_str = substr($0,i,n);
 				}

 END {