adding in capability of no wobble / just exact duplicates


second

second

#!/usr/local/bin/gawk -E

#
# Initialize a new project
#

# The MIT License (MIT)
#
# Copyright (c) 2016 by User:Green Cardamom (at en.wikipedia.org)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

@include "init.awk"
@include "library.awk"
@include "getopt.awk"

BEGIN {

  while ((C = getopt(ARGC, ARGV, "abicxhp:d:m:s:")) != -1) {
      opts++
      if(C == "p")                 #  -p <project>   Use project name. No default.
        pid = verifypid(Optarg)
      if(C == "d")                 #  -d <data dir>  Data directory. Defaults to default.data in project.cfg
        did = verifyval(Optarg)
      if(C == "m")                 #  -m <data dir>  Meta directory. Defaults to default.meta in project.cfg
        mid = verifyval(Optarg)
      if(C == "c")                 #  -c             Create project files. -p required
        type = "create"
      if(C == "i")                 #  -i             Create index from data directory. -p required
        type = "index"
      if(C == "x")                 #  -x             Delete project files. -p required
        type = "delete"
      if(C == "a")                 #  -a             Verify fixemptyarchive. -p required
        type = "fixemptyarchive"
      if(C == "b")                 #  -b             Verify fixspurone. -p required
        type = "fixspurone"
      if(C == "s") {               #  -s <filename>  Search for a string inside a file in the data directory
        type = "checkstring"
        searchfile = verifyval(Optarg)
      }

      if(C == "h") {
        usage()
        exit
      }
  }

 # No options or an empty -p given
  if( type == "" || (type ~ /delete|create|index/ && pid == "") || pid ~ /error/ ){
    usage()
    exit
  }

 # Load default Data and Meta from project.cfg
  if(did == "")
    did = Config["default"]["data"]
  if(mid == "")
    mid = Config["default"]["meta"]
  if(did == "" || mid == "") {
    print "Unable to determine Data or Meta directories. Create project.cfg with default.data and default.meta"
    exit
  }

  if(type ~ /checkstring/) {
    if(length(searchfile) > 0)
      check_string(searchfile, pid, did, mid)
    exit
  }
  if(type ~ /fixemptyarchive/) {
    check_fixemptyarchive(pid, did, mid)
    exit
  }
  if(type ~ /fixspurone/) {
    check_fixspurone(pid, did, mid)
    exit
  }

  if(type ~ /index/ ) {
    makeindex(pid,did,mid)
    exit
  }

  if(type ~ /delete/) {
    deleteproject(pid,mid,did)
    exit
  }

 # Everything following is type = create

 # Check everything looks ok
  if(substr(did,length(did),1) != "/" || substr(mid,length(mid),1) != "/") {
    print "data = " data
    print "meta = " meta
    print "data and meta should end in a trailing slash. Maybe check project.cfg for default.data/meta"
    exit
  }

 # Make Data and Meta directories
  if(checkexists(did pid))
    print "Data directory already exists: " did pid
  else {
    print "OK Creating " did pid
    sys2var(Exe["mkdir"] " " did pid)
    checkexists(did pid, "project.awk", "exit")
  }
  if(checkexists(mid pid))
    print "Meta directory already exists: " mid pid
  else {
    print "OK Creating " mid pid
    sys2var(Exe["mkdir"] " " mid pid)
    checkexists(mid pid, "project.awk", "exit")
  }

 # Write new project.cfg

  # Remove leading and trailing blank lines
  print stripfile(Home "project.cfg") > Home "project.cfg.orig"
  close(Home "project.cfg.orig")
  print "OK Saving project.cfg to project.cfg.orig"

  c = split(readfile(Home "project.cfg.orig"),a,"\n")

  # Set new default
  i = 0
  re = "^default[.]id"
  while(i++ < c) {
    if(a[i] ~ re) {
      a[i] = "default.id = " pid
      break
    }
  }
  if(i == c)
    print "Unable to set default.id"
  else
    print "OK Setting default.id = " pid

  # Create new .data and .meta
  a[c + 1] = pid ".data = " did pid "/"
  a[c + 2] = pid ".meta = " mid pid "/"

  print "OK Writing new project.cfg"

  if(checkexists(Home "project.cfg"))
    sys2var( Exe["rm"] " " Home "project.cfg")

  i = 0
  while(i++ < c + 2)
    print a[i] >> Home "project.cfg"

  close(Home "project.cfg")

 # Create .auth file
  split(pid,a,".")
  if(checkexists(mid a[1] ".auth")) {
    if(a[2] ~ /[0-9]{1,5}[-][0-9]{1,5}/) {
      c = split(a[2],b,"-")
      if(c == 2 && strip(b[1]) ~ /^[0-9]+$/ && strip(b[2]) ~ /^[0-9]+$/) {
        start = strip(b[1])
        end = strip(b[2])
        if(! checkexists(mid pid "/auth")) {
          # head -n 750 meta/births1870.auth | tail -n 250 > /home/adminuser/wi-awb/meta/births1870.501-750/auth
          command = Exe["head"] " -n " end " " mid a[1] ".auth | " Exe["tail"] " -n " int( int(end) - int( int(start) - 1) ) " > " mid pid "/auth"
          print "OK creating " mid pid "/auth"
          sys2var(command)
        }
        else
          print "Auth file " mid pid "/auth already exists. Not creating new one."
      }
      else
        print "(1) Project ID doesn't take the form Name.####-#### - Unable to create " mid pid "/auth"
    }
    else
      print "(2) Project ID doesn't take the form Name.####-#### - Unable to create " mid pid "/auth"
  }
  else {
    print "Unable to find " mid a[1] ".auth - Unable to create " mid pid "/auth"
  }

}

#
# Return 1 if name is in file
#
function nameisinfile(name, filen,    s, a, re) {

  checkexists(filen, "project.awk nameisinfile()", "exit")

  re = "^" regesc2(strip(name)) "$"
  while ((getline s < filen ) > 0) {
    split(s, a, "|")
    if(strip(a[1]) ~ re) {
      close(filen)
      return 1
    }
  }
  close(filen)
  return 0
}

#
# Make an index based on files in data directory
#
function makeindex(pid,did,mid,    data,meta,a) {

  data = did pid "/"
  meta = mid pid "/"

  if( ! checkexists(data) || ! checkexists(meta) ) {
    print "Unable to find " data " OR " meta
    exit
  }
  if(checkexists(meta "ia9out-index")){
    print "File exists, aborting."
    print "To delete: rm " meta "index"
    exit
  }

  # list directories only
  # https://stackoverflow.com/questions/14352290/listing-only-directories-using-ls-in-bash-an-examination
  c = split( sys2var(Exe["ls"] " -d1 " data "wm*/"), a, "\n")
  while(i++ < c) {
    if( ! exists(a[i] "namewiki.txt") )
      print "Unable to find " a[i] "namewiki.txt" > "/dev/stderr"
    else
      print strip(readfile(a[i] "namewiki.txt")) "|" a[i] >> meta "index"
  }
  close(meta "index")

}

function deleteproject(pid,mid,did,    i,c,re,a) {

 # Delete Data and Meta directories
  if( ! checkexists(did pid))
    print "Data directory doesn't exist: " did pid
  else {
    print "OK Deleting " did pid
    sys2var(Exe["rm"] " -r " did pid)
  }
  if( ! checkexists(mid pid))
    print "Meta directory doesn't exist: " mid pid
  else {
    print "OK Deleting " mid pid
    sys2var(Exe["rm"] " -r " mid pid)
  }

 # Remove .meta and .data lines from project.cfg but leave default.* lines untouched
  if(checkexists(Home "project.cfg.out"))
    sys2var( Exe["rm"] " project.cfg.out")
  if(checkexists(Home "project.cfg.orig"))
    sys2var( Exe["rm"] " project.cfg.orig")
  if(checkexists(Home "project.cfg"))
    command = Exe["mv"] " " Home "project.cfg" " " Home "project.cfg.orig"
  else {
    print "Unable to find " Home "project.cfg"
    return
  }
  print "OK Making backup project.cfg -> project.cfg.orig"
  sys2var(command)
  system("")
  c = split(readfile(Home "project.cfg.orig"),a,"\n")
  re = "^" regesc2(pid) "[.](data|meta)"
  while(i++ < c) {
    if(a[i] ~ re) { # delete if re matches
    }
    else {
      print a[i] >> Home "project.cfg.out"
    }
  }
  close(Home "project.cfg.out")
  if(checkexists(Home "project.cfg.out")) {
    print stripfile(Home "project.cfg.out") > Home "project.cfg"
    close(Home "project.cfg")
    print "OK Removed data & meta lines from project.cfg (default.id untouched)"
  }
  else {
    print "Unable to modify project.cfg - restoring backup"
    sys2var(Exe["mv"] " " Home "project.cfg.orig" " " Home "project.cfg")
  }
}

#
# Check each article.txt in the project data directory to verify which have the fixemptyarchive bug
#
function check_fixemptyarchive(pid, did, mid,    command,files,stampdir,c,i,re) {

  re = "archive[-]?url=[ ]?[|}]"
  files = sys2var(Exe["ls"] " " did pid "/")
  c = split(files, stampdir, "\n")
  while(i++ < c) {
    command = Exe["grep"] " -cE \"" re "\" " did pid "/" stampdir[i] "/article.txt"
    count = sys2var(command)
    if(count > 0) {
      print whatisindexname(did pid "/" stampdir[i], mid pid "/index") " ( cd " did pid "/" stampdir[i] " )"
    }
  }

}
#
# Check each article.txt in the project data directory to verify which have the fixspurone bug
#
function check_fixspurone(pid, did, mid,    command,files,stampdir,c,i,re) {

  re = "[|][ ]?1[ ]?=[ ]{0,2}[|}]"
  files = sys2var(Exe["ls"] " " did pid "/")
  c = split(files, stampdir, "\n")
  while(i++ < c) {
    command = Exe["grep"] " -cE \"" re "\" " did pid "/" stampdir[i] "/article.txt"
    count = sys2var(command)
    if(count > 0) {
      print whatisindexname(did pid "/" stampdir[i], mid pid "/index") " ( cd " did pid "/" stampdir[i] " )"
    }
  }
}

#
# Search for string "re" in <filename> across entire project
#
function check_string(filename, pid, did, mid,    re,c,files,stampdir,i,command,count) {

 # Be careful with escaping as unsure how grep responds
  re = "archiveurl=archive"

  files = sys2var(Exe["ls"] " " did pid "/")

  if(pid == "") {
    print "\nRequires -p <projectid> .. available project IDs:\n"
    print files
    print ""
    exit
  }

  c = split(files, stampdir, "\n")
  while(i++ < c) {
    if(checkexists(did pid "/" stampdir[i] "/" filename)) {
      command = Exe["grep"] " -ciE \"" re "\" " did pid "/" stampdir[i] "/" filename
      count = sys2var(command)
      if(count > 0) {
        print whatisindexname(did pid "/" stampdir[i], mid pid "/index") " ( cd " did pid "/" stampdir[i] " )"
      }
    }
  }
}

#
# Return the name portion given a path/tempid in an index (eg. /home/adminuser/wi-awb/temp/wi-awb-0202173111/)
#
function whatisindexname(name, filepath,      s, a, re) {

  if(! checkexists(filepath) ) {
    print("Error unable to find " filepath ". " name )
    return 0
  }
  re = name
  while ((getline s < filepath ) > 0) {
    split(s, a, "|")
    if(strip(a[2]) ~ re) {
      close(filepath)
      return strip(a[1])
    }
  }
  close(filepath)
  return 0
}

function usage() {

  print ""
  print "Project - manage projects."
  print ""
  print "Usage:"
  print "       -i             Create index from ~/data files. -p required"
  print "       -c             Create project files. -p required"
  print "       -x             Delete project files. -p required"
  print "       -p <project>   Project name."
  print "       -d <data dir>  Data directory. Defaults to default.data in project.cfg"
  print "       -m <meta dir>  Meta directory. Defaults to default.meta in project.cfg"
  print ""
  print "       -s <filename>  Find a string (defined in source) in <filename> in the data directory"
  print "       -a             Find all fixemptyarchive"
  print "       -h             Help"
  print ""
  print "Path names for -d and -m end with trailing slash."
  print ""

}

#!/usr/local/bin/gawk -E

#
# Initialize a new project
#

# The MIT License (MIT)
#
# Copyright (c) 2016 by User:Green Cardamom (at en.wikipedia.org)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

@include "init.awk"
@include "library.awk"
@include "getopt.awk"

BEGIN {

  while ((C = getopt(ARGC, ARGV, "abicxhp:d:m:r:s:")) != -1) {
      opts++
      if(C == "p")                 #  -p <project>   Use project name. No default.
        pid = verifypid(Optarg)
      if(C == "d")                 #  -d <data dir>  Data directory. Defaults to default.data in project.cfg
        did = verifyval(Optarg)
      if(C == "m")                 #  -m <data dir>  Meta directory. Defaults to default.meta in project.cfg
        mid = verifyval(Optarg)
      if(C == "r") {               #  -r <date>      Retrieve list of articles edited by Cyberbot during day 20160214
        type = "retrieve"
        year = verifyval(Optarg)
      }
      if(C == "c")                 #  -c             Create project files. -p required
        type = "create"
      if(C == "i")                 #  -i             Create index from data directory. -p required
        type = "index"
      if(C == "x")                 #  -x             Delete project files. -p required
        type = "delete"
      if(C == "a")                 #  -a             Verify fixemptyarchive. -p required
        type = "fixemptyarchive"
      if(C == "b")                 #  -b             Verify fixspurone. -p required
        type = "fixspurone"
      if(C == "s") {               #  -s <filename>  Search for a string inside a file in the data directory
        type = "checkstring"
        searchfile = verifyval(Optarg)
      }

      if(C == "h") {
        usage()
        exit
      }
  }

 # No options or an empty -p given
  if( type == "" || (type ~ /delete|create|index/ && pid == "") || pid ~ /error/ ){
    usage()
    exit
  }

 # Load default Data and Meta from project.cfg
  if(did == "")
    did = Config["default"]["data"]
  if(mid == "")
    mid = Config["default"]["meta"]
  if(did == "" || mid == "") {
    print "Unable to determine Data or Meta directories. Create project.cfg with default.data and default.meta"
    exit
  }

  if(type ~ /checkstring/) {
    if(length(searchfile) > 0)
      check_string(searchfile, pid, did, mid)
    exit
  }
  if(type ~ /fixemptyarchive/) {
    check_fixemptyarchive(pid, did, mid)
    exit
  }
  if(type ~ /fixspurone/) {
    check_fixspurone(pid, did, mid)
    exit
  }

  if(type ~ /index/ ) {
    makeindex(pid,did,mid)
    exit
  }

  if(type ~ /retrieve/) {
    getcatbirths(year,mid)
    exit
  }

  if(type ~ /delete/) {
    deleteproject(pid,mid,did)
    exit
  }

 # Everything following is type = create

 # Check everything looks ok
  if(substr(did,length(did),1) != "/" || substr(mid,length(mid),1) != "/") {
    print "data = " data
    print "meta = " meta
    print "data and meta should end in a trailing slash. Maybe check project.cfg for default.data/meta"
    exit
  }

 # Make Data and Meta directories
  if(checkexists(did pid))
    print "Data directory already exists: " did pid
  else {
    print "OK Creating " did pid
    sys2var(Exe["mkdir"] " " did pid)
    checkexists(did pid, "project.awk", "exit")
  }
  if(checkexists(mid pid))
    print "Meta directory already exists: " mid pid
  else {
    print "OK Creating " mid pid
    sys2var(Exe["mkdir"] " " mid pid)
    checkexists(mid pid, "project.awk", "exit")
  }
#  if(checkexists(mid pid "/db"))
#    print "tc.in directory already exists: " mid pid "/db"
#  else {
#    print "OK Creating " mid pid "/db"
#    sys2var(Exe["mkdir"] " " mid pid "/db")
#    checkexists(mid pid "/db", "project.awk", "exit")
#  }

 # Write new project.cfg

  # Remove leading and trailing blank lines
  # https://stackoverflow.com/questions/7359527/removing-trailing-starting-newlines-with-sed-awk-tr-and-friends
  # $AWK '{ LINES=LINES $0 "\n"; } /./ { printf "%s", LINES; LINES=""; }' "$SHAREDIR"article.txt | $SED '/./,$\!d' > "$SHAREDIR""o"
  # command = Exe["awk"] " '{ LINES=LINES $0 \"\\n\"; } /./ { printf \"%s\", LINES; LINES=\"\"; }' " Home "project.cfg | " Exe["sed"] " '/./,$!d' > " Home "project.cfg.orig"
  # sys2var(command)
  # system("")
  print stripfile(Home "project.cfg") > Home "project.cfg.orig"
  close(Home "project.cfg.orig")
  print "OK Saving project.cfg to project.cfg.orig"

  c = split(readfile(Home "project.cfg.orig"),a,"\n")

  # Set new default
  i = 0
  re = "^default[.]id"
  while(i++ < c) {
    if(a[i] ~ re) {
      a[i] = "default.id = " pid
      break
    }
  }
  if(i == c)
    print "Unable to set default.id"
  else
    print "OK Setting default.id = " pid

  # Create new .data and .meta
  a[c + 1] = pid ".data = " did pid "/"
  a[c + 2] = pid ".meta = " mid pid "/"

  print "OK Writing new project.cfg"

  if(checkexists(Home "project.cfg"))
    sys2var( Exe["rm"] " " Home "project.cfg")

  i = 0
  while(i++ < c + 2)
    print a[i] >> Home "project.cfg"

  close(Home "project.cfg")

 # Create .auth file
  split(pid,a,".")
  if(checkexists(mid a[1] ".auth")) {
    if(a[2] ~ /[0-9]{1,5}[-][0-9]{1,5}/) {
      c = split(a[2],b,"-")
      if(c == 2 && strip(b[1]) ~ /^[0-9]+$/ && strip(b[2]) ~ /^[0-9]+$/) {
        start = strip(b[1])
        end = strip(b[2])
        if(! checkexists(mid pid "/auth")) {
          # head -n 750 meta/births1870.auth | tail -n 250 > /home/adminuser/wi-awb/meta/births1870.501-750/auth
          command = Exe["head"] " -n " end " " mid a[1] ".auth | " Exe["tail"] " -n " int( int(end) - int( int(start) - 1) ) " > " mid pid "/auth"
          print "OK creating " mid pid "/auth"
          sys2var(command)
        }
        else
          print "Auth file " mid pid "/auth already exists. Not creating new one."
      }
      else
        print "(1) Project ID doesn't take the form Name.####-#### - Unable to create " mid pid "/auth"
    }
    else
      print "(2) Project ID doesn't take the form Name.####-#### - Unable to create " mid pid "/auth"
  }
  else {
    print "Unable to find " mid a[1] ".auth - Unable to create " mid pid "/auth"
  }

}

#
# Return 1 if name is in file
#
function nameisinfile(name, filen,    s, a, re) {

  checkexists(filen, "project.awk nameisinfile()", "exit")

  re = "^" regesc2(strip(name)) "$"
  while ((getline s < filen ) > 0) {
    split(s, a, "|")
    if(strip(a[1]) ~ re) {
      close(filen)
      return 1
    }
  }
  close(filen)
  return 0
}

#
# Make an index based on files in data directory
#
function makeindex(pid,did,mid,    data,meta,a) {

  data = did pid "/"
  meta = mid pid "/"

  if( ! checkexists(data) || ! checkexists(meta) ) {
    print "Unable to find " data " OR " meta
    exit
  }
  if(checkexists(meta "ia9out-index")){
    print "File exists, aborting."
    print "To delete: rm " meta "index"
    exit
  }

  # list directories only
  # https://stackoverflow.com/questions/14352290/listing-only-directories-using-ls-in-bash-an-examination
  c = split( sys2var(Exe["ls"] " -d1 " data "wm*/"), a, "\n")
  while(i++ < c) {
    if( ! exists(a[i] "namewiki.txt") )
      print "Unable to find " a[i] "namewiki.txt" > "/dev/stderr"
    else
      print strip(readfile(a[i] "namewiki.txt")) "|" a[i] >> meta "index"
  }
  close(meta "index")

}

#
# Download full list of names in [[Category:#### births]] to births####.auth in the Meta directory
#
function getcatbirths(year,mid,   a) {

  if(year ~ /^[0-9]{4}$/) {

    # wget -q -O- "https://tools.wmflabs.org/ext-lnk-discover/sc/sc.php?category=1840_births" | \
    # grep '
' | awk '{c=split($0,a,/
/); while(i++ < c){if(a[i] != "") print a[i]} }' | \
    # sort > births1840.auth

    command = Exe["wget"] " -q -O- \"https://tools.wmflabs.org/ext-lnk-discover/sc/sc.php?category=" year "_births\" | " Exe["grep"] " '
' | " Exe["awk"] " '{c=split($0,a,/
/); while(i++ < c){if(a[i] != \"\") print a[i]} }' | " Exe["sort"] " > " mid "births" year ".auth"
    sys2var(command)
    print "OK Created " mid "births" year ".auth"
    command = Exe["wc"] " -l " mid "births" year ".auth"
    split(sys2var(command), a, " ")
    if(int(a[1]) > 999) a[1] = a[1]
    else if(int(a[1]) < 1000 && int(a[1]) > 99) a[1] = "0" a[1]
    else if(int(a[1]) < 100 && int(a[1]) > 9) a[1] = "00" a[1]
    else if(int(a[1]) < 10 && int(a[1]) > 0) a[1] = "000" a[1]
    else a[1] = "error"

    print "sed 's/PNAME/births" year ".0001-" a[1] "/g' 0INSTRUCTIONS > 0INSTRUCTIONS." year

    # print "Suggested Project: births" year ".0001-" a[1]
  }
  else {
    print "Unable to determine year: " year
    exit
  }

}

function deleteproject(pid,mid,did,    i,c,re,a) {

 # Delete Data and Meta directories
  if( ! checkexists(did pid))
    print "Data directory doesn't exist: " did pid
  else {
    print "OK Deleting " did pid
    sys2var(Exe["rm"] " -r " did pid)
  }
  if( ! checkexists(mid pid))
    print "Meta directory doesn't exist: " mid pid
  else {
    print "OK Deleting " mid pid
    sys2var(Exe["rm"] " -r " mid pid)
  }

 # Remove .meta and .data lines from project.cfg but leave default.* lines untouched
  if(checkexists(Home "project.cfg.out"))
    sys2var( Exe["rm"] " project.cfg.out")
  if(checkexists(Home "project.cfg.orig"))
    sys2var( Exe["rm"] " project.cfg.orig")
  if(checkexists(Home "project.cfg"))
    command = Exe["mv"] " " Home "project.cfg" " " Home "project.cfg.orig"
  else {
    print "Unable to find " Home "project.cfg"
    return
  }
  print "OK Making backup project.cfg -> project.cfg.orig"
  sys2var(command)
  system("")
  c = split(readfile(Home "project.cfg.orig"),a,"\n")
  re = "^" regesc2(pid) "[.](data|meta)"
  while(i++ < c) {
    if(a[i] ~ re) { # delete if re matches
    }
    else {
      print a[i] >> Home "project.cfg.out"
    }
  }
  close(Home "project.cfg.out")
  if(checkexists(Home "project.cfg.out")) {
    print stripfile(Home "project.cfg.out") > Home "project.cfg"
    close(Home "project.cfg")
    print "OK Removed data & meta lines from project.cfg (default.id untouched)"
  }
  else {
    print "Unable to modify project.cfg - restoring backup"
    sys2var(Exe["mv"] " " Home "project.cfg.orig" " " Home "project.cfg")
  }
}

#
# Check each article.txt in the project data directory to verify which have the fixemptyarchive bug
#
function check_fixemptyarchive(pid, did, mid,    command,files,stampdir,c,i,re) {

  re = "archive[-]?url=[ ]?[|}]"
  files = sys2var(Exe["ls"] " " did pid "/")
  c = split(files, stampdir, "\n")
  while(i++ < c) {
    command = Exe["grep"] " -cE \"" re "\" " did pid "/" stampdir[i] "/article.txt"
    count = sys2var(command)
    if(count > 0) {
      print whatisindexname(did pid "/" stampdir[i], mid pid "/index") " ( cd " did pid "/" stampdir[i] " )"
    }
  }

}
#
# Check each article.txt in the project data directory to verify which have the fixspurone bug
#
function check_fixspurone(pid, did, mid,    command,files,stampdir,c,i,re) {

  re = "[|][ ]?1[ ]?=[ ]{0,2}[|}]"
  files = sys2var(Exe["ls"] " " did pid "/")
  c = split(files, stampdir, "\n")
  while(i++ < c) {
    command = Exe["grep"] " -cE \"" re "\" " did pid "/" stampdir[i] "/article.txt"
    count = sys2var(command)
    if(count > 0) {
      print whatisindexname(did pid "/" stampdir[i], mid pid "/index") " ( cd " did pid "/" stampdir[i] " )"
    }
  }
}

function check_string(filename, pid, did, mid,    re,c,files,stampdir,i,command,count) {

 # Be careful with escaping as unsure how grep responds
  re = "archiveurl=archive"

  files = sys2var(Exe["ls"] " " did pid "/")

  if(pid == "") {
    print "\nRequires -p <projectid> .. available project IDs:\n"
    print files
    print ""
    exit
  }

  c = split(files, stampdir, "\n")
  while(i++ < c) {
    if(checkexists(did pid "/" stampdir[i] "/" filename)) {
      command = Exe["grep"] " -ciE \"" re "\" " did pid "/" stampdir[i] "/" filename
      count = sys2var(command)
      if(count > 0) {
        print whatisindexname(did pid "/" stampdir[i], mid pid "/index") " ( cd " did pid "/" stampdir[i] " )"
      }
    }
  }
}

#
# Return the name portion given a path/tempid in an index (eg. /home/adminuser/wi-awb/temp/wi-awb-0202173111/)
#
function whatisindexname(name, filepath,      s, a, re) {

  if(! checkexists(filepath) ) {
    print("Error unable to find " filepath ". " name )
    return 0
  }
#  re = "^" regesc(strip(name)) "$"
  re = name
  while ((getline s < filepath ) > 0) {
    split(s, a, "|")
    if(strip(a[2]) ~ re) {
      close(filepath)
      return strip(a[1])
    }
  }
  close(filepath)
  return 0
}

function usage() {

  print ""
  print "Project - manage projects."
  print ""
  print "Usage:"
  print "       -r <date>      Download list of articles edited by Cyberbot during 20160214"
  print "       -i             Create index from ~/data files. -p required"
  print "       -c             Create project files. -p required"
  print "       -x             Delete project files. -p required"
  print "       -p <project>   Project name."
  print "       -d <data dir>  Data directory. Defaults to default.data in project.cfg"
  print "       -m <meta dir>  Meta directory. Defaults to default.meta in project.cfg"
  print ""
  print "       -s <filename>  Find a string (defined in source) in <filename> in the data directory"
  print "       -a             Find all fixemptyarchive"
  print "       -h             Help"
  print ""
  print "Examples: project -x -p cb14feb16"
  print "          project -c -d /mnt/data/ -m /mnt/meta/ -p cb14feb16"
  print "          project -r 20160214"
  print ""
  print "Path names for -d and -m end with trailing slash."
  print ""

}


Translate: enlarge timeout to 1000ms

Translate: enlarge timeout to 1000ms

####################################################################
# Translate.awk                                                    #
####################################################################

function provides(engineName) {
    Translator[tolower(engineName)] = TRUE
}

function engineMethod(methodName,    engine, translator) {
    if (!Translator[Option["engine"]]) {
        # case-insensitive match engine name
        engine = tolower(Option["engine"])
        if (!Translator[engine]) # fuzzy match engine name
            for (translator in Translator)
                if (Translator[translator] && # there IS such a translator
                    translator ~ "^"engine) {
                    engine = translator
                    break
                }
        if (!Translator[engine]) {
            e("[ERROR] Translator not found: " Option["engine"] "\n"    \
              "        Run '-list-engines / -S' to see a list of available engines.")
            exit 1
        }
        Option["engine"] = engine
    }
    return Option["engine"] methodName
}

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mpv" SUPOUT SUPERR) ?
        "mpv --no-config" :
        (!system("mplayer" SUPOUT SUPERR) ?
         "mplayer" :
         (!system("mpg123 --version" SUPOUT SUPERR) ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say ''" SUPOUT SUPERR) ?
        "say" :
        (!system("espeak ''" SUPOUT SUPERR) ?
         "espeak" :
         "")
}

# Detect external terminal pager (less, more, most).
function initPager() {
    Pager = !system("less -V" SUPOUT SUPERR) ?
        "less" :
        (!system("more -V" SUPOUT SUPERR) ?
         "more" :
         (!system("most" SUPOUT SUPERR) ?
          "most" :
          ""))
}

# Initialize `HttpService`.
function initHttpService(    inet) {
    _Init()

    inet = "inet"
    if (Option["ip-version"])
        inet = inet Option["ip-version"]
    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?(([^:]+):([^@]+)@)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpAuthUser = HttpProxySpec[3]
        HttpAuthPass = HttpProxySpec[4]
        HttpAuthCredentials = base64(unquote(HttpAuthUser) ":" HttpAuthPass)
        HttpService = "/" inet "/tcp/0/" HttpProxySpec[5] "/" HttpProxySpec[6]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/" inet "/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }

    # How long to wait in milliseconds for IPv6 before trying IPv4
    PROCINFO[HttpService, "READ_TIMEOUT"] = 1000
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# [OBSOLETE] Is this function still relevant?
# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP GET request and get response from an online translator.
function getResponse(text, sl, tl, hl,
                     ####
                     content, header, isBody, url, group, status, location) {
    url = _RequestUrl(text, sl, tl, hl)

    header = "GET " url " HTTP/1.1\r\n"           \
        "Host: " HttpHost "\r\n"                  \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print header |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if ((status == "301" || status == "302") && location)
        content = curl(location)

    return assert(content, "[ERROR] Null response.")
}

# Send an HTTP POST request and return response from an online translator.
function postResponse(text, sl, tl, hl, type,
                      ####
                      content, contentLength, contentType, group,
                      header, isBody, reqBody, url, status, location) {
    url = _PostRequestUrl(text, sl, tl, hl, type)
    contentType = _PostRequestContentType(text, sl, tl, hl, type)
    reqBody = _PostRequestBody(text, sl, tl, hl, type)
    if (DumpContentengths[reqBody])
        contentLength = DumpContentengths[reqBody]
    else
        contentLength = DumpContentengths[reqBody] = dump(reqBody, group)

    header = "POST " url " HTTP/1.1\r\n"                  \
        "Host: " HttpHost "\r\n"                          \
        "Connection: close\r\n"                           \
        "Content-Length: " contentLength "\r\n"           \
        "Content-Type: " contentType "\r\n"     # must!
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print (header "\r\n" reqBody) |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if (status == "404") {
        e("[ERROR] 404 Not Found")
        exit 1
    }
    if ((status == "301" || status == "302") && location) {
        url = "https" substr(url, 5) # switch to HTTPS; don't use location!
        content = curlPost(url, reqBody)
    }

    return content
}

# Print a string (to output file or terminal pager).
function p(string) {
    if (Option["view"]) {
        print string | Option["pager"]
        close(Option["pager"])
    } else
        print string > Option["output"]
}

# Play using a Text-to-Speech engine.
function play(text, tl,    url, status) {
    url = _TTSUrl(text, tl)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    status = system(Option["player"] " " parameterize(url) SUPOUT SUPERR)

    if (status)
        w("Voice output isn't available for " getName(tl))

    return status
}

# Download audio from a Text-to-Speech engine.
function download_audio(text, tl,    url, output) {
    url = _TTSUrl(text, tl)

    if (Option["download-audio-as"])
        output = Option["download-audio-as"]
    else
        output = text " [" Option["engine"] "] (" Option["narrator"] ").ts"

    if (url ~ /^\//)
        system("mv -- " parameterize(url) " " parameterize(output))
    else
        curl(url, output)
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist, returnIl) {
    return _Translate(text, sl, tl, hl,
                      isVerbose, toSpeech, returnPlaylist, returnIl)
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl,    temp) {
    temp = _WebTranslateUrl(uri, sl, tl, hl)
    if (temp)
        system(Option["browser"] " " parameterize(temp) SUPOUT SUPERR)
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, il, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (isRTL(Option["hl"])) {
        # Check if home language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["hl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (isRTL(Option["sl"])) {
        # Check if source language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["sl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            # translate URL only from command-line parameters (inline)
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            # translate URL only from command-line parameters (inline)
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            if (!Option["no-translate"])
                p(getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"] || Option["download-audio"], playlist, il))
            else
                il[0] = Option["sl"] == "auto" ? "en" : Option["sl"]

            if (Option["play"] == 1) {
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
            } else if (Option["play"] == 2) {
                if (Option["player"])
                    play(text, il[0])
                else if (SpeechSynthesizer)
                    print text | SpeechSynthesizer
            }

            if (Option["download-audio"] == 1) {
                # Download the translation unless used with -sp or -no-trans
                if (Option["play"] != 2 && !Option["no-translate"])
                    download_audio(playlist[length(playlist) - 1]["text"], \
                                   playlist[length(playlist) - 1]["tl"])
                else
                    download_audio(text, il[0])
            }
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Translate the source text (from all source languages).
function translates(text, inline,
                    ####
                    i) {
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["sls"]) {
        # Non-interactive verbose mode: separator between sources
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        Option["sl"] = Option["sls"][i]
        translate(text, inline)
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    if (Option["input"] == STDIN || fileExists(Option["input"])) {
        i = 0
        while (getline line < Option["input"])
            if (line) {
                # Non-interactive verbose mode: separator between sources
                if (!Option["interactive"])
                    if (Option["verbose"] && i++ > 0)
                        p(prettify("source-seperator",
                                   replicate(Option["chr-source-seperator"],
                                             Option["width"])))

                if (Option["interactive"])
                    repl(line)
                else
                    translates(line)
            } else {
                # Non-interactive brief mode: preserve line breaks
                if (!Option["interactive"])
                    if (!Option["verbose"])
                        p(line)
            }
    } else
        e("[ERROR] File not found: " Option["input"])
}

####################################################################
# Translate.awk                                                    #
####################################################################

function provides(engineName) {
    Translator[tolower(engineName)] = TRUE
}

function engineMethod(methodName,    engine, translator) {
    if (!Translator[Option["engine"]]) {
        # case-insensitive match engine name
        engine = tolower(Option["engine"])
        if (!Translator[engine]) # fuzzy match engine name
            for (translator in Translator)
                if (Translator[translator] && # there IS such a translator
                    translator ~ "^"engine) {
                    engine = translator
                    break
                }
        if (!Translator[engine]) {
            e("[ERROR] Translator not found: " Option["engine"] "\n"    \
              "        Run '-list-engines / -S' to see a list of available engines.")
            exit 1
        }
        Option["engine"] = engine
    }
    return Option["engine"] methodName
}

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mpv" SUPOUT SUPERR) ?
        "mpv --no-config" :
        (!system("mplayer" SUPOUT SUPERR) ?
         "mplayer" :
         (!system("mpg123 --version" SUPOUT SUPERR) ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say ''" SUPOUT SUPERR) ?
        "say" :
        (!system("espeak ''" SUPOUT SUPERR) ?
         "espeak" :
         "")
}

# Detect external terminal pager (less, more, most).
function initPager() {
    Pager = !system("less -V" SUPOUT SUPERR) ?
        "less" :
        (!system("more -V" SUPOUT SUPERR) ?
         "more" :
         (!system("most" SUPOUT SUPERR) ?
          "most" :
          ""))
}

# Initialize `HttpService`.
function initHttpService(    inet) {
    _Init()

    inet = "inet"
    if (Option["ip-version"])
        inet = inet Option["ip-version"]
    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?(([^:]+):([^@]+)@)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpAuthUser = HttpProxySpec[3]
        HttpAuthPass = HttpProxySpec[4]
        HttpAuthCredentials = base64(unquote(HttpAuthUser) ":" HttpAuthPass)
        HttpService = "/" inet "/tcp/0/" HttpProxySpec[5] "/" HttpProxySpec[6]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/" inet "/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }

    # How long to wait in milliseconds for IPv6 before trying IPv4
    PROCINFO[HttpService, "READ_TIMEOUT"] = 300
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# [OBSOLETE] Is this function still relevant?
# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP GET request and get response from an online translator.
function getResponse(text, sl, tl, hl,
                     ####
                     content, header, isBody, url, group, status, location) {
    url = _RequestUrl(text, sl, tl, hl)

    header = "GET " url " HTTP/1.1\r\n"           \
        "Host: " HttpHost "\r\n"                  \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print header |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if ((status == "301" || status == "302") && location)
        content = curl(location)

    return assert(content, "[ERROR] Null response.")
}

# Send an HTTP POST request and return response from an online translator.
function postResponse(text, sl, tl, hl, type,
                      ####
                      content, contentLength, contentType, group,
                      header, isBody, reqBody, url, status, location) {
    url = _PostRequestUrl(text, sl, tl, hl, type)
    contentType = _PostRequestContentType(text, sl, tl, hl, type)
    reqBody = _PostRequestBody(text, sl, tl, hl, type)
    if (DumpContentengths[reqBody])
        contentLength = DumpContentengths[reqBody]
    else
        contentLength = DumpContentengths[reqBody] = dump(reqBody, group)

    header = "POST " url " HTTP/1.1\r\n"                  \
        "Host: " HttpHost "\r\n"                          \
        "Connection: close\r\n"                           \
        "Content-Length: " contentLength "\r\n"           \
        "Content-Type: " contentType "\r\n"     # must!
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print (header "\r\n" reqBody) |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if (status == "404") {
        e("[ERROR] 404 Not Found")
        exit 1
    }
    if ((status == "301" || status == "302") && location) {
        url = "https" substr(url, 5) # switch to HTTPS; don't use location!
        content = curlPost(url, reqBody)
    }

    return content
}

# Print a string (to output file or terminal pager).
function p(string) {
    if (Option["view"]) {
        print string | Option["pager"]
        close(Option["pager"])
    } else
        print string > Option["output"]
}

# Play using a Text-to-Speech engine.
function play(text, tl,    url, status) {
    url = _TTSUrl(text, tl)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    status = system(Option["player"] " " parameterize(url) SUPOUT SUPERR)

    if (status)
        w("Voice output isn't available for " getName(tl))

    return status
}

# Download audio from a Text-to-Speech engine.
function download_audio(text, tl,    url, output) {
    url = _TTSUrl(text, tl)

    if (Option["download-audio-as"])
        output = Option["download-audio-as"]
    else
        output = text " [" Option["engine"] "] (" Option["narrator"] ").ts"

    if (url ~ /^\//)
        system("mv -- " parameterize(url) " " parameterize(output))
    else
        curl(url, output)
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist, returnIl) {
    return _Translate(text, sl, tl, hl,
                      isVerbose, toSpeech, returnPlaylist, returnIl)
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl,    temp) {
    temp = _WebTranslateUrl(uri, sl, tl, hl)
    if (temp)
        system(Option["browser"] " " parameterize(temp) SUPOUT SUPERR)
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, il, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (isRTL(Option["hl"])) {
        # Check if home language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["hl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (isRTL(Option["sl"])) {
        # Check if source language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["sl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            # translate URL only from command-line parameters (inline)
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            # translate URL only from command-line parameters (inline)
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            if (!Option["no-translate"])
                p(getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"] || Option["download-audio"], playlist, il))
            else
                il[0] = Option["sl"] == "auto" ? "en" : Option["sl"]

            if (Option["play"] == 1) {
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
            } else if (Option["play"] == 2) {
                if (Option["player"])
                    play(text, il[0])
                else if (SpeechSynthesizer)
                    print text | SpeechSynthesizer
            }

            if (Option["download-audio"] == 1) {
                # Download the translation unless used with -sp or -no-trans
                if (Option["play"] != 2 && !Option["no-translate"])
                    download_audio(playlist[length(playlist) - 1]["text"], \
                                   playlist[length(playlist) - 1]["tl"])
                else
                    download_audio(text, il[0])
            }
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Translate the source text (from all source languages).
function translates(text, inline,
                    ####
                    i) {
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["sls"]) {
        # Non-interactive verbose mode: separator between sources
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        Option["sl"] = Option["sls"][i]
        translate(text, inline)
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    if (Option["input"] == STDIN || fileExists(Option["input"])) {
        i = 0
        while (getline line < Option["input"])
            if (line) {
                # Non-interactive verbose mode: separator between sources
                if (!Option["interactive"])
                    if (Option["verbose"] && i++ > 0)
                        p(prettify("source-seperator",
                                   replicate(Option["chr-source-seperator"],
                                             Option["width"])))

                if (Option["interactive"])
                    repl(line)
                else
                    translates(line)
            } else {
                # Non-interactive brief mode: preserve line breaks
                if (!Option["interactive"])
                    if (!Option["verbose"])
                        p(line)
            }
    } else
        e("[ERROR] File not found: " Option["input"])
}


use GLOBAL TT to label qc graph

use GLOBAL TT to label qc graph

#! /usr/bin/gawk -f

# label_dot_edge.awk
# Augment ontological terms with their label
# example usage:
# ${DIPPER}/scripts/label_dot_edge.awk
#	${DIPPER}/translationtable/GLOBAL_TERMS.yaml  in.dot > out.dot
#
# Arguments: two files
# Loads the first file as a map from  to curie to label :
 # (expects: "label": "curie",)
# Reads the second Graphviz digraph dot format file:
# looks for <term> as edge-text
# and appends "! <label> (Iff <term> is found in map)
# note: This "<term> ! <label>" structure is a convention used by Ontologists.

BEGIN{FS="\""; OFS="\t"}
# load labels
FNR == NR && /^"/{label[$4]=$2}

# output lines that do not get labels
FNR != NR && !/.* -> .*label=.*/ {print}

# output lines w/labels if they are available

FNR != NR && /.* -> .*label=.*/ {
	line="";
	split($0, row, / /)

	split(row[4], term, /"/)
	split(term[2], lclid, ":")

	if (term[2] in label && lclid[2] != label[term[2]]){
		term[2] = term[2] " ! " label[term[2]];
		row[4] = term[1];
		for(i=2;i<=length(term);i++)
			row[4] = row[4] "\"" term[i]
	}
	line = row[1]
	for(i=2;i<=length(row);i++)
		line = line " " row[i]
	print  line
}

#! /usr/bin/gawk -f

# label_dot_edge.awk
# Attempts to augment opaque ontological terms with their current label
# example usage:
# label_dot_edge.awk predicate_curie_term_label.tab  in.dot > out.dot
#
# Arguments: two files
# Loads the first file as a map from terms to labels:
# (expects: [namespace:]<tab><term><tab><label>[<tab><whatever>])
# the first set was generated via cypher queries against our scigraph
# (neo4j) instance loaded with the ontologies we use.
# Reads the second Graphviz digraph dot format file:
# looks for <term> as edge-text
# and appends "! <label> (Iff <term> is found in map)
# note: This "<term> ! <label>" structure is a convention used by Ontologists.

BEGIN{FS="\t"; OFS="\t"}
# load labels
FNR == NR && $3 != "null"{label[$1 $2]=$3}

# output lines that do not get labels
FNR != NR && !/.* -> .*label=.*/ {print}

# output lines w/labels if they are available
FNR != NR && /.* -> .*label=.*/ {
	line="";
	split($0, row, / /)
	split(row[4], term, /"/)
	if (term[2] in label){
		term[2] = term[2] " ! " label[term[2]];
		row[4] = term[1];
		for(i=2;i<=length(term);i++)
			row[4] = row[4] "\"" term[i]
	}
	line = row[1]
	for(i=2;i<=length(row);i++)
		line = line " " row[i]
	print  line
}


Help: show proxy and user-agent in getVersion()

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead, platform) {
    initAudioPlayer()
    initPager()
    platform = detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", platform)                      \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", Option["sl"])           \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCE") "]"                            \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view")) RS                             \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-sl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-source ") ansi("underline", "CODE")) RS \
        ins(2, "Specify the source language.") RS                       \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")) RS \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists("man/" Command ".1"))
        system("man man/" Command ".1" SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, i, j, name, r, rows, saveSortedIn, t, tt) {
    rows = int(length(Locale) / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        r = "┌" replicate("─", 22) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│ "
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t = cols[j][i] == "bo" ||
                        cols[j][i] == "dz" ||
                        cols[j][i] == "he" ||
                        cols[j][i] == "hi" ||
                        cols[j][i] == "hu" ||
                        cols[j][i] == "la" ||
                        cols[j][i] == "ml" ||
                        cols[j][i] == "mn" ||
                        cols[j][i] == "ne" ||
                        cols[j][i] == "ny" ||
                        cols[j][i] == "pa" ||
                        cols[j][i] == "pl" ||
                        cols[j][i] == "ro" ||
                        cols[j][i] == "sr" ||
                        cols[j][i] == "te" ||
                        cols[j][i] == "tg" ||
                        cols[j][i] == "tr" ||
                        cols[j][i] == "ur" ||
                        cols[j][i] == "wo" ||
                        cols[j][i] == "yi" ||
                        cols[j][i] == "yo" ||
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW" &&
                         length(getEndonym(cols[j][i])) < 6) ? "\t\t " :
                        cols[j][i] == "id" ? "" :
                        "\t "
                    tt = length(cols[j][i]) == 3 ? " │" :
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW") ? "  │" : ""
                    r = r getDisplay(cols[j][i]) t "- " ansi("bold", cols[j][i]) tt " "
                } else
                    r = r "\t\t       │"
            }
            r = r RS
        }
        r = r "└" replicate("─", 22) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    } else {
        r = "┌" replicate("─", 22) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│ "
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t = cols[j][i] == "he" ||
                        cols[j][i] == "kk" ||
                        cols[j][i] == "ko" ||
                        cols[j][i] == "ky" ||
                        cols[j][i] == "ne" ||
                        cols[j][i] == "pl" ||
                        cols[j][i] == "ps" ||
                        cols[j][i] == "sd" ||
                        cols[j][i] == "sk" ||
                        cols[j][i] == "sm" ||
                        cols[j][i] == "so" ||
                        cols[j][i] == "te" ||
                        cols[j][i] == "ug" ||
                        cols[j][i] == "yo" ||
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW" &&
                         length(getName(cols[j][i])) < 6) ? "\t\t " :
                        cols[j][i] == "fy" ||
                        cols[j][i] == "gd" ||
                        cols[j][i] == "ht" ? " " :
                        "\t "
                    tt = length(cols[j][i]) == 3 ? " │" :
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW") ? "  │" : ""
                    name = getName(cols[j][i])
                    if (cols[j][i] == "zh-CN" ||
                        cols[j][i] == "zh-TW")
                        name = substr(name, 1, 12) "."
                    r = r name t "- " ansi("bold", cols[j][i]) tt " "
                } else
                    r = r "\t\t       │"
            }
            r = r RS
        }
        r = r "└" replicate("─", 22) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead, platform) {
    initAudioPlayer()
    initPager()
    platform = detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", platform)                      \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", Option["sl"])           \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCE") "]"                            \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view")) RS                             \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-sl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-source ") ansi("underline", "CODE")) RS \
        ins(2, "Specify the source language.") RS                       \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")) RS \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists("man/" Command ".1"))
        system("man man/" Command ".1" SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, i, j, name, r, rows, saveSortedIn, t, tt) {
    rows = int(length(Locale) / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        r = "┌" replicate("─", 22) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│ "
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t = cols[j][i] == "bo" ||
                        cols[j][i] == "dz" ||
                        cols[j][i] == "he" ||
                        cols[j][i] == "hi" ||
                        cols[j][i] == "hu" ||
                        cols[j][i] == "la" ||
                        cols[j][i] == "ml" ||
                        cols[j][i] == "mn" ||
                        cols[j][i] == "ne" ||
                        cols[j][i] == "ny" ||
                        cols[j][i] == "pa" ||
                        cols[j][i] == "pl" ||
                        cols[j][i] == "ro" ||
                        cols[j][i] == "sr" ||
                        cols[j][i] == "te" ||
                        cols[j][i] == "tg" ||
                        cols[j][i] == "tr" ||
                        cols[j][i] == "ur" ||
                        cols[j][i] == "wo" ||
                        cols[j][i] == "yi" ||
                        cols[j][i] == "yo" ||
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW" &&
                         length(getEndonym(cols[j][i])) < 6) ? "\t\t " :
                        cols[j][i] == "id" ? "" :
                        "\t "
                    tt = length(cols[j][i]) == 3 ? " │" :
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW") ? "  │" : ""
                    r = r getDisplay(cols[j][i]) t "- " ansi("bold", cols[j][i]) tt " "
                } else
                    r = r "\t\t       │"
            }
            r = r RS
        }
        r = r "└" replicate("─", 22) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    } else {
        r = "┌" replicate("─", 22) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│ "
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t = cols[j][i] == "he" ||
                        cols[j][i] == "kk" ||
                        cols[j][i] == "ko" ||
                        cols[j][i] == "ky" ||
                        cols[j][i] == "ne" ||
                        cols[j][i] == "pl" ||
                        cols[j][i] == "ps" ||
                        cols[j][i] == "sd" ||
                        cols[j][i] == "sk" ||
                        cols[j][i] == "sm" ||
                        cols[j][i] == "so" ||
                        cols[j][i] == "te" ||
                        cols[j][i] == "ug" ||
                        cols[j][i] == "yo" ||
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW" &&
                         length(getName(cols[j][i])) < 6) ? "\t\t " :
                        cols[j][i] == "fy" ||
                        cols[j][i] == "gd" ||
                        cols[j][i] == "ht" ? " " :
                        "\t "
                    tt = length(cols[j][i]) == 3 ? " │" :
                        (cols[j][i] != "zh-CN" && cols[j][i] != "zh-TW") ? "  │" : ""
                    name = getName(cols[j][i])
                    if (cols[j][i] == "zh-CN" ||
                        cols[j][i] == "zh-TW")
                        name = substr(name, 1, 12) "."
                    r = r name t "- " ansi("bold", cols[j][i]) tt " "
                } else
                    r = r "\t\t       │"
            }
            r = r RS
        }
        r = r "└" replicate("─", 22) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}