Fixed problem of BSD awk returning -0 from 10 % 5.

darcs-hash:20060228153855-65a35-bda8dcdb283315c33c9b00b1290057437185821d.gz

#! /usr/bin/env awk
#
# This script recreates C files containing header-specific boilerplate stuff
# using the given list of headers (usually obtained from the master structure).
#
# It can also create a parser table.
#
# --------------------------------------------------------------------
#
# This file is part of the Sofia-SIP package
#
# Copyright (C) 2005 Nokia Corporation.
#
# Contact: Pekka Pessi <pekka.pessi@nokia.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# --------------------------------------------------------------------
#
# Contributor(s): Pekka.Pessi@nokia.com.
#
# Created: Fri Apr  6 12:59:59 2001 ppessi
#

BEGIN {
  "date '+%a %b %e %H:%M:%S %Y'" | getline date;

  ascii =			       \
     "                               " \
    " !\"#$%&'()*+,-./0123456789:;<=>?" \
    "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_" \
    "`abcdefghijklmnopqrstuvwxyz{|}~";
  lower_case = "abcdefghijklmnopqrstuvwxyz";

  N=0;
  # Initialize these as arrays
  split("", symbols);
  split("", names);
  split("", comments);
  split("", hashes);
  split("", NAMES);
  split("", Comments);
  split("", COMMENTS);

  template="";
  template1="";
  template2="";
  template3=="";
}

function name_hash (name)
{
  hash = 0;

  len = split(name, chars, "");

  for (i = 1; i <= len; i++) {
    c = tolower(chars[i]);
    hash = (38501 * (hash + index(ascii, c))) % 65536;
  }

  if (0) {
    # Test that hash algorithm above agrees with the C version
    pipe = ("../msg/msg_name_hash " name);
    pipe | getline;
    close(pipe);
    if (hash != $0) {
      print name ": " hash " != " $0 > "/dev/stderr";
    }
  }

  return hash "";
}

#
# Replace magic patterns in template p with header-specific values
#
function protos (name, comment, hash)
{
  NAME=toupper(name);
  sub(/.*[\/][*][*][<][ 	]*/, "", comment);
  sub(/[ 	]*[*][\/].*/, "", comment);
  sub(/[ 	]+/, " ", comment);

  short = match(comment, /[(][a-z][)]/);
  if (short) {
    short = substr(comment, short + 1, 1);
    sub(/ *[(][a-z][)]/, "", comment);
    shorts[index(lower_case, short)] = name;
  }

  do_hash = hash == 0;

  if (do_hash) {
    split(comment, parts, " ");
    name2 = tolower(parts[1]);
    gsub(/-/, "_", name2);
    if (name2 != name) {
      print name " mismatch with " comment " (" real ")" > "/dev/stderr";
    }

    hash = name_hash(parts[1]);

    hashed[name] = hash;

    if (comment !~ /header/) {
      comment = comment " header";
    }
  }

  Comment = comment;
  if (!do_hash) {
    comment = tolower(comment);
  }
  COMMENT = toupper(comment);

  # Store the various forms into an array for the footer processing
  N++;
  hashes[N] = hash;
  names[N] = name;
  NAMES[N] = NAME;
  comments[N] = comment;
  Comments[N] = comment;
  COMMENTS[N] = COMMENT;

  symbols[name] = comment;

  if (PR) {
    replace(template, hash, name, NAME, comment, Comment, COMMENT);
    replace(template1, hash, name, NAME, comment, Comment, COMMENT);
    replace(template2, hash, name, NAME, comment, Comment, COMMENT);
    replace(template3, hash, name, NAME, comment, Comment, COMMENT);
  }
}

function replace (p, hash, name, NAME, comment, Comment, COMMENT)
{
  if (p) {
    gsub(/#hash#/, hash, p);
    gsub(/#xxxxxx#/, name, p);
    gsub(/#XXXXXX#/, NAME, p);
    gsub(/#xxxxxxx_xxxxxxx#/, comment, p);
    gsub(/#Xxxxxxx_Xxxxxxx#/, Comment, p);
    gsub(/#XXXXXXX_XXXXXXX#/, COMMENT, p);

    print p > PR;
  }
}

#
# Repeat each line in the footer containing the magic replacement
# pattern with a instance of all headers
#
function process_footer (text)
{
  if (!match(tolower(text), /#(xxxxxx(x_xxxxxxx)?|hash)#/)) {
    n = length(text);
    while (substr(text, n) == "\n") {
      n = n - 1;
      text = substr(text, 1, n);
    }
    if (n > 0)
      print text > PR;
    return;
  }

  n = split(text, lines, RS);

  for (i = 1; i <= n; i++) {
    l = lines[i];
    if (match(tolower(l), /#(xxxxxx(x_xxxxxxx)?|hash)#/)) {
      for (j = 1; j <= N; j++) {
	l = lines[i];
	gsub(/#hash#/, hashes[j], l);
	gsub(/#xxxxxxx_xxxxxxx#/, comments[j], l);
	gsub(/#Xxxxxxx_Xxxxxxx#/, Comments[j], l);
	gsub(/#XXXXXXX_XXXXXXX#/, COMMENTS[j], l);
	gsub(/#xxxxxx#/, names[j], l);
	gsub(/#XXXXXX#/, NAMES[j], l);
	print l > PR;
      }
    } else {
      print l > PR;
    }
  }
}

#
# Read flags used with headers
#
function read_header_flags (flagfile,    line, tokens, name, value)
{
  while ((getline line < flagfile) > 0) {
    sub(/^[ \t]+/, "", line);
    sub(/[ \t]+$/, "", line);
    if (line ~ /^#/ || line ~ /^$/)
      continue;

    split(line, tokens,  /[ \t]*=[ \t]*/);
    name = tolower(tokens[1]);
    gsub(/-/, "_", name);
    gsub(/,/, " ", name);
    # print "FLAG: " name " = " tokens[2]

    if (header_flags[name]) {
      print flagfile ": already defined " tokens[1];
    }
    else if (!symbols[name]) {
      print flagfile ": unknown header \"" tokens[1] "\"";
    }
    else {
      header_flags[name] = tokens[2];
    }
  }
  close(flagfile);
}

/^ *\/\* === Headers start here \*\// {

  in_header_list=1;

  if (!auto) {
    auto = FILENAME;
    sub(/.*\//, "", auto);
    auto = "This file is automatically generated from <" auto "> by msg_parser.awk.";

    if (PR) {
      if (TEMPLATE == "") { TEMPLATE = PR ".in"; }
      RS0=RS; RS="\f\n";
      getline theader < TEMPLATE;
      getline header < TEMPLATE;
      getline template < TEMPLATE;
      getline footer < TEMPLATE;

      if (TEMPLATE1) {
	getline dummy < TEMPLATE1;
	getline dummy < TEMPLATE1;
	getline template1 < TEMPLATE1;
	getline dummy < TEMPLATE1;
      }

      if (TEMPLATE2) {
	getline dummy < TEMPLATE2;
	getline dummy < TEMPLATE2;
	getline template2 < TEMPLATE2;
	getline dummy < TEMPLATE2;
      }

      if (TEMPLATE3) {
	getline dummy < TEMPLATE3;
	getline dummy < TEMPLATE3;
	getline template3 < TEMPLATE3;
	getline dummy < TEMPLATE3;
      }

      sub(/.*[\/]/, "", TEMPLATE);
      gsub(/#AUTO#/, auto, header);
      gsub(/#DATE#/, "@date Generated: " date, header);
      if (PACKAGE_NAME) gsub(/#PACKAGE_NAME#/, PACKAGE_NAME, header);
      if (PACKAGE_VERSION) gsub(/#PACKAGE_VERSION#/, PACKAGE_VERSION, header);
      print header > PR;

      RS=RS0;
    }

    if (!NO_FIRST) {
      protos("request", "/**< Request line */", -1);
      protos("status", "/**< Status line */", -2);
    }
  }
}

in_header_list && /^ *\/\* === Headers end here \*\// { in_header_list=0;}

in_header_list && PT && /^ *\/\* === Hash headers end here \*\// { in_header_list=0;}

in_header_list && /^  (sip|rtsp|http|msg|mp)_[a-z_0-9]+_t/ {
  n=$0
  sub(/;.*$/, "", n);
  sub(/^ *(sip|rtsp|http|msg|mp)_[a-z0-9_]*_t[ 	]*/, "", n);
  sub(/^[*](sip|rtsp|http|msg|mp)_/, "", n);

  if ($0 !~ /[\/][*][*][<]/) {
    getline;
  }
  if ($0 !~ /[\/][*][*][<]/) {
    printf "msg_protos.awk: header %s is malformed\n", n;
    exit 1;
  }

  if (!NO_MIDDLE)
    protos(n, $0, 0);

  headers[total++] = n;
}

END {
  if (!prefix) { prefix = module; }
  if (!tprefix) { tprefix = prefix; }

  if (!NO_LAST) {
    protos("unknown", "/**< Unknown headers */", -3);
    protos("error", "/**< Erroneous headers */", -4);
    protos("separator", "/**< Separator line between headers and payload */", -5);
    protos("payload", "/**< Message payload */", -6);
    if (multipart)
      protos("multipart", "/**< Multipart payload */", -7);
  }

  if (PR) {
    process_footer(footer);
  }
  else if (PT) {
    if (FLAGFILE)
      read_header_flags(FLAGFILE);

    if (TEMPLATE == "") { TEMPLATE = PT ".in"; }
    RS0=RS; RS="\n";
    getline theader < TEMPLATE;
    getline header < TEMPLATE;
    getline template < TEMPLATE;
    getline footer < TEMPLATE;
    RS=RS0;

    sub(/.*[\/]/, "", TEMPLATE);
    gsub(/#AUTO#/, auto, header);
    gsub(/#DATE#/, "@date Generated: " date, header);
    print header > PT;

    if (MC_SHORT_SIZE) {
      printf("static msg_href_t const " \
	     "%s_short_forms[MC_SHORT_SIZE] = \n{\n",
	     module) > PT;

      for (i = 1; i <= MC_SHORT_SIZE; i = i + 1) {
	c = (i == MC_SHORT_SIZE) ? "" : ",";
	if (i in shorts) {
	  n = shorts[i];
        flags = header_flags[n]; if (flags) flags = ",\n      " flags;

	  printf("  { /* %s */ %s_%s_class, offsetof(%s_t, %s_%s)%s }%s\n",
		 substr(lower_case, i, 1),
		 tprefix, n, module, prefix, n, flags, c)	\
	    > PT;
	}
	else {
	  printf("  { NULL }%s\n", c) \
	    > PT;
	}
      }
      printf("};\n\n") > PT;
    }

    # printf("extern msg_hclass_t msg_multipart_class[];\n\n") > PT;

    printf("msg_mclass_t const %s_mclass[1] = \n{{\n", module) > PT;
    printf("# if defined (%s_HCLASS)\n", toupper(module)) > PT;
    printf("  %s_HCLASS,\n", toupper(module)) > PT;
    printf("#else\n") > PT;
    printf("  {{ 0 }},\n") > PT;
    printf("#endif\n") > PT;
    printf("  %s_VERSION_CURRENT,\n", toupper(module)) > PT;
    printf("  %s_PROTOCOL_TAG,\n", toupper(module)) > PT;
    printf("#if defined (%s_PARSER_FLAGS)\n", toupper(module)) > PT;
    printf("  %s_PARSER_FLAGS,\n", toupper(module)) > PT;
    printf("#else\n") > PT;
    printf("  0,\n") > PT;
    printf("#endif\n") > PT;
    printf("  sizeof(%s_t),\n", module) > PT;
    printf("  %s_extract_body,\n", module) > PT;

    len = split("request status separator payload unknown error", unnamed, " ");

    for (i = 1; i <= len; i++) {
      printf("  {{ %s_%s_class, offsetof(%s_t, %s_%s) }},\n",
	     tprefix, unnamed[i], module, prefix, unnamed[i]) > PT;
    }
    if (multipart) {
      printf("  {{ %s_class, offsetof(%s_t, %s_multipart) }},\n",
	     multipart, module, prefix) > PT;
    } else {
      printf("  {{ NULL, 0 }},\n") > PT;
    }
    if (MC_SHORT_SIZE) {
      printf("  %s_short_forms, \n", module) > PT;
    }
    else {
      printf("  NULL, \n") > PT;
    }
    printf("  %d, %d, \n", MC_HASH_SIZE, total) > PT;
    printf("  {\n") > PT;

    for (i = 0; i < total; i++) {
      n = headers[i];
      h = hashed[n];

      if (h < 0)
	continue;

      j = h % MC_HASH_SIZE; if (j == -0) j = 0;

      for (; j in header_hash;) {
	if (++j == MC_HASH_SIZE) {
	  j = 0;
	}
      }

      header_hash[j] = n;
    }

    for (i = 0; i < MC_HASH_SIZE; i++) {
      c = (i + 1 == MC_HASH_SIZE) ? "" : ",";
      if (i in header_hash) {
	n = header_hash[i];
        flags = header_flags[n]; if (flags) flags = ",\n      " flags;

	printf("    { %s_%s_class, offsetof(%s_t, %s_%s)%s }%s\n",
	       tprefix, n, module, prefix, n, flags, c) > PT;
      }
      else {
	printf("    { NULL, 0 }%s\n", c) > PT;
      }
    }
    printf("  }\n}};\n\n") > PT;
  }
}

#! /usr/bin/env awk
#
# This script recreates C files containing header-specific boilerplate stuff
# using the given list of headers (usually obtained from the master structure).
#
# It can also create a parser table.
#
# --------------------------------------------------------------------
#
# This file is part of the Sofia-SIP package
#
# Copyright (C) 2005 Nokia Corporation.
#
# Contact: Pekka Pessi <pekka.pessi@nokia.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# --------------------------------------------------------------------
#
# Contributor(s): Pekka.Pessi@nokia.com.
#
# Created: Fri Apr  6 12:59:59 2001 ppessi
#

BEGIN {
  "date '+%a %b %e %H:%M:%S %Y'" | getline date;

  ascii =			       \
     "                               " \
    " !\"#$%&'()*+,-./0123456789:;<=>?" \
    "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_" \
    "`abcdefghijklmnopqrstuvwxyz{|}~";
  lower_case = "abcdefghijklmnopqrstuvwxyz";

  N=0;
  # Initialize these as arrays
  split("", symbols);
  split("", names);
  split("", comments);
  split("", hashes);
  split("", NAMES);
  split("", Comments);
  split("", COMMENTS);

  template="";
  template1="";
  template2="";
  template3=="";
}

function name_hash (name)
{
  hash = 0;

  len = split(name, chars, "");

  for (i = 1; i <= len; i++) {
    c = tolower(chars[i]);
    hash = (38501 * (hash + index(ascii, c))) % 65536;
  }

  if (0) {
    # Test that hash algorithm above agrees with the C version
    pipe = ("../msg/msg_name_hash " name);
    pipe | getline;
    close(pipe);
    if (hash != $0) {
      print name ": " hash " != " $0 > "/dev/stderr";
    }
  }

  return hash "";
}

#
# Replace magic patterns in template p with header-specific values
#
function protos (name, comment, hash)
{
  NAME=toupper(name);
  sub(/.*[\/][*][*][<][ 	]*/, "", comment);
  sub(/[ 	]*[*][\/].*/, "", comment);
  sub(/[ 	]+/, " ", comment);

  short = match(comment, /[(][a-z][)]/);
  if (short) {
    short = substr(comment, short + 1, 1);
    sub(/ *[(][a-z][)]/, "", comment);
    shorts[index(lower_case, short)] = name;
  }

  do_hash = hash == 0;

  if (do_hash) {
    split(comment, parts, " ");
    name2 = tolower(parts[1]);
    gsub(/-/, "_", name2);
    if (name2 != name) {
      print name " mismatch with " comment " (" real ")" > "/dev/stderr";
    }

    hash = name_hash(parts[1]);

    hashed[name] = hash;

    if (comment !~ /header/) {
      comment = comment " header";
    }
  }

  Comment = comment;
  if (!do_hash) {
    comment = tolower(comment);
  }
  COMMENT = toupper(comment);

  # Store the various forms into an array for the footer processing
  N++;
  hashes[N] = hash;
  names[N] = name;
  NAMES[N] = NAME;
  comments[N] = comment;
  Comments[N] = comment;
  COMMENTS[N] = COMMENT;

  symbols[name] = comment;

  if (PR) {
    replace(template, hash, name, NAME, comment, Comment, COMMENT);
    replace(template1, hash, name, NAME, comment, Comment, COMMENT);
    replace(template2, hash, name, NAME, comment, Comment, COMMENT);
    replace(template3, hash, name, NAME, comment, Comment, COMMENT);
  }
}

function replace (p, hash, name, NAME, comment, Comment, COMMENT)
{
  if (p) {
    gsub(/#hash#/, hash, p);
    gsub(/#xxxxxx#/, name, p);
    gsub(/#XXXXXX#/, NAME, p);
    gsub(/#xxxxxxx_xxxxxxx#/, comment, p);
    gsub(/#Xxxxxxx_Xxxxxxx#/, Comment, p);
    gsub(/#XXXXXXX_XXXXXXX#/, COMMENT, p);

    print p > PR;
  }
}

#
# Repeat each line in the footer containing the magic replacement
# pattern with a instance of all headers
#
function process_footer (text)
{
  if (!match(tolower(text), /#(xxxxxx(x_xxxxxxx)?|hash)#/)) {
    n = length(text);
    while (substr(text, n) == "\n") {
      n = n - 1;
      text = substr(text, 1, n);
    }
    if (n > 0)
      print text > PR;
    return;
  }

  n = split(text, lines, RS);

  for (i = 1; i <= n; i++) {
    l = lines[i];
    if (match(tolower(l), /#(xxxxxx(x_xxxxxxx)?|hash)#/)) {
      for (j = 1; j <= N; j++) {
	l = lines[i];
	gsub(/#hash#/, hashes[j], l);
	gsub(/#xxxxxxx_xxxxxxx#/, comments[j], l);
	gsub(/#Xxxxxxx_Xxxxxxx#/, Comments[j], l);
	gsub(/#XXXXXXX_XXXXXXX#/, COMMENTS[j], l);
	gsub(/#xxxxxx#/, names[j], l);
	gsub(/#XXXXXX#/, NAMES[j], l);
	print l > PR;
      }
    } else {
      print l > PR;
    }
  }
}

#
# Read flags used with headers
#
function read_header_flags (flagfile,    line, tokens, name, value)
{
  while ((getline line < flagfile) > 0) {
    sub(/^[ \t]+/, "", line);
    sub(/[ \t]+$/, "", line);
    if (line ~ /^#/ || line ~ /^$/)
      continue;

    split(line, tokens,  /[ \t]*=[ \t]*/);
    name = tolower(tokens[1]);
    gsub(/-/, "_", name);
    gsub(/,/, " ", name);
    # print "FLAG: " name " = " tokens[2]

    if (header_flags[name]) {
      print flagfile ": already defined " tokens[1];
    }
    else if (!symbols[name]) {
      print flagfile ": unknown header \"" tokens[1] "\"";
    }
    else {
      header_flags[name] = tokens[2];
    }
  }
  close(flagfile);
}

/^ *\/\* === Headers start here \*\// {

  in_header_list=1;

  if (!auto) {
    auto = FILENAME;
    sub(/.*\//, "", auto);
    auto = "This file is automatically generated from <" auto "> by msg_parser.awk.";

    if (PR) {
      if (TEMPLATE == "") { TEMPLATE = PR ".in"; }
      RS0=RS; RS="\f\n";
      getline theader < TEMPLATE;
      getline header < TEMPLATE;
      getline template < TEMPLATE;
      getline footer < TEMPLATE;

      if (TEMPLATE1) {
	getline dummy < TEMPLATE1;
	getline dummy < TEMPLATE1;
	getline template1 < TEMPLATE1;
	getline dummy < TEMPLATE1;
      }

      if (TEMPLATE2) {
	getline dummy < TEMPLATE2;
	getline dummy < TEMPLATE2;
	getline template2 < TEMPLATE2;
	getline dummy < TEMPLATE2;
      }

      if (TEMPLATE3) {
	getline dummy < TEMPLATE3;
	getline dummy < TEMPLATE3;
	getline template3 < TEMPLATE3;
	getline dummy < TEMPLATE3;
      }

      sub(/.*[\/]/, "", TEMPLATE);
      gsub(/#AUTO#/, auto, header);
      gsub(/#DATE#/, "@date Generated: " date, header);
      if (PACKAGE_NAME) gsub(/#PACKAGE_NAME#/, PACKAGE_NAME, header);
      if (PACKAGE_VERSION) gsub(/#PACKAGE_VERSION#/, PACKAGE_VERSION, header);
      print header > PR;

      RS=RS0;
    }

    if (!NO_FIRST) {
      protos("request", "/**< Request line */", -1);
      protos("status", "/**< Status line */", -2);
    }
  }
}

in_header_list && /^ *\/\* === Headers end here \*\// { in_header_list=0;}

in_header_list && PT && /^ *\/\* === Hash headers end here \*\// { in_header_list=0;}

in_header_list && /^  (sip|rtsp|http|msg|mp)_[a-z_0-9]+_t/ {
  n=$0
  sub(/;.*$/, "", n);
  sub(/^ *(sip|rtsp|http|msg|mp)_[a-z0-9_]*_t[ 	]*/, "", n);
  sub(/^[*](sip|rtsp|http|msg|mp)_/, "", n);

  if ($0 !~ /[\/][*][*][<]/) {
    getline;
  }
  if ($0 !~ /[\/][*][*][<]/) {
    printf "msg_protos.awk: header %s is malformed\n", n;
    exit 1;
  }

  if (!NO_MIDDLE)
    protos(n, $0, 0);

  headers[total++] = n;
}

END {
  if (!prefix) { prefix = module; }
  if (!tprefix) { tprefix = prefix; }

  if (!NO_LAST) {
    protos("unknown", "/**< Unknown headers */", -3);
    protos("error", "/**< Erroneous headers */", -4);
    protos("separator", "/**< Separator line between headers and payload */", -5);
    protos("payload", "/**< Message payload */", -6);
    if (multipart)
      protos("multipart", "/**< Multipart payload */", -7);
  }

  if (PR) {
    process_footer(footer);
  }
  else if (PT) {
    if (FLAGFILE)
      read_header_flags(FLAGFILE);

    if (TEMPLATE == "") { TEMPLATE = PT ".in"; }
    RS0=RS; RS="\n";
    getline theader < TEMPLATE;
    getline header < TEMPLATE;
    getline template < TEMPLATE;
    getline footer < TEMPLATE;
    RS=RS0;

    sub(/.*[\/]/, "", TEMPLATE);
    gsub(/#AUTO#/, auto, header);
    gsub(/#DATE#/, "@date Generated: " date, header);
    print header > PT;

    if (MC_SHORT_SIZE) {
      printf("static msg_href_t const " \
	     "%s_short_forms[MC_SHORT_SIZE] = \n{\n",
	     module) > PT;

      for (i = 1; i <= MC_SHORT_SIZE; i = i + 1) {
	c = (i == MC_SHORT_SIZE) ? "" : ",";
	if (i in shorts) {
	  n = shorts[i];
        flags = header_flags[n]; if (flags) flags = ",\n      " flags;

	  printf("  { /* %s */ %s_%s_class, offsetof(%s_t, %s_%s)%s }%s\n",
		 substr(lower_case, i, 1),
		 tprefix, n, module, prefix, n, flags, c)	\
	    > PT;
	}
	else {
	  printf("  { NULL }%s\n", c) \
	    > PT;
	}
      }
      printf("};\n\n") > PT;
    }

    # printf("extern msg_hclass_t msg_multipart_class[];\n\n") > PT;

    printf("msg_mclass_t const %s_mclass[1] = \n{{\n", module) > PT;
    printf("# if defined (%s_HCLASS)\n", toupper(module)) > PT;
    printf("  %s_HCLASS,\n", toupper(module)) > PT;
    printf("#else\n") > PT;
    printf("  {{ 0 }},\n") > PT;
    printf("#endif\n") > PT;
    printf("  %s_VERSION_CURRENT,\n", toupper(module)) > PT;
    printf("  %s_PROTOCOL_TAG,\n", toupper(module)) > PT;
    printf("#if defined (%s_PARSER_FLAGS)\n", toupper(module)) > PT;
    printf("  %s_PARSER_FLAGS,\n", toupper(module)) > PT;
    printf("#else\n") > PT;
    printf("  0,\n") > PT;
    printf("#endif\n") > PT;
    printf("  sizeof(%s_t),\n", module) > PT;
    printf("  %s_extract_body,\n", module) > PT;

    len = split("request status separator payload unknown error", unnamed, " ");

    for (i = 1; i <= len; i++) {
      printf("  {{ %s_%s_class, offsetof(%s_t, %s_%s) }},\n",
	     tprefix, unnamed[i], module, prefix, unnamed[i]) > PT;
    }
    if (multipart) {
      printf("  {{ %s_class, offsetof(%s_t, %s_multipart) }},\n",
	     multipart, module, prefix) > PT;
    } else {
      printf("  {{ NULL, 0 }},\n") > PT;
    }
    if (MC_SHORT_SIZE) {
      printf("  %s_short_forms, \n", module) > PT;
    }
    else {
      printf("  NULL, \n") > PT;
    }
    printf("  %d, %d, \n", MC_HASH_SIZE, total) > PT;
    printf("  {\n") > PT;

    for (i = 0; i < total; i++) {
      n = headers[i];
      h = hashed[n];

      if (h < 0)
	continue;

      for (j = h % MC_HASH_SIZE; j in header_hash;) {
	if (++j == MC_HASH_SIZE) {
	  j = 0;
	}
      }

      header_hash[j] = n;
    }

    for (i = 0; i < MC_HASH_SIZE; i++) {
      c = (i + 1 == MC_HASH_SIZE) ? "" : ",";
      if (i in header_hash) {
	n = header_hash[i];
        flags = header_flags[n]; if (flags) flags = ",\n      " flags;

	printf("    { %s_%s_class, offsetof(%s_t, %s_%s)%s }%s\n",
	       tprefix, n, module, prefix, n, flags, c) > PT;
      }
      else {
	printf("    { NULL, 0 }%s\n", c) > PT;
      }
    }
    printf("  }\n}};\n\n") > PT;
  }
}


Fix bug

Fix bug

BEGIN {
  year=strftime("%Y");
  starttime=0;
  ended_jobs=0;
  total_used=0;
  goodputZ=0;
  goodputNZ=0;
  badputSignal=0;
  badputOther=0;
  goodjobsZ=0;
  goodjobsNZ=0;
  badjobsSignal=0;
  badjobsOther=0;
}

# year is a 4 digit string or number
# date is month/day
# time is hour:minute:seconds
function convert_date(year,date,time) {
  split(date,datearr,"/");
  split(time,timearr,":");
  timestr=sprintf("%04d %02d %02d %02d %02d %02d",year,datearr[1],datearr[2],timearr[1],timearr[2],timearr[3]);
  return mktime(timestr)
}

function percent(a,b) {
  if (b==0) {
    return 0;
  } else {
    return a*100/b;
  }
}

########################################################################
/ Communicating with shadow /{
  # 7/17 10:42:20 (pid:26855) Communicating with shadow <131.225.204.208:34839>
  #
  for (i=1; i<NF; i++) {
    if ($i=="Communicating") {
      changenr=i;
      break;
    }
  }
  shadow=$(changenr+3);
  shadow=substr(shadow,2,length(shadow)-2)
  #print  $1,$2,"Shadow:",shadow
}
/ Starting .* job with ID: /{
  # 7/17 10:42:21 (pid:26855) Starting a VANILLA universe job with ID: 40.0
  #
  for (i=1; i<NF; i++) {
    if ($i=="ID:") {
      changenr=i;
      break;
    }
  }
  jid=$(changenr+1);
  #print  $1,$2,"Jid:",jid
}
/ Create_Process succeeded, /{
  # 7/17 10:42:21 (pid:26855) Create_Process succeeded, pid=26858
  #
  for (i=1; i<NF; i++) {
    if ($i=="Create_Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  split(pidstr,pidarr,"=");
  pid=pidarr[2];
  #print  $1,$2,"PID:",pid

  starttime=convert_date(year,$1,$2);
  print $1,$2,"Starting job",jid,"from",shadow
}
/ Process exited, /{
  # 7/17 10:50:31 (pid:26855) Process exited, pid=26858, signal=9
  # 7/17 11:23:04 (pid:23112) Process exited, pid=23114, status=0
  #
  for (i=1; i<NF; i++) {
    if ($i=="Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  reasonstr=$(changenr+3);

  split(reasonstr,reasonarr,"=");

  ended=convert_date(year,$1,$2);
  if (ended<starttime) { # around new year
    ended=convert_date(year+1,$1,$2);
  }

  if (starttime>0) {
    joblength=ended-starttime;
  } else {
    # protect from incomplete data
    joblength=0;
  }

  ended_jobs++;
  total_used+=joblength;

  print $1,$2,"Terminated job",jid,"from",shadow,reasonarr[1],reasonarr[2],"duration",joblength
  if (reasonarr[1]=="status") {
    if (reasonarr[2]=="0") {
      goodputZ+=joblength;
      goodjobsZ++;
    } else {
      goodputNZ+=joblength;
      goodjobsNZ++;
    }
  } else if (reasonarr[1]=="signal") {
    badputSignal+=joblength;
    badjobsSignal++
  } else {
    badputOther+=joblength;
    babjobsOther++;
  }

  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}

########################################################################
END {
  print "=================";
  print "Total jobs",ended_jobs,"utilization",total_used;
  print "Total goodZ jobs",goodjobsZ " (" percent(goodjobsZ,ended_jobs) "%)","utilization",goodputZ " (" percent(goodputZ,total_used) "%)";
  print "Total goodNZ jobs",goodjobsNZ " (" percent(goodjobsNZ,ended_jobs) "%)","utilization",goodputNZ " (" percent(goodputNZ,total_used) "%)";
  print "Total badSignal jobs",badjobsSignal " (" percent(badjobsSignal,ended_jobs) "%)","utilization",badputSignal " (" percent(badputSignal,total_used) "%)";
  print "Total badOther jobs",badjobsOther " (" percent(badjobsOther,ended_jobs) "%)","utilization",badputOther " (" percent(badputOther,total_used) "%)";
}

BEGIN {
  year=strftime("%Y");
  ended_jobs=0;
  total_used=0;
  goodputZ=0;
  goodputNZ=0;
  badputSignal=0;
  badputOther=0;
  goodjobsZ=0;
  goodjobsNZ=0;
  badjobsSignal=0;
  badjobsOther=0;
}

# year is a 4 digit string or number
# date is month/day
# time is hour:minute:seconds
function convert_date(year,date,time) {
  split(date,datearr,"/");
  split(time,timearr,":");
  timestr=sprintf("%04d %02d %02d %02d %02d %02d",year,datearr[1],datearr[2],timearr[1],timearr[2],timearr[3]);
  return mktime(timestr)
}

function percent(a,b) {
  if (b==0) {
    return 0;
  } else {
    return a*100/b;
  }
}

########################################################################
/ Communicating with shadow /{
  # 7/17 10:42:20 (pid:26855) Communicating with shadow <131.225.204.208:34839>
  #
  for (i=1; i<NF; i++) {
    if ($i=="Communicating") {
      changenr=i;
      break;
    }
  }
  shadow=$(changenr+3);
  shadow=substr(shadow,3,length(shadow)-4)
  #print  $1,$2,"Shadow:",shadow
}
/ Starting .* job with ID /{
  # 7/17 10:42:21 (pid:26855) Starting a VANILLA universe job with ID: 40.0
  #
  for (i=1; i<NF; i++) {
    if ($i=="ID:") {
      changenr=i;
      break;
    }
  }
  jid=$(changenr+1);
  #print  $1,$2,"Jid:",jid
}
/ Create_Process succeeded, /{
  # 7/17 10:42:21 (pid:26855) Create_Process succeeded, pid=26858
  #
  for (i=1; i<NF; i++) {
    if ($i=="Create_Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  split(pidstr,pidarr,"=");
  pid=pidarr[2];
  #print  $1,$2,"PID:",pid

  starttime=convert_date(year,$1,$2);
  print $1,$2,"Starting job",jid,"from",shadow
}
/ Process exited, /{
  # 7/17 10:50:31 (pid:26855) Process exited, pid=26858, signal=9
  # 7/17 11:23:04 (pid:23112) Process exited, pid=23114, status=0
  #
  for (i=1; i<NF; i++) {
    if ($i=="Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  reasonstr=$(changenr+3);

  split(reasonstr,reasonarr,"=");

  ended=convert_date(year,$1,$2);
  if (ended<starttime) { # around new year
    ended=convert_date(year+1,$1,$2);
  }
  joblength=ended-starttime;

  ended_jobs++;
  total_used+=joblength;

  print $1,$2,"Terminated job",jid,"from",shadow,reasonarr[1],reasonarr[2],"duration",joblength
  if (reasonarr[1]=="status") {
    if (reasonarr[2]=="0") {
      goodputZ+=joblength;
      goodjobsZ++;
    } else {
      goodputNZ+=joblength;
      goodjobsNZ++;
    }
  } else if (reasonarr[1]=="signal") {
    badputSignal+=joblength;
    badjobsSignal++
  } else {
    badputOther+=joblength;
    babjobsOther++;
  }

  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
}

########################################################################
END {
  print "=================";
  print "Total jobs",ended_jobs,"utilization",total_used;
  print "Total goodZ jobs",goodjobsZ " (" percent(goodjobsZ,ended_jobs) "%)","utilization",goodputZ " (" percent(goodputZ,total_used) "%)";
  print "Total goodNZ jobs",goodjobsNZ " (" percent(goodjobsNZ,ended_jobs) "%)","utilization",goodputNZ " (" percent(goodputNZ,total_used) "%)";
  print "Total badSignal jobs",badjobsSignal " (" percent(badjobsSignal,ended_jobs) "%)","utilization",badputSignal " (" percent(badputSignal,total_used) "%)";
  print "Total badOther jobs",badjobsOther " (" percent(badjobsOther,ended_jobs) "%)","utilization",badputOther " (" percent(badputOther,total_used) "%)";
}


new: vendor id for Apple Inc.

new: vendor id for Apple Inc.

#
# Warning: this script used GNU Awk features.
#

#@include "../utils.awk"

BEGIN {
  # boolean value
  parsing_fwts_bios_info=0;
}

parse_fwts_bios_info();

END {
  print_shell_variables();
}

#
# Print shell variables that may be required for scripting.
# So far:
#   - part of the BIOS Information table to build the directory where results should be stored.
#
function print_shell_variables() {
  print "# automatically parsed from ", FILENAME, ". Please check.";
  print_shell_variables_bios_info();
}

#
# Print BIOS information as POSIX shell variables.
# For each bios information entry, print a variable assignment of the value quoted.
# Example:
# VENDOR_BIOS='intel'
#
function print_shell_variables_bios_info() {
  print to_shell_variable("FWTR_BIOS_INFO_BIOS_VENDOR",       sanitize_vendor(bios_vendor));
  print to_shell_variable("FWTR_BIOS_INFO_BIOS_VERSION",      sanitize_text(bios_version));
  print to_shell_variable("FWTR_BIOS_INFO_BIOS_RELEASE_DATE", sanitize_text(bios_release_date));
  print to_shell_variable("FWTR_BIOS_INFO_BOARD_NAME",        sanitize_text(board_name));
  print to_shell_variable("FWTR_BIOS_INFO_BOARD_VERSION",     sanitize_text(board_version));
  print to_shell_variable("FWTR_BIOS_INFO_PRODUCT_NAME",      sanitize_text(product_name));
  print to_shell_variable("FWTR_BIOS_INFO_PRODUCT_VERSION",   sanitize_text(product_version));
  print to_shell_variable("FWTR_BIOS_INFO_SYSTEM_VENDOR",     sanitize_vendor(system_vendor));
}

#
# Parse the BIOS Information Table from FWTS test results.
# Assumed beginning with `bios_info:` and ending with `System Vendor:` rows.
#
# Example:
# > bios_info: Gather BIOS DMI information.
# > --------------------------------------------------------------------------------
# > Test 1 of 1: Gather BIOS DMI information
# > BIOS Vendor       : Intel Corp.
# > BIOS Version      : TYBYT10H.86A.0046.2015.1014.1057
# > BIOS Release Date : 10/14/2015
# > Board Name        : DE3815TYKH
# > Board Serial #    :
# > Board Version     : H26998-401
# > Board Asset Tag   :
# > Chassis Serial #  :
# > Chassis Type      : 3
# > Chassis Vendor    :
# > Chassis Version   :
# > Chassic Asset Tag :
# > Product Name      :
# > Product Serial #  :
# > Product UUID      :
# > Product Version   :
# > System Vendor     :
#
function parse_fwts_bios_info() {
  if ($0 ~ /bios_info:/) {
    parsing_fwts_bios_info=1; # true, we are parsing the BIOS Infos
    FS=":"; # `:` as field-separator. Cf. the bios info table example
  }

  if (parsing_fwts_bios_info) {
    parse_fwts_bios_info_entry("BIOS Vendor",       "bios_vendor");
    parse_fwts_bios_info_entry("BIOS Version",      "bios_version");
    parse_fwts_bios_info_entry("BIOS Release Date", "bios_release_date");
    parse_fwts_bios_info_entry("Board Name",        "board_name");
    # ignored: Board Serial #
    parse_fwts_bios_info_entry("Board Version",     "board_version");
    # ignored: Board Asset Tag
    # ignored: Chassis Serial #
    # ignored: Chassis Type
    # ignored: Chassis Vendor
    # ignored: Chassis Version
    # ignored: Chassis Asset Tag
    parse_fwts_bios_info_entry("Product Name",      "product_name");
    # ignored: Product Serial #
    # ignored: Product UUID
    parse_fwts_bios_info_entry("Product Version",   "product_version");
    # last line of the bios info table reached
    parse_fwts_bios_info_entry("System Vendor",     "system_vendor") && parsing_fwts_bios_info=0;

  }
}

#
# Return the vendor ID string: a lower-case unique and single company name.
# Instead of trying to remove `Inc.`, `Corp.`, etc. directly match company substrings.
#
function to_vendor_id(vendor)
{
  vendor = tolower(vendor);

  switch (vendor) {
    # Apple Inc.
    case /apple/:
      return "apple";

    # Phoenix Technologies Ltd.
    case /phoenix/:
      return "phoenix"

    # Intel Corp.
    case /intel/:
      return "intel";

    case /american megatrends/:
      return "ami";

    case /award/:
      return "award";

    case /gigabyte/:
      return "gigabyte";

    case /dell/:
      return "dell";

    case /samsung/:
      return "samsung";

    # failsafe attempt:
    # vendor names with a single word sould be usable as they are.
    # matches things like: ibm, asus, intel, lenovo, dell...
    # Names compound with `Inc.`
    case /^[[:alnum:]]{2,}$/:
      return vendor;

    default:
      print "ERROR: unknown vendor name `" vendor "`" > "/dev/stderr";
      print "       please add your new vendor name in `to_vendor_id()` function"  > "/dev/stderr";
      return "error";
  }
}

#
# Sanitize a vendor text string.
#
function sanitize_vendor(vendor) {
  if (vendor ~ /^$/)
    return "";

  vendor=sanitize_text(vendor);
  vendor=to_vendor_id(vendor)
  gsub(/[^/\-_ \\[:alnum:]]/, "", vendor);
  return vendor;
}

#
# Retrieve an entry value from the BIOS DMI table into `variable` when it matches `pattern`.
#
function parse_fwts_bios_info_entry(pattern, variable) {
  if ($0 ~ pattern) {
    sub(/<.*>/, "", $2);
    set(variable, trim($2));
    return 1;
  }
  return 0;
}

#
# Setter functions required by `set()`.
#
function set_bios_vendor(value)       { bios_vendor=value; }
function set_bios_version(value)      { bios_version=value; }
function set_bios_release_date(value) { bios_release_date=value; }
function set_board_name(value)        { board_name=value; }
function set_board_version(value)     { board_version=value; }
function set_product_name(value)      { product_name=value; }
function set_product_version(value)   { product_version=value; }
function set_system_vendor(value)     { system_vendor=value; }

#
# Warning: this script used GNU Awk features.
#

#@include "../utils.awk"

BEGIN {
  # boolean value
  parsing_fwts_bios_info=0;
}

parse_fwts_bios_info();

END {
  print_shell_variables();
}

#
# Print shell variables that may be required for scripting.
# So far:
#   - part of the BIOS Information table to build the directory where results should be stored.
#
function print_shell_variables() {
  print "# automatically parsed from ", FILENAME, ". Please check.";
  print_shell_variables_bios_info();
}

#
# Print BIOS information as POSIX shell variables.
# For each bios information entry, print a variable assignment of the value quoted.
# Example:
# VENDOR_BIOS='intel'
#
function print_shell_variables_bios_info() {
  print to_shell_variable("FWTR_BIOS_INFO_BIOS_VENDOR",       sanitize_vendor(bios_vendor));
  print to_shell_variable("FWTR_BIOS_INFO_BIOS_VERSION",      sanitize_text(bios_version));
  print to_shell_variable("FWTR_BIOS_INFO_BIOS_RELEASE_DATE", sanitize_text(bios_release_date));
  print to_shell_variable("FWTR_BIOS_INFO_BOARD_NAME",        sanitize_text(board_name));
  print to_shell_variable("FWTR_BIOS_INFO_BOARD_VERSION",     sanitize_text(board_version));
  print to_shell_variable("FWTR_BIOS_INFO_PRODUCT_NAME",      sanitize_text(product_name));
  print to_shell_variable("FWTR_BIOS_INFO_PRODUCT_VERSION",   sanitize_text(product_version));
  print to_shell_variable("FWTR_BIOS_INFO_SYSTEM_VENDOR",     sanitize_vendor(system_vendor));
}

#
# Parse the BIOS Information Table from FWTS test results.
# Assumed beginning with `bios_info:` and ending with `System Vendor:` rows.
#
# Example:
# > bios_info: Gather BIOS DMI information.
# > --------------------------------------------------------------------------------
# > Test 1 of 1: Gather BIOS DMI information
# > BIOS Vendor       : Intel Corp.
# > BIOS Version      : TYBYT10H.86A.0046.2015.1014.1057
# > BIOS Release Date : 10/14/2015
# > Board Name        : DE3815TYKH
# > Board Serial #    :
# > Board Version     : H26998-401
# > Board Asset Tag   :
# > Chassis Serial #  :
# > Chassis Type      : 3
# > Chassis Vendor    :
# > Chassis Version   :
# > Chassic Asset Tag :
# > Product Name      :
# > Product Serial #  :
# > Product UUID      :
# > Product Version   :
# > System Vendor     :
#
function parse_fwts_bios_info() {
  if ($0 ~ /bios_info:/) {
    parsing_fwts_bios_info=1; # true, we are parsing the BIOS Infos
    FS=":"; # `:` as field-separator. Cf. the bios info table example
  }

  if (parsing_fwts_bios_info) {
    parse_fwts_bios_info_entry("BIOS Vendor",       "bios_vendor");
    parse_fwts_bios_info_entry("BIOS Version",      "bios_version");
    parse_fwts_bios_info_entry("BIOS Release Date", "bios_release_date");
    parse_fwts_bios_info_entry("Board Name",        "board_name");
    # ignored: Board Serial #
    parse_fwts_bios_info_entry("Board Version",     "board_version");
    # ignored: Board Asset Tag
    # ignored: Chassis Serial #
    # ignored: Chassis Type
    # ignored: Chassis Vendor
    # ignored: Chassis Version
    # ignored: Chassis Asset Tag
    parse_fwts_bios_info_entry("Product Name",      "product_name");
    # ignored: Product Serial #
    # ignored: Product UUID
    parse_fwts_bios_info_entry("Product Version",   "product_version");
    # last line of the bios info table reached
    parse_fwts_bios_info_entry("System Vendor",     "system_vendor") && parsing_fwts_bios_info=0;

  }
}

#
# Return the vendor ID string: a lower-case unique and single company name.
# Instead of trying to remove `Inc.`, `Corp.`, etc. directly match company substrings.
#
function to_vendor_id(vendor)
{
  vendor = tolower(vendor);

  switch (vendor) {
    # Phoenix Technologies Ltd.
    case /phoenix/:
      return "phoenix"

    # Intel Corp.
    case /intel/:
      return "intel";

    case /american megatrends/:
      return "ami";

    case /award/:
      return "award";

    case /gigabyte/:
      return "gigabyte";

    case /dell/:
      return "dell";

    case /samsung/:
      return "samsung";

    # failsafe attempt:
    # vendor names with a single word sould be usable as they are.
    # matches things like: ibm, asus, intel, lenovo, dell...
    # Names compound with `Inc.`
    case /^[[:alnum:]]{2,}$/:
      return vendor;

    default:
      print "ERROR: unknown vendor name `" vendor "`" > "/dev/stderr";
      print "       please add your new vendor name in `to_vendor_id()` function"  > "/dev/stderr";
      return "error";
  }
}

#
# Sanitize a vendor text string.
#
function sanitize_vendor(vendor) {
  if (vendor ~ /^$/)
    return "";

  vendor=sanitize_text(vendor);
  vendor=to_vendor_id(vendor)
  gsub(/[^/\-_ \\[:alnum:]]/, "", vendor);
  return vendor;
}

#
# Retrieve an entry value from the BIOS DMI table into `variable` when it matches `pattern`.
#
function parse_fwts_bios_info_entry(pattern, variable) {
  if ($0 ~ pattern) {
    sub(/<.*>/, "", $2);
    set(variable, trim($2));
    return 1;
  }
  return 0;
}

#
# Setter functions required by `set()`.
#
function set_bios_vendor(value)       { bios_vendor=value; }
function set_bios_version(value)      { bios_version=value; }
function set_bios_release_date(value) { bios_release_date=value; }
function set_board_name(value)        { board_name=value; }
function set_board_version(value)     { board_version=value; }
function set_product_name(value)      { product_name=value; }
function set_product_version(value)   { product_version=value; }
function set_system_vendor(value)     { system_vendor=value; }


add awk program

add awk program


Print last two lines in Avg

Print last two lines in Avg

# Generate MHz_TV_Shows_minutes.csv by processing  SHORT_SPREADSHEET to figure
# average episode length
#

BEGIN {
    FS = "\t"
}

/^Title/ {
    printf ("%s\t%s\t%s\t%s\tAvg Len\t%s\t%s\t%s\t%s\t%s\n",
            $1,$2,$3,$4,$5,$6,$7,$8,$9)
}

/=HYPERLINK/ {
    dur=$4
    sub (/h/,"",dur)
    sub (/m/,"",dur)
    split (dur,fld," ")
    eplen = (fld[1]*60+fld[2])/$3
    printf ("%s\t%s\t%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n",
            $1,$2,$3,$4,eplen,$5,$6,$7,$8,$9)
}

/^Non-blank values/ || /^Total seasons & episodes/ { print }


Do not check nisas

Do not check nisas

#!/usr/bin/awk -f
BEGIN {
    FS = "\t";

    URN           = "urn:swn:";

    LABEL         = "<http://www.w3.org/2000/01/rdf-schema#label>";
    TYPE          = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";

    CONCEPT       = "<http://www.w3.org/2004/02/skos/core#Concept>";
    LEXICALSENSE  = "<http://lemon-model.net/lemon#LexicalSense>";
    WORD          = "<http://lemon-model.net/lemon#Word>";
    FORM          = "<http://lemon-model.net/lemon#Form>";

    WRITTENREP    = "<http://lemon-model.net/lemon#writtenRep>";
    CANONICALFORM = "<http://lemon-model.net/lemon#canonicalForm>";
    BROADER       = "<http://lemon-model.net/lemon#broader>";
    NARROWER      = "<http://lemon-model.net/lemon#narrower>";
    SENSE         = "<http://lemon-model.net/lemon#sense>";
    ISSENSEOF     = "<http://lemon-model.net/lemon#isSenseOf>";
    REFERENCE     = "<http://lemon-model.net/lemon#reference>";
    ISREFERENCEOF = "<http://lemon-model.net/lemon#isReferenceOf>";
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0) next;

    print "<" URN "c" $1 ">", TYPE,  CONCEPT,            ".";
    print "<" URN "c" $1 ">", LABEL, "\"synset" $1 "\"", ".";

    for (sense_id in senses) {
        sense = senses[sense_id];

        sub(":.*$", "", sense);

        if (!(sense in sindex)) {
            sindex[sense] = scount++;

            word = sense;
            gsub("_", " ",  word);
            sub("#.*$", "", word);

            if (!(word in windex)) {
                windex[word] = wcount++;

                print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
            }

            print "<" URN "s" sindex[sense] ">", TYPE,          LEXICALSENSE,                  ".";
            print "<" URN "s" sindex[sense] ">", LABEL,         "\"" sense "\"",               ".";
            print "<" URN "s" sindex[sense] ">", ISSENSEOF,     "<" URN "w" windex[word] ">",  ".";
            print "<" URN "w" windex[word]  ">", SENSE,         "<" URN "s" sindex[sense] ">", ".";
            print "<" URN "s" sindex[sense] ">", REFERENCE,     "<" URN "c" $1 ">",            ".";
            print "<" URN "c" $1 ">",            ISREFERENCEOF, "<" URN "s" sindex[sense] ">", ".";
        }

        for (isa_id in isas) {
            isa = isas[isa_id];

            sub(":.*$", "", isa);

            if (!(isa in sindex)) {
                sindex[isa] = scount++;

                word = isa;
                gsub("_", " ",  word);
                sub("#.*$", "", word);

                if (!(word in windex)) {
                    windex[word] = wcount++;

                    print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                    print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                    print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                    print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
                }

                print "<" URN "s" sindex[isa] ">",  TYPE,      LEXICALSENSE,                 ".";
                print "<" URN "s" sindex[isa] ">",  LABEL,     "\"" isa "\"",                ".";
                print "<" URN "s" sindex[isa] ">",  ISSENSEOF, "<" URN "w" windex[word] ">", ".";
                print "<" URN "w" windex[word] ">", SENSE,     "<" URN "s" sindex[isa] ">",  ".";
            }

            print "<" URN "s" sindex[sense] ">", BROADER,  "<" URN "s" sindex[isa] ">",   ".";
            print "<" URN "s" sindex[isa] ">",   NARROWER, "<" URN "s" sindex[sense] ">", ".";
        }
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = "\t";

    URN           = "urn:swn:";

    LABEL         = "<http://www.w3.org/2000/01/rdf-schema#label>";
    TYPE          = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";

    CONCEPT       = "<http://www.w3.org/2004/02/skos/core#Concept>";
    LEXICALSENSE  = "<http://lemon-model.net/lemon#LexicalSense>";
    WORD          = "<http://lemon-model.net/lemon#Word>";
    FORM          = "<http://lemon-model.net/lemon#Form>";

    WRITTENREP    = "<http://lemon-model.net/lemon#writtenRep>";
    CANONICALFORM = "<http://lemon-model.net/lemon#canonicalForm>";
    BROADER       = "<http://lemon-model.net/lemon#broader>";
    NARROWER      = "<http://lemon-model.net/lemon#narrower>";
    SENSE         = "<http://lemon-model.net/lemon#sense>";
    ISSENSEOF     = "<http://lemon-model.net/lemon#isSenseOf>";
    REFERENCE     = "<http://lemon-model.net/lemon#reference>";
    ISREFERENCEOF = "<http://lemon-model.net/lemon#isReferenceOf>";
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0 || nisas == 0) next;

    print "<" URN "c" $1 ">", TYPE,  CONCEPT,            ".";
    print "<" URN "c" $1 ">", LABEL, "\"synset" $1 "\"", ".";

    for (sense_id in senses) {
        sense = senses[sense_id];

        sub(":.*$", "", sense);

        if (!(sense in sindex)) {
            sindex[sense] = scount++;

            word = sense;
            gsub("_", " ",  word);
            sub("#.*$", "", word);

            if (!(word in windex)) {
                windex[word] = wcount++;

                print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
            }

            print "<" URN "s" sindex[sense] ">", TYPE,          LEXICALSENSE,                  ".";
            print "<" URN "s" sindex[sense] ">", LABEL,         "\"" sense "\"",               ".";
            print "<" URN "s" sindex[sense] ">", ISSENSEOF,     "<" URN "w" windex[word] ">",  ".";
            print "<" URN "w" windex[word]  ">", SENSE,         "<" URN "s" sindex[sense] ">", ".";
            print "<" URN "s" sindex[sense] ">", REFERENCE,     "<" URN "c" $1 ">",            ".";
            print "<" URN "c" $1 ">",            ISREFERENCEOF, "<" URN "s" sindex[sense] ">", ".";
        }

        for (isa_id in isas) {
            isa = isas[isa_id];

            sub(":.*$", "", isa);

            if (!(isa in sindex)) {
                sindex[isa] = scount++;

                word = isa;
                gsub("_", " ",  word);
                sub("#.*$", "", word);

                if (!(word in windex)) {
                    windex[word] = wcount++;

                    print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                    print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                    print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                    print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
                }

                print "<" URN "s" sindex[isa] ">",  TYPE,      LEXICALSENSE,                 ".";
                print "<" URN "s" sindex[isa] ">",  LABEL,     "\"" isa "\"",                ".";
                print "<" URN "s" sindex[isa] ">",  ISSENSEOF, "<" URN "w" windex[word] ">", ".";
                print "<" URN "w" windex[word] ">", SENSE,     "<" URN "s" sindex[isa] ">",  ".";
            }

            print "<" URN "s" sindex[sense] ">", BROADER,  "<" URN "s" sindex[isa] ">",   ".";
            print "<" URN "s" sindex[isa] ">",   NARROWER, "<" URN "s" sindex[sense] ">", ".";
        }
    }
}


Translators/BingTranslator: fix for Hmong Daw

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 8 Jun 2022
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Set IG, IID, and BingTokenKey (a URL-encoded string containing token and key).
function bingSetup(    ast, content, cookie, group, header, isBody, key,
                       location, status, token, tokens, url) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie
    Cookie = cookie

    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
        l(IG, "IG")
    } else {
        e("[ERROR] Failed to extract IG.")
        exit 1
    }

    match(content, /data-iid="([^"]+)"/, group)
    if (group[1]) {
        IID = group[1]
        l(IID, "IID")
    } else {
        e("[ERROR] Failed to extract IID.")
        exit 1
    }

    match(content, /params_RichTranslateHelper = ([^;]+);/, group)
    if (group[1]) {
        tokenize(tokens, group[1])
        parseJson(ast, tokens)
        key = ast[0 SUBSEP 0]
        token = unparameterize(ast[0 SUBSEP 1])
        BingTokenKey = sprintf("&token=%s&key=%s", quote(token), quote(key))
        l(BingTokenKey, "BingTokenKey")
    } else {
        e("[ERROR] Failed to extract token & key.")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3" sprintf("?IG=%s&IID=%s", IG, IID)
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestUserAgent(text, sl, tl, hl, type) {
    return ""
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl BingTokenKey
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetup()

    # Hot-patches for Bing's own translator language codes
    # See: <https://docs.microsoft.com/en-us/azure/cognitive-services/Translator/language-support>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "tl")    _sl = "fil" # Bing uses 'fil' for Filipino
    if (_sl == "hmn")   _sl = "mww" # Bing uses 'mww' for Hmong Daw
    if (_sl == "no")    _sl = "nb"  # Bing uses Norwegian Bokmål
    # Bing uses 'pt' or 'pt-br' for Brazilian Portuguese, 'pt-pt' for European Portuguese
    if (_sl == "pt-BR") _sl = "pt" # just pt-br
    else if (_sl == "pt-PT") _sl = "pt" # FIXME: support pt-pt
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "tl")    _tl = "fil"
    if (_tl == "hmn")   _tl = "mww"
    if (_tl == "no")    _tl = "nb"
    if (_tl == "pt-BR") _tl = "pt"
    else if (_tl == "pt-PT") _tl = "pt-pt"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            delete ast  # purge old AST
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 \
                                            SUBSEP "transliteration" SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 8 Jun 2022
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Set IG, IID, and BingTokenKey (a URL-encoded string containing token and key).
function bingSetup(    ast, content, cookie, group, header, isBody, key,
                       location, status, token, tokens, url) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie
    Cookie = cookie

    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
        l(IG, "IG")
    } else {
        e("[ERROR] Failed to extract IG.")
        exit 1
    }

    match(content, /data-iid="([^"]+)"/, group)
    if (group[1]) {
        IID = group[1]
        l(IID, "IID")
    } else {
        e("[ERROR] Failed to extract IID.")
        exit 1
    }

    match(content, /params_RichTranslateHelper = ([^;]+);/, group)
    if (group[1]) {
        tokenize(tokens, group[1])
        parseJson(ast, tokens)
        key = ast[0 SUBSEP 0]
        token = unparameterize(ast[0 SUBSEP 1])
        BingTokenKey = sprintf("&token=%s&key=%s", quote(token), quote(key))
        l(BingTokenKey, "BingTokenKey")
    } else {
        e("[ERROR] Failed to extract token & key.")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3" sprintf("?IG=%s&IID=%s", IG, IID)
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestUserAgent(text, sl, tl, hl, type) {
    return ""
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl BingTokenKey
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetup()

    # Hot-patches for Bing's own translator language codes
    # See: <https://docs.microsoft.com/en-us/azure/cognitive-services/Translator/language-support>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "tl")    _sl = "fil" # Bing uses 'fil' for Filipino
    if (_sl == "no")    _sl = "nb"  # Bing uses Norwegian Bokmål
    # Bing uses 'pt' or 'pt-br' for Brazilian Portuguese, 'pt-pt' for European Portuguese
    if (_sl == "pt-BR") _sl = "pt" # just pt-br
    else if (_sl == "pt-PT") _sl = "pt" # FIXME: support pt-pt
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "tl")    _tl = "fil"
    if (_tl == "no")    _tl = "nb"
    if (_tl == "pt-BR") _tl = "pt"
    else if (_tl == "pt-PT") _tl = "pt-pt"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            delete ast  # purge old AST
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 \
                                            SUBSEP "transliteration" SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}