Improved pivot_by_region.awk to not render duplicated regional identifiers

#! /usr/bin/awk -f

{
  region = $1

  if (NF == 1) {
    results[region] = ""
  } else {
    for (i = 2; i <= NF; i++) {
      if ($i ~ /^arn:/) {
        split($i, arn_parts, /:/)
        aws_service             = arn_parts[3]
        aws_resource_identifier = arn_parts[6]

        switch(aws_service) {
          case "cloudformation":
            split(aws_resource_identifier, identifier_parts, "/")
            stack_constant = identifier_parts[1]
            stack_name     = identifier_parts[2]
            stack_additional_identifier = identifier_parts[3]

            results[region, aws_service] = results[region, aws_service] " " stack_name
        }
      } else {
        split($i, parts, /-/)
        aws_resource_type = parts[1]
        if (aws_resource_type ~ /^(aki|ami|ari|eipalloc|elb|subnet|i|vpc)$/) {
          aws_resource_identifier = $i
          if (length(results[region, aws_resource_type]) == 0)
            results[region, aws_resource_type] = aws_resource_identifier
          else
            if (index(results[region, aws_resource_type], aws_resource_identifier) == 0)
              results[region, aws_resource_type] = results[region, aws_resource_type] " " aws_resource_identifier
        }
      }
    }
  }
}

END {
  for (item in results) {
    split(item, keys, SUBSEP)
    print keys[1], keys[2], results[item]
  }
}

#! /usr/bin/awk -f

{
  region = $1

  if (NF == 1) {
    results[region] = ""
  } else {
    for (i = 2; i <= NF; i++) {
      if ($i ~ /^arn:/) {
        split($i, arn_parts, /:/)
        aws_service             = arn_parts[3]
        aws_resource_identifier = arn_parts[6]

        switch(aws_service) {
          case "cloudformation":
            split(aws_resource_identifier, identifier_parts, "/")
            stack_constant = identifier_parts[1]
            stack_name     = identifier_parts[2]
            stack_additional_identifier = identifier_parts[3]

            results[region, aws_service] = results[region, aws_service] " " stack_name
        }
      } else {
        split($i, parts, /-/)
        aws_resource_type = parts[1]
        if (aws_resource_type ~ /^(aki|ami|ari|eipalloc|elb|subnet|i|vpc)$/) {
          aws_resource_identifier = $i
          if (length(results[region, aws_resource_type]) == 0)
            results[region, aws_resource_type] = results[region, aws_resource_type] aws_resource_identifier
          else
            results[region, aws_resource_type] = results[region, aws_resource_type] " " aws_resource_identifier
        }
      }
    }
  }
}

END {
  for (item in results) {
    split(item, keys, SUBSEP)
    print keys[1], keys[2], results[item]
  }
}


added todo

added todo

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
#
#   -v surrogate
#   -v sourceID
#   -v datasetID
#   -v datasetVersion
#   -v subjectDiscriminator
#   -v conversionID
#
#   -v cellDelimiter
#   -v header
#   -v dataStart
#   -v dataEnd
#
#   -v onlyIfCol
#   -v repeatAboveIfEmptyCol
#   -v interpretAsNull
#
#   -v whoami
#   -v machine_uri
#   -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix dbpediaprop:   <http://dbpedia.org/property/> ."
      print "@prefix dbpediaowl:    <http://dbpedia.org/ontology/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix bibo:          <http://purl.org/ontology/bibo/> ."
      print "@prefix prov:          <http://www.w3.org/ns/prov#> ."
      print "@prefix doap:          <http://usefulinc.com/ns/doap#> ."
      print "@prefix nfo:           <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#> ."
      print "@prefix sio:           <http://semanticscience.org/resource/> ."
      print "@prefix org:           <http://www.w3.org/ns/org#> ."
      print "@prefix vsr:           <http://purl.org/twc/vocab/vsr#> ."
      print "@prefix dcat:          <http://www.w3.org/ns/dcat#> ."
      print "@prefix qb:            <http://purl.org/linked-data/cube#> ."
      print "@prefix pml:           <http://provenanceweb.org/ns/pml#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   conversion:name_template \"[/sd]company/[#2]/[r]\";\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
   if(!length(conversionID))        printf("   conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("   conversion:enhancement_identifier \"%s\";\n",conversionID);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }
                                    printf("\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   ov:csvRow 2;\n");
                                    printf("      #   a conversion:DataStartRow;\n");
                                    printf("      #];\n");
                                    printf("\n");
   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n"); # TODO: Jena likes "\u0009" better.
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n"); # TODO: Jena likes "\u0009" better.
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [\n",                        comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",           comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];\n",                                            comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:domain_template \"thing_[r]\";\n");
                                    printf("      #   conversion:domain_name     \"Thing\";\n");
                                    printf("      #];\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:class_name \"Thing\";\n");
                                    printf("      #   conversion:subclass_of <http://purl.org/...>;\n");
                                    printf("      #];\n");
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:bundled_by [ ov:csvCol 1 ];\n");
      printf("         #conversion:label   \"%s\";\n",cellValue);
      printf("         #conversion:equivalent_property dcterms:identifier;\n");
      printf("         #conversion:subproperty_of      dcterms:identifier;\n");
   }
   printf("         conversion:comment \"\";\n");
   if(length(conversionID)) {
      printf("         #conversion:range_template  \"[/sd]thing[.]\";\n");
   }
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      printf("         #conversion:range_name  \"Thing\";\n");
   }
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
#
#   -v surrogate
#   -v sourceID
#   -v datasetID
#   -v datasetVersion
#   -v subjectDiscriminator
#   -v conversionID
#
#   -v cellDelimiter
#   -v header
#   -v dataStart
#   -v dataEnd
#
#   -v onlyIfCol
#   -v repeatAboveIfEmptyCol
#   -v interpretAsNull
#
#   -v whoami
#   -v machine_uri
#   -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix dbpediaprop:   <http://dbpedia.org/property/> ."
      print "@prefix dbpediaowl:    <http://dbpedia.org/ontology/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix bibo:          <http://purl.org/ontology/bibo/> ."
      print "@prefix prov:          <http://www.w3.org/ns/prov#> ."
      print "@prefix doap:          <http://usefulinc.com/ns/doap#> ."
      print "@prefix nfo:           <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#> ."
      print "@prefix sio:           <http://semanticscience.org/resource/> ."
      print "@prefix org:           <http://www.w3.org/ns/org#> ."
      print "@prefix vsr:           <http://purl.org/twc/vocab/vsr#> ."
      print "@prefix dcat:          <http://www.w3.org/ns/dcat#> ."
      print "@prefix qb:            <http://purl.org/linked-data/cube#> ."
      print "@prefix pml:           <http://provenanceweb.org/ns/pml#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   conversion:name_template \"[/sd]company/[#2]/[r]\";\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
   if(!length(conversionID))        printf("   conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("   conversion:enhancement_identifier \"%s\";\n",conversionID);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }
                                    printf("\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   ov:csvRow 2;\n");
                                    printf("      #   a conversion:DataStartRow;\n");
                                    printf("      #];\n");
                                    printf("\n");
   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n");
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [\n",                        comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",           comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];\n",                                            comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:domain_template \"thing_[r]\";\n");
                                    printf("      #   conversion:domain_name     \"Thing\";\n");
                                    printf("      #];\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:class_name \"Thing\";\n");
                                    printf("      #   conversion:subclass_of <http://purl.org/...>;\n");
                                    printf("      #];\n");
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:bundled_by [ ov:csvCol 1 ];\n");
      printf("         #conversion:label   \"%s\";\n",cellValue);
      printf("         #conversion:equivalent_property dcterms:identifier;\n");
      printf("         #conversion:subproperty_of      dcterms:identifier;\n");
   }
   printf("         conversion:comment \"\";\n");
   if(length(conversionID)) {
      printf("         #conversion:range_template  \"[/sd]thing[.]\";\n");
   }
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      printf("         #conversion:range_name  \"Thing\";\n");
   }
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}


bin/util/header2params2.awk accepts new param for cellDelimiter

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:         <http://open.vocab.org/terms/> ."
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(showConversionProcess)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:         <http://open.vocab.org/terms/> ."
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}


Don't modify episodeNumber for trailers

Don't modify episodeNumber for trailers

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode type, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-.[[:digit:]]{3,4}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2) + 0
    shortURL = episodeURL
    sub (/.*watch/,"watch",shortURL)
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a Trailer or Finale (i.e. First look), set episodeType to "T"
    if (episodeURL ~ /-trailer-|-finale-/)
        episodeType = "T"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeURL ~ /-c-x[[:digit:]]{3,4}$/)
        episodeType = "X"
    episodeTitle = fld[4]
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number, and correct if necessary
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0 + 0
    if ((episodeURL ~ /-c-.[[:digit:]]{3,4}$/) && episodeNumber != episodeNumberFromURL) {
        printf ("==> Corrected %s%02d to %s%02d: %s\n", episodeType, episodeNumber, episodeType, \
                episodeNumberFromURL, shortURL) >> ERROR_FILE
        episodeNumber = episodeNumberFromURL
    }
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Corrected %s00 to %s%02d: %s\n", episodeType, episodeType,
                        episodeNumberFromURL, shortURL) >> ERROR_FILE
                episodeNumber = episodeNumberFromURL
            } else {
                printf ("==> Episode number is 00: %s\n", shortURL) >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
                SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeType, episodeNumber, \
                episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " shortURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode type, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-.[[:digit:]]{3,4}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2) + 0
    shortURL = episodeURL
    sub (/.*watch/,"watch",shortURL)
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a Trailer or Finale (i.e. First look), set episodeType to "T"
    if (episodeURL ~ /-trailer-|-finale-/)
        episodeType = "T"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeURL ~ /-c-x[[:digit:]]{3,4}$/)
        episodeType = "X"
    episodeTitle = fld[4]
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number, and correct if necessary
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0 + 0
    if (episodeType == "T") {
        printf ("==> Corrected %s%02d to T01: %s\n", episodeType, episodeNumber, \
                shortURL) >> ERROR_FILE
        episodeNumber = 1
    }
    if ((episodeURL ~ /-c-.[[:digit:]]{3,4}$/) && episodeNumber != episodeNumberFromURL) {
        printf ("==> Corrected %s%02d to %s%02d: %s\n", episodeType, episodeNumber, episodeType, \
                episodeNumberFromURL, shortURL) >> ERROR_FILE
        episodeNumber = episodeNumberFromURL
    }
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Corrected %s00 to %s%02d: %s\n", episodeType, episodeType,
                        episodeNumberFromURL, shortURL) >> ERROR_FILE
                episodeNumber = episodeNumberFromURL
            } else {
                printf ("==> Episode number is 00: %s\n", shortURL) >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
                SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeType, episodeNumber, \
                episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " shortURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}


Help: update

Help: update

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead) {
    initAudioPlayer()
    initPager()
    Platform = Platform ? Platform : detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", Platform)                      \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", join(Option["sls"], "+")) \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "translation engine", Option["engine"])    \
        sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCES") "]"                           \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-S") ", " ansi("bold", "-list-engines")) RS \
        ins(2, "List available translation engines and exit.") RS       \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Translator options:" RS                                     \
        ins(1, ansi("bold", "-e ") ansi("underline", "ENGINE")          \
            ", " ansi("bold", "-engine ") ansi("underline", "ENGINE")) RS \
        ins(2, "Specify the translation engine to use.") RS             \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-identify")) RS                            \
        ins(2, "Language identification.") RS                           \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        ins(1, ansi("bold", "-no-autocorrect")) RS                      \
        ins(2, "Do not autocorrect. (if defaulted by the translation engine)") RS \
        ins(1, ansi("bold", "-no-bidi")) RS                             \
        ins(2, "Do not convert bidirectional texts.") RS                \
        ins(1, ansi("bold", "-no-warn")) RS                             \
        ins(2, "Do not write warning messages to stderr.") RS           \
        ins(1, ansi("bold", "-dump")) RS                                \
        ins(2, "Print raw API response instead.") RS                    \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-speak")) RS                               \
        ins(2, "Listen to the original text.") RS                       \
        ins(1, ansi("bold", "-n ") ansi("underline", "VOICE")           \
            ", " ansi("bold", "-narrator ") ansi("underline", "VOICE")) RS \
        ins(2, "Specify the narrator, and listen to the translation.") RS \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        ins(1, ansi("bold", "-no-translate")) RS                        \
        ins(2, "Do not translate anything when using -speak.") RS       \
        ins(1, ansi("bold", "-download-audio")) RS                      \
        ins(2, "Download the audio to the current directory.") RS       \
        ins(1, ansi("bold", "-download-audio-as ") ansi("underline", "FILENAME")) RS \
        ins(2, "Download the audio to the specified file.") RS          \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view")) RS                             \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-sl ") ansi("underline", "CODES")        \
            ", " ansi("bold", "-source ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-from ") ansi("underline", "CODES")) RS  \
        ins(2, "Specify the source language(s), joined by '+'.") RS     \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODES")        \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-to ") ansi("underline", "CODES")) RS    \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists(ENVIRON["TRANS_DIR"] "/man/" Command ".1"))
        system("man " parameterize(ENVIRON["TRANS_DIR"] "/man/" Command ".1") SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, i, j, name, num, r, rows, saveSortedIn,
                      t1, t2) {
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + (num % 3 ? 1 : 0)
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        # show languages only with stable support
        if (Locale[code]["support"] != "unstable") {
            col = int(i / rows)
            append(cols[col], code)
            i++
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getDisplay(cols[j][i])
                    switch (cols[j][i]) { # fix rendered text width
                    case "he":
                        t1 = sprintf(" %-18s", t1)
                        break
                    case "ur":
                        t1 = sprintf(" %-17s", t1)
                        break
                    case "hi": case "gu": case "km": case "kn":
                    case "my": case "ne": case "pa": case "si":
                    case "ta": case "te": case "yi":
                        t1 = sprintf(" %-16s", t1)
                        break
                    case "yue":
                        t1 = sprintf(" %-13s", t1)
                        break
                    case "ja": case "ko":
                        t1 = sprintf(" %-12s", t1)
                        break
                    case "zh-CN": case "zh-TW":
                        t1 = sprintf(" %-11s", t1)
                        break
                    default:
                        if (length(t1) <= 15)
                            t1 = sprintf(" %-15s", t1)
                    }
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    } else {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getName(cols[j][i])
                    if (length(t1) > 15)
                        t1 = substr(t1, 1, 12) "..."
                    t1 = sprintf(" %-15s", t1)
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead) {
    initAudioPlayer()
    initPager()
    Platform = Platform ? Platform : detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", Platform)                      \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", Option["sl"])           \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "translation engine", Option["engine"])    \
        sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCE") "]"                            \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-S") ", " ansi("bold", "-list-engines")) RS \
        ins(2, "List available translation engines and exit.") RS       \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Translator options:" RS                                     \
        ins(1, ansi("bold", "-e ") ansi("underline", "ENGINE")          \
            ", " ansi("bold", "-engine ") ansi("underline", "ENGINE")) RS \
        ins(2, "Specify the translation engine to use.") RS             \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-identify")) RS                            \
        ins(2, "Language identification.") RS                           \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        ins(1, ansi("bold", "-no-autocorrect")) RS                      \
        ins(2, "Do not autocorrect. (if defaulted by the translation engine)") RS \
        ins(1, ansi("bold", "-no-bidi")) RS                             \
        ins(2, "Do not convert bidirectional texts.") RS                \
        ins(1, ansi("bold", "-no-warn")) RS                             \
        ins(2, "Do not write warning messages to stderr.") RS           \
        ins(1, ansi("bold", "-dump")) RS                                \
        ins(2, "Print raw API response instead.") RS                    \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-speak")) RS                               \
        ins(2, "Listen to the original text.") RS                       \
        ins(1, ansi("bold", "-n ") ansi("underline", "VOICE")           \
            ", " ansi("bold", "-narrator ") ansi("underline", "VOICE")) RS \
        ins(2, "Specify the narrator, and listen to the translation.") RS \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        ins(1, ansi("bold", "-no-translate")) RS                        \
        ins(2, "Do not translate anything when using -speak.") RS       \
        ins(1, ansi("bold", "-download-audio")) RS                      \
        ins(2, "Download the audio to the current directory.") RS       \
        ins(1, ansi("bold", "-download-audio-as ") ansi("underline", "FILENAME")) RS \
        ins(2, "Download the audio to the specified file.") RS          \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view")) RS                             \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-sl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-source ") ansi("underline", "CODE")     \
            ", " ansi("bold", "-from ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify the source language.") RS                       \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-to ") ansi("underline", "CODES")) RS    \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists(ENVIRON["TRANS_DIR"] "/man/" Command ".1"))
        system("man " parameterize(ENVIRON["TRANS_DIR"] "/man/" Command ".1") SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, i, j, name, num, r, rows, saveSortedIn,
                      t1, t2) {
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + (num % 3 ? 1 : 0)
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        # show languages only with stable support
        if (Locale[code]["support"] != "unstable") {
            col = int(i / rows)
            append(cols[col], code)
            i++
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getDisplay(cols[j][i])
                    switch (cols[j][i]) { # fix rendered text width
                    case "he":
                        t1 = sprintf(" %-18s", t1)
                        break
                    case "ur":
                        t1 = sprintf(" %-17s", t1)
                        break
                    case "hi": case "gu": case "km": case "kn":
                    case "my": case "ne": case "pa": case "si":
                    case "ta": case "te": case "yi":
                        t1 = sprintf(" %-16s", t1)
                        break
                    case "yue":
                        t1 = sprintf(" %-13s", t1)
                        break
                    case "ja": case "ko":
                        t1 = sprintf(" %-12s", t1)
                        break
                    case "zh-CN": case "zh-TW":
                        t1 = sprintf(" %-11s", t1)
                        break
                    default:
                        if (length(t1) <= 15)
                            t1 = sprintf(" %-15s", t1)
                    }
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    } else {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getName(cols[j][i])
                    if (length(t1) > 15)
                        t1 = substr(t1, 1, 12) "..."
                    t1 = sprintf(" %-15s", t1)
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}