added commented out interpret empty as null params for easy uncommenting.

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(showConversionProcess)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:         <http://open.vocab.org/terms/> ."
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:intepretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(showConversionProcess)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:         <http://open.vocab.org/terms/> ."
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull) && length(conversionID)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}


When generating the version map file, order versions oldest
first to make it easier for rtld to choose the oldest version
of a symbol.

Sumbitted by:	kan

#
# Copyright (C) 2006 Daniel M. Eischen.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# $FreeBSD$
#

#
# Make a list of all the library versions listed in the master file.
#
#   versions[] - array indexed by version name, contains number
#                of symbols (+ 1) found for each version.
#   successors[] - array index by version name, contains successor
#                  version name.
#   symbols[][] - array index by [version name, symbol index], contains
#                 names of symbols defined for each version.
#
BEGIN {
	brackets = 0;
	errors = 0;
	version_count = 0;
	current_version = "";
	stderr = "/dev/stderr";
	while (getline < vfile) {
		# Strip comments.
		sub("#.*$", "", $0);

		# Strip trailing spaces.
		sub(" *$", "", $0);

		if (/^[ \t]*[a-zA-Z0-9._]+ *{/) {
			brackets++;
			symver = $1;
			versions[symver] = 1;
			successors[symver] = "";
			generated[symver] = 0;
			version_count++;
		}
		else if (/^[ \t]*} *[a-zA-Z0-9._]+ *;/) {
			# Strip semicolon.
			gsub(";", "", $2);
			if (symver == "")
				printf("Unmatched bracket.\n");
			else if (versions[$2] != 1)
				printf("File %s: %s has unknown " \
				    "successor %s\n", vfile, symver, $2);
			else
				successors[symver] = $2;
			brackets--;
		}
		else if (/^[ \t]*};/) {
			if (symver == "")
				printf("File %s: Unmatched bracket.\n",
				    vfile) > stderr;
			# No successor
			brackets--;
		}
		else if (/^[ \t]*}/) {
			printf("File %s: Missing ending semi-colon.\n",
			    vfile) > stderr;
		}
		else if (/^$/)
			;  # Ignore blank lines.
		else
			printf("File %s: Unknown directive: %s\n",
			    vfile, $0) > stderr;
	}
	brackets = 0;
}

/.*/ {
	# Delete comments, preceding and trailing whitespace, then
	# consume blank lines.
	sub("#.*$", "", $0);
	sub("^[ \t]+", "", $0);
	sub("[ \t]+$", "", $0);
	if ($0 == "")
		next;
}

/^[a-zA-Z0-9._]+ +{$/ {
	# Strip bracket from version name.
	sub("{", "", $1);
	if (current_version != "")
		printf("File %s, line %d: Illegal nesting detected.\n",
		    FILENAME, FNR) > stderr;
	else if (versions[$1] == 0) {
		printf("File %s, line %d: Undefined " \
		    "library version %s\n", FILENAME, FNR, $1) > stderr;
		# Remove this entry from the versions.
		delete versions[$1];
	}
	else
		current_version = $1;
	brackets++;
	next;
}

/^[a-zA-Z0-9._]+ *;$/ {
	if (current_version != "") {
		count = versions[current_version];
		versions[current_version]++;
		symbols[current_version, count] = $1;
	}
	next;
}

/^} *;$/ {
	brackets--;
	if (brackets < 0) {
		printf("File %s, line %d: Unmatched bracket.\n",
		    FILENAME, FNR, $1) > stderr;
		brackets = 0;	# Reset
	}
	current_version = "";
	next;
}

/.*/ {
	printf("File %s, line %d: Unknown directive: '%s'\n",
	    FILENAME, FNR, $0) > stderr;
}

function print_version(v)
{
	# This function is recursive, so return if this version
	# has already been printed.  Otherwise, if there is an
	# ancestral version, recursively print its symbols before
	# printing the symbols for this version.
	#
	if (generated[v] == 1)
		return;
	if (successors[v] != "")
		print_version(successors[v]);

	printf("%s {\n", v);

	# The version count is always one more that actual,
	# so the loop ranges from 1 to n-1.
	#
	for (i = 1; i < versions[v]; i++) {
		if (i == 1)
			printf("global:\n");
		printf("\t%s\n", symbols[v, i]);
	}
	if (successors[v] == "") {
		# This version succeeds no other version.
		printf("local:\n");
		printf("\t*;\n");
		printf("};\n");
	}
	else
		printf("} %s;\n", successors[v]);
	printf("\n");

	generated[v] = 1;
    }
END {
	for (v in versions) {
		print_version(v);
	}
}

#
# Copyright (C) 2006 Daniel M. Eischen.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# $FreeBSD$
#

#
# Make a list of all the library versions listed in the master file.
#
#   versions[] - array indexed by version name, contains number
#                of symbols (+ 1) found for each version.
#   successors[] - array index by version name, contains successor
#                  version name.
#   symbols[][] - array index by [version name, symbol index], contains
#                 names of symbols defined for each version.
#
BEGIN {
	brackets = 0;
	errors = 0;
	version_count = 0;
	current_version = "";
	stderr = "/dev/stderr";
	while (getline < vfile) {
		# Strip comments.
		sub("#.*$", "", $0);

		# Strip trailing spaces.
		sub(" *$", "", $0);

		if (/^[ \t]*[a-zA-Z0-9._]+ *{/) {
			brackets++;
			symver = $1;
			versions[symver] = 1;
			successors[symver] = "";
			version_count++;
		}
		else if (/^[ \t]*} *[a-zA-Z0-9._]+ *;/) {
			# Strip semicolon.
			gsub(";", "", $2);
			if (symver == "")
				printf("Unmatched bracket.\n");
			else if (versions[$2] != 1)
				printf("File %s: %s has unknown " \
				    "successor %s\n", vfile, symver, $2);
			else
				successors[symver] = $2;
			brackets--;
		}
		else if (/^[ \t]*};/) {
			if (symver == "")
				printf("File %s: Unmatched bracket.\n",
				    vfile) > stderr;
			# No successor
			brackets--;
		}
		else if (/^[ \t]*}/) {
			printf("File %s: Missing ending semi-colon.\n",
			    vfile) > stderr;
		}
		else if (/^$/)
			;  # Ignore blank lines.
		else
			printf("File %s: Unknown directive: %s\n",
			    vfile, $0) > stderr;
	}
	brackets = 0;
}

/.*/ {
	# Delete comments, preceding and trailing whitespace, then
	# consume blank lines.
	sub("#.*$", "", $0);
	sub("^[ \t]+", "", $0);
	sub("[ \t]+$", "", $0);
	if ($0 == "")
		next;
}

/^[a-zA-Z0-9._]+ +{$/ {
	# Strip bracket from version name.
	sub("{", "", $1);
	if (current_version != "")
		printf("File %s, line %d: Illegal nesting detected.\n",
		    FILENAME, FNR) > stderr;
	else if (versions[$1] == 0) {
		printf("File %s, line %d: Undefined " \
		    "library version %s\n", FILENAME, FNR, $1) > stderr;
		# Remove this entry from the versions.
		delete versions[$1];
	}
	else
		current_version = $1;
	brackets++;
	next;
}

/^[a-zA-Z0-9._]+ *;$/ {
	if (current_version != "") {
		count = versions[current_version];
		versions[current_version]++;
		symbols[current_version, count] = $1;
	}
	next;
}

/^} *;$/ {
	brackets--;
	if (brackets < 0) {
		printf("File %s, line %d: Unmatched bracket.\n",
		    FILENAME, FNR, $1) > stderr;
		brackets = 0;	# Reset
	}
	current_version = "";
	next;
}

/.*/ {
	printf("File %s, line %d: Unknown directive: '%s'\n",
	    FILENAME, FNR, $0) > stderr;
}

END {
	for (v in versions) {
		printf("\n");
		printf("%s {\n", v);

		# The version count is always one more that actual,
		# so the loop ranges from 1 to n-1.
		#
		for (i = 1; i < versions[v]; i++) {
			if (i == 1)
				printf("global:\n");
			printf("\t%s\n", symbols[v, i]);
		}
		if (successors[v] == "") {
			# This version succeeds no other version.
			printf("local:\n");
			printf("\t*;\n");
			printf("};\n");
		}
		else
			printf("} %s;\n", successors[v]);
	}
}


do not create _STR_ defines; this is useless, makes pcmciadevs.h larger and unclean.

#! /usr/bin/awk -f
#	$OpenBSD: devlist2h.awk,v 1.4 2000/02/01 16:54:41 fgsch Exp $
#	$NetBSD: devlist2h.awk,v 1.2 1998/07/22 11:47:13 christos Exp $
#
# Copyright (c) 1998, Christos Zoulas
# Copyright (c) 1995, 1996 Christopher G. Demetriou
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Christopher G. Demetriou.
#      This product includes software developed by Christos Zoulas
# 4. The name of the author(s) may not be used to endorse or promote products
#    derived from this software without specific prior written permission
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
function collectline(f, line) {
	oparen = 0
	line = ""
	while (f <= NF) {
		if ($f == "#") {
			line = line "("
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			line = line $f
			if (f < NF)
				line = line " "
			f++
			continue
		}
		line = line $f
		if (f < NF)
			line = line " "
		f++
	}
	if (oparen)
		line = line ")"
	return line
}
BEGIN {
	nproducts = nvendors = 0
	hfile="pcmciadevs.h"
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)

	printf("/*\t\$OpenBSD\$\t*/\n\n") > hfile
	printf("/*\n") > hfile
	printf(" * THIS FILE AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > hfile
	printf(" *\n") > hfile
	printf(" * generated from:\n") > hfile
	printf(" *\t%s\n", VERSION) > hfile
	printf(" */\n") > hfile

	next
}
$1 == "vendor" {
	nvendors++

	vendorindex[$2] = nvendors;		# record index for this name, for later.
	vendors[nvendors, 1] = $2;		# name
	vendors[nvendors, 2] = $3;		# id
	printf("#define\tPCMCIA_VENDOR_%s\t%s\t", vendors[nvendors, 1],
	    vendors[nvendors, 2]) > hfile
	vendors[nvendors, 3] = collectline(4, line)
	printf("/* %s */\n", vendors[nvendors, 3]) > hfile
	next
}
$1 == "product" {
	nproducts++

	products[nproducts, 1] = $2;		# vendor name
	products[nproducts, 2] = $3;		# product id
	products[nproducts, 3] = $4;		# id

	f = 5;

	if ($4 == "{") {
		products[nproducts, 3] = "PCMCIA_PRODUCT_INVALID"
		z = "{ "
		for (i = 0; i < 4; i++) {
			if (f <= NF) {
				gsub("&sp", " ", $f)
				gsub("&tab", "\t", $f)
				gsub("&nl", "\n", $f)
				z = z $f " "
				f++
			}
			else {
				if (i == 3)
					z = z "NULL "
				else
					z = z "NULL, "
			}
		}
		products[nproducts, 4] = z $f
		f++
	}
	else {
		products[nproducts, 4] = "{ NULL, NULL, NULL, NULL }"
	}
	printf("#define\tPCMCIA_CIS_%s_%s\t%s\n",
	    products[nproducts, 1], products[nproducts, 2],
	    products[nproducts, 4]) > hfile
	printf("#define\tPCMCIA_PRODUCT_%s_%s\t%s\n", products[nproducts, 1],
	    products[nproducts, 2], products[nproducts, 3]) > hfile

#	products[nproducts, 5] = collectline(f, line)
#
#	printf("#define\tPCMCIA_STR_%s_%s\t\"%s\"\n",
#	    products[nproducts, 1], products[nproducts, 2],
#	    products[nproducts, 5]) > hfile

	next
}
{
	if ($0 == "")
		blanklines++
	print $0 > hfile
}

#! /usr/bin/awk -f
#	$OpenBSD: devlist2h.awk,v 1.3 1999/08/18 22:30:04 d Exp $
#	$NetBSD: devlist2h.awk,v 1.2 1998/07/22 11:47:13 christos Exp $
#
# Copyright (c) 1998, Christos Zoulas
# Copyright (c) 1995, 1996 Christopher G. Demetriou
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Christopher G. Demetriou.
#      This product includes software developed by Christos Zoulas
# 4. The name of the author(s) may not be used to endorse or promote products
#    derived from this software without specific prior written permission
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
function collectline(f, line) {
	oparen = 0
	line = ""
	while (f <= NF) {
		if ($f == "#") {
			line = line "("
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			line = line $f
			if (f < NF)
				line = line " "
			f++
			continue
		}
		line = line $f
		if (f < NF)
			line = line " "
		f++
	}
	if (oparen)
		line = line ")"
	return line
}
BEGIN {
	nproducts = nvendors = 0
	hfile="pcmciadevs.h"
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)

	printf("/*\t\$OpenBSD\$\t*/\n\n") > hfile
	printf("/*\n") > hfile
	printf(" * THIS FILE AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > hfile
	printf(" *\n") > hfile
	printf(" * generated from:\n") > hfile
	printf(" *\t%s\n", VERSION) > hfile
	printf(" */\n") > hfile

	next
}
$1 == "vendor" {
	nvendors++

	vendorindex[$2] = nvendors;		# record index for this name, for later.
	vendors[nvendors, 1] = $2;		# name
	vendors[nvendors, 2] = $3;		# id
	printf("#define\tPCMCIA_VENDOR_%s\t%s\t", vendors[nvendors, 1],
	    vendors[nvendors, 2]) > hfile
	vendors[nvendors, 3] = collectline(4, line)
	printf("/* %s */\n", vendors[nvendors, 3]) > hfile
	next
}
$1 == "product" {
	nproducts++

	products[nproducts, 1] = $2;		# vendor name
	products[nproducts, 2] = $3;		# product id
	products[nproducts, 3] = $4;		# id

	f = 5;

	if ($4 == "{") {
		products[nproducts, 3] = "PCMCIA_PRODUCT_INVALID"
		z = "{ "
		for (i = 0; i < 4; i++) {
			if (f <= NF) {
				gsub("&sp", " ", $f)
				gsub("&tab", "\t", $f)
				gsub("&nl", "\n", $f)
				z = z $f " "
				f++
			}
			else {
				if (i == 3)
					z = z "NULL "
				else
					z = z "NULL, "
			}
		}
		products[nproducts, 4] = z $f
		f++
	}
	else {
		products[nproducts, 4] = "{ NULL, NULL, NULL, NULL }"
	}
	printf("#define\tPCMCIA_CIS_%s_%s\t%s\n",
	    products[nproducts, 1], products[nproducts, 2],
	    products[nproducts, 4]) > hfile
	printf("#define\tPCMCIA_PRODUCT_%s_%s\t%s\n", products[nproducts, 1],
	    products[nproducts, 2], products[nproducts, 3]) > hfile

	products[nproducts, 5] = collectline(f, line)

	printf("#define\tPCMCIA_STR_%s_%s\t\"%s\"\n",
	    products[nproducts, 1], products[nproducts, 2],
	    products[nproducts, 5]) > hfile

	next
}
{
	if ($0 == "")
		blanklines++
	print $0 > hfile
}


Commons: use string "\0" to indicate NONE

####################################################################
# Commons.awk                                                      #
####################################################################

# Initialize constants.
function initConst() {
    NULLSTR = ""
    NONE = "\0"
    TRUE = 1

    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
    PIPE = " | "
}

## Arrays:

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Return 1 if the value is non-empty or an array that contains anything;
# Otherwise, return 0.
function exists(value) {
    if (isarray(value))
        return anything(value)
    else
        return value ? 1 : 0
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return NULLSTR
}

# Return non-zero if two values are identical;
# Otherwise, return 0.
function identical(x, y,
                   ####
                   i) {
    if (!isarray(x) && !isarray(y))
        return x == y
    else if (isarray(x) && isarray(y)) {
        if (length(x) != length(y)) return 0
        for (i in x)
            if (!identical(x[i], y[i])) return 0
        return 1
    } else
        return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Comparator function used for controlling array scanning order.
# Like @ind_num_asc, but compare on index fields separated by SUBSEP.
function compareByIndexFields(i1, v1, i2, v2,
                              ####
                              t1, t2, tl, j) {
    split(i1, t1, SUBSEP)
    split(i2, t2, SUBSEP)
    tl = length(t1) < length(t2) ? length(t1) : length(t2)
    for (j = 1; j <= tl; j++) {
        if (t1[j] < t2[j])
            return -1
        else if (t1[j] > t2[j])
            return 1
    }
    return 0
}

## Strings:

# Return non-zero if the string represents a numeral;
# Otherwise, return 0.
function isnum(string) {
    return string == string + 0
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return NULLSTR
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return NULLSTR
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = NULLSTR
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Reverse a string.
function reverse(string,
                 ####
                 i, temp) {
    temp = NULLSTR
    for (i = length(string); i > 0; i--)
        temp = temp substr(string, i, 1);
    return temp
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameter
    if (!sortedIn)
        sortedIn = "compareByIndexFields"

    temp = NULLSTR
    j = 0
    if (isarray(array)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != NULLSTR)
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array

    return temp
}

# Split a string into characters.
function explode(string, array) {
    split(string, array, NULLSTR)
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    case "u0026":
        return "&" # Unicode Character 'AMPERSAND'
    case "u003c":
        return "<" # Unicode Character 'LESS-THAN SIGN'
    case "u003e":
        return ">" # Unicode Character 'GREATER-THAN SIGN'
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, cc, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    explode(string, s)
    string = NULLSTR
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            if (cc) {
                cc = cc c
                if (length(cc) == 5) {
                    string = string escapeChar(cc)
                    escaping = 0 # escape ends
                    cc = NULLSTR
                }
            } else if (c == "u") {
                cc = c
            } else {
                string = string escapeChar(c)
                escaping = 0 # escape ends
            }
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/\\/, "\\\\", string) # substitute backslashes first
    gsub(/"/, "\\\"", string)

    return string
}

# Reverse of escape(string).
function unescape(string) {
    gsub(/\\"/, "\"", string)
    gsub(/\\\\/, "\\", string) # substitute backslashes last

    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Reverse of parameterize(string, quotationMark).
function unparameterize(string,    temp) {
    match(string, /^'(.*)'$/, temp)
    if (temp[0]) { # use temp[0] (there IS a match for quoted empty string)
        string = temp[1]
        gsub(/'\\''/, "'", string)
        return string
    }
    match(string, /^"(.*)"$/, temp)
    if (temp[0]) {
        string = temp[1]
        return unescape(string)
    }
    return string
}

# Convert any value to human-readable string.
function toString(value, inline, heredoc, valOnly, numSub, level, sortedIn,
                  ####
                  i, items, j, k, p, saveSortedIn, temp, v) {
    if (!level) level = 0
    if (!sortedIn)
        sortedIn = "compareByIndexFields"

    if (isarray(value)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        p = 0
        for (i in value) {
            split(i, j, SUBSEP); k = join(j, ",")
            if (!numSub || !isnum(k)) k = parameterize(k, "\"")
            v = toString(value[i], inline, heredoc, valOnly, numSub, level + 1, sortedIn)
            if (!isarray(value[i])) v = parameterize(v, "\"")
            if (valOnly)
                items[p++] = inline ? v : (replicate("\t", level) v)
            else
                items[p++] = inline ? (k ": " v) :
                    (replicate("\t", level) k "\t" v)
        }
        PROCINFO["sorted_in"] = saveSortedIn
        temp = inline ? join(items, ", ") :
            ("\n" join(items, "\n") "\n" replicate("\t", level))
        temp = valOnly ? ("[" temp "]") : ("{" temp "}")
        return temp
    } else {
        if (heredoc)
            return "'''\n" value "\n'''"
        else
            return value
    }
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, NULLSTR, line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, NULLSTR, line)
    # Remove in-line comment
    gsub(/#[^"/]*$/, NULLSTR, line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, NULLSTR, line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 0 if the string starts with '0', 'f', 'n' or 'off';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^([0fn]|off)/) ? 0 : 1
}

## Display & Debugging:

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("yellow", text)) > STDERR
}

# What a terrible failure.
function wtf(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Debug any value.
function da(value, name, inline, heredoc, valOnly, numSub, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!name)
        name = "_"
    if (!sortedIn)
        sortedIn = "compareByIndexFields"

    d(name " = " toString(value, inline, heredoc, valOnly, numSub, 0, sortedIn))
    #if (isarray(value)) {
    #    saveSortedIn = PROCINFO["sorted_in"]
    #    PROCINFO["sorted_in"] = sortedIn
    #    for (i in value) {
    #        split(i, j, SUBSEP)
    #        da(value[i], sprintf(name "[%s]", join(j, ",")), sortedIn)
    #    }
    #    PROCINFO["sorted_in"] = saveSortedIn
    #} else
    #    d(name " = " value)
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

## URLs:

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\t"] = "%09"
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = NULLSTR
    explode(string, s)
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Return the URL-decoded string.
function unquote(string,    i, k, r, s, temp) {
    r = NULLSTR
    explode(string, s)
    temp = NULLSTR
    for (i = 1; i <= length(s); i++)
        if (temp) {
            temp = temp s[i]
            if (length(temp) > 2) {
                for (k in UrlEncoding)
                    if (temp == UrlEncoding[k]) {
                        r = r k
                        temp = NULLSTR
                        break
                    }
                if (temp) {
                    r = r temp
                    temp = NULLSTR
                }
            }
        } else {
            if (s[i] != "%")
                r = r s[i]
            else
                temp = s[i]
        }
    if (temp)
        r = r temp
    return r
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

## System:

# Read from a file and return its content.
function readFrom(file,    line, text) {
    if (!file) file = "/dev/stdin"
    text = NULLSTR
    while (getline line < file)
        text = (text ? text "\n" : NULLSTR) line
    return text
}

# Write text to file.
function writeTo(text, file) {
    if (!file) file = "/dev/stdout"
    print text > file
}

# Return the output of a command.
function getOutput(command,    content, line) {
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    close(command)
    return content
}

# Return non-zero if file exists; otherwise return 0.
function fileExists(file) {
    return !system("test -f " parameterize(file))
}

# Return non-zero if file exists and is a directory; otherwise return 0.
function dirExists(file) {
    return !system("test -d " parameterize(file))
}

# Detect whether a program exists in path.
# Return the name (or output) if the program call writes anything to stdout;
# Otherwise, return a null string.
function detectProgram(prog, arg, returnOutput,    command, temp) {
    command = prog " " arg SUPERR
    command | getline temp
    close(command)
    if (returnOutput)
        return temp
    if (temp)
        return prog
    return NULLSTR
}

# Return the HEAD revision if the current directory is a git repo;
# Otherwise return a null string.
function getGitHead(    line, group) {
    if (fileExists(".git/HEAD")) {
        getline line < ".git/HEAD"
        match(line, /^ref: (.*)$/, group)
        if (fileExists(".git/" group[1])) {
            getline line < (".git/" group[1])
            return substr(line, 1, 7)
        } else
            return NULLSTR
    } else
        return NULLSTR
}

BEGIN {
    initConst()
    initAnsiCode()
    initUrlEncoding()
    initUriSchemes()
}

####################################################################
# Commons.awk                                                      #
####################################################################

# Initialize constants.
function initConst() {
    NULLSTR = ""
    TRUE = 1

    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
    PIPE = " | "
}

## Arrays:

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Return 1 if the value is non-empty or an array that contains anything;
# Otherwise, return 0.
function exists(value) {
    if (isarray(value))
        return anything(value)
    else
        return value ? 1 : 0
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return NULLSTR
}

# Return non-zero if two values are identical;
# Otherwise, return 0.
function identical(x, y,
                   ####
                   i) {
    if (!isarray(x) && !isarray(y))
        return x == y
    else if (isarray(x) && isarray(y)) {
        if (length(x) != length(y)) return 0
        for (i in x)
            if (!identical(x[i], y[i])) return 0
        return 1
    } else
        return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Comparator function used for controlling array scanning order.
# Like @ind_num_asc, but compare on index fields separated by SUBSEP.
function compareByIndexFields(i1, v1, i2, v2,
                              ####
                              t1, t2, tl, j) {
    split(i1, t1, SUBSEP)
    split(i2, t2, SUBSEP)
    tl = length(t1) < length(t2) ? length(t1) : length(t2)
    for (j = 1; j <= tl; j++) {
        if (t1[j] < t2[j])
            return -1
        else if (t1[j] > t2[j])
            return 1
    }
    return 0
}

## Strings:

# Return non-zero if the string represents a numeral;
# Otherwise, return 0.
function isnum(string) {
    return string == string + 0
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return NULLSTR
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return NULLSTR
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = NULLSTR
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Reverse a string.
function reverse(string,
                 ####
                 i, temp) {
    temp = NULLSTR
    for (i = length(string); i > 0; i--)
        temp = temp substr(string, i, 1);
    return temp
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameter
    if (!sortedIn)
        sortedIn = "compareByIndexFields"

    temp = NULLSTR
    j = 0
    if (isarray(array)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != NULLSTR)
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array

    return temp
}

# Split a string into characters.
function explode(string, array) {
    split(string, array, NULLSTR)
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    case "u0026":
        return "&" # Unicode Character 'AMPERSAND'
    case "u003c":
        return "<" # Unicode Character 'LESS-THAN SIGN'
    case "u003e":
        return ">" # Unicode Character 'GREATER-THAN SIGN'
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, cc, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    explode(string, s)
    string = NULLSTR
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            if (cc) {
                cc = cc c
                if (length(cc) == 5) {
                    string = string escapeChar(cc)
                    escaping = 0 # escape ends
                    cc = NULLSTR
                }
            } else if (c == "u") {
                cc = c
            } else {
                string = string escapeChar(c)
                escaping = 0 # escape ends
            }
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/\\/, "\\\\", string) # substitute backslashes first
    gsub(/"/, "\\\"", string)

    return string
}

# Reverse of escape(string).
function unescape(string) {
    gsub(/\\"/, "\"", string)
    gsub(/\\\\/, "\\", string) # substitute backslashes last

    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Reverse of parameterize(string, quotationMark).
function unparameterize(string,    temp) {
    match(string, /^'(.*)'$/, temp)
    if (temp[0]) { # use temp[0] (there IS a match for quoted empty string)
        string = temp[1]
        gsub(/'\\''/, "'", string)
        return string
    }
    match(string, /^"(.*)"$/, temp)
    if (temp[0]) {
        string = temp[1]
        return unescape(string)
    }
    return string
}

# Convert any value to human-readable string.
function toString(value, inline, heredoc, valOnly, numSub, level, sortedIn,
                  ####
                  i, items, j, k, p, saveSortedIn, temp, v) {
    if (!level) level = 0
    if (!sortedIn)
        sortedIn = "compareByIndexFields"

    if (isarray(value)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        p = 0
        for (i in value) {
            split(i, j, SUBSEP); k = join(j, ",")
            if (!numSub || !isnum(k)) k = parameterize(k, "\"")
            v = toString(value[i], inline, heredoc, valOnly, numSub, level + 1, sortedIn)
            if (!isarray(value[i])) v = parameterize(v, "\"")
            if (valOnly)
                items[p++] = inline ? v : (replicate("\t", level) v)
            else
                items[p++] = inline ? (k ": " v) :
                    (replicate("\t", level) k "\t" v)
        }
        PROCINFO["sorted_in"] = saveSortedIn
        temp = inline ? join(items, ", ") :
            ("\n" join(items, "\n") "\n" replicate("\t", level))
        temp = valOnly ? ("[" temp "]") : ("{" temp "}")
        return temp
    } else {
        if (heredoc)
            return "'''\n" value "\n'''"
        else
            return value
    }
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, NULLSTR, line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, NULLSTR, line)
    # Remove in-line comment
    gsub(/#[^"/]*$/, NULLSTR, line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, NULLSTR, line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 0 if the string starts with '0', 'f', 'n' or 'off';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^([0fn]|off)/) ? 0 : 1
}

## Display & Debugging:

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("yellow", text)) > STDERR
}

# What a terrible failure.
function wtf(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Debug any value.
function da(value, name, inline, heredoc, valOnly, numSub, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!name)
        name = "_"
    if (!sortedIn)
        sortedIn = "compareByIndexFields"

    d(name " = " toString(value, inline, heredoc, valOnly, numSub, 0, sortedIn))
    #if (isarray(value)) {
    #    saveSortedIn = PROCINFO["sorted_in"]
    #    PROCINFO["sorted_in"] = sortedIn
    #    for (i in value) {
    #        split(i, j, SUBSEP)
    #        da(value[i], sprintf(name "[%s]", join(j, ",")), sortedIn)
    #    }
    #    PROCINFO["sorted_in"] = saveSortedIn
    #} else
    #    d(name " = " value)
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

## URLs:

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\t"] = "%09"
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = NULLSTR
    explode(string, s)
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Return the URL-decoded string.
function unquote(string,    i, k, r, s, temp) {
    r = NULLSTR
    explode(string, s)
    temp = NULLSTR
    for (i = 1; i <= length(s); i++)
        if (temp) {
            temp = temp s[i]
            if (length(temp) > 2) {
                for (k in UrlEncoding)
                    if (temp == UrlEncoding[k]) {
                        r = r k
                        temp = NULLSTR
                        break
                    }
                if (temp) {
                    r = r temp
                    temp = NULLSTR
                }
            }
        } else {
            if (s[i] != "%")
                r = r s[i]
            else
                temp = s[i]
        }
    if (temp)
        r = r temp
    return r
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

## System:

# Read from a file and return its content.
function readFrom(file,    line, text) {
    if (!file) file = "/dev/stdin"
    text = NULLSTR
    while (getline line < file)
        text = (text ? text "\n" : NULLSTR) line
    return text
}

# Write text to file.
function writeTo(text, file) {
    if (!file) file = "/dev/stdout"
    print text > file
}

# Return the output of a command.
function getOutput(command,    content, line) {
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    close(command)
    return content
}

# Return non-zero if file exists; otherwise return 0.
function fileExists(file) {
    return !system("test -f " parameterize(file))
}

# Return non-zero if file exists and is a directory; otherwise return 0.
function dirExists(file) {
    return !system("test -d " parameterize(file))
}

# Detect whether a program exists in path.
# Return the name (or output) if the program call writes anything to stdout;
# Otherwise, return a null string.
function detectProgram(prog, arg, returnOutput,    command, temp) {
    command = prog " " arg SUPERR
    command | getline temp
    close(command)
    if (returnOutput)
        return temp
    if (temp)
        return prog
    return NULLSTR
}

# Return the HEAD revision if the current directory is a git repo;
# Otherwise return a null string.
function getGitHead(    line, group) {
    if (fileExists(".git/HEAD")) {
        getline line < ".git/HEAD"
        match(line, /^ref: (.*)$/, group)
        if (fileExists(".git/" group[1])) {
            getline line < (".git/" group[1])
            return substr(line, 1, 7)
        } else
            return NULLSTR
    } else
        return NULLSTR
}

BEGIN {
    initConst()
    initAnsiCode()
    initUrlEncoding()
    initUriSchemes()
}


Fix missing episode numbers in MHz

Fix missing episode numbers in MHz

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://mhzchoice.vhx.tv/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://mhzchoice.vhx.tv/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-[[:digit:]]{5}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2)
    episodeTitle = fld[4]
    gsub (/"/,"\"\"",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Corrected E00: " episodeURL " to E" episodeNumber >> ERROR_FILE
            } else {
                print "==> Episode number is 00: " episodeURL >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02dE%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
            SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeNumber, episodeTitle, \
            episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " episodeURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://mhzchoice.vhx.tv/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://mhzchoice.vhx.tv/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    episodeTitle = fld[4]
    gsub (/"/,"\"\"",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Episode number is 00: " episodeURL >> ERROR_FILE
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02dE%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
            SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeNumber, episodeTitle, \
            episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " episodeURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}


Oops, don't why I missed out 'mode=remove' changes. Fixed remove mode for static interface


cleaning global.e1.params.ttl with bin/util/update-e-params-subject-discrim.awk

# update-e-params-subject-discrim.awk
# params:
#    baseURI
#    sourceID
#    dataset_identifier
#    datasetVersion
#    layerID
#    subjectDiscriminator

{
   if( $1 == "@prefix" && $2 == ":" ) {
      # Fix up default namespace.
      printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/enhancement/%s/> .\n",baseURI,sourceID,dataset_identifier,datasetVersion,layerID);

   }else if( $0 == ":dataset a void:Dataset;" ) {
      # Change subject to name of layer dataset itself.
      printf("<%s/source/%s/dataset/%s/version/%s/conversion/enhancement/%s>\n",baseURI,sourceID,dataset_identifier,datasetVersion,layerID);

   }else if( $0~"conversion:subject_discriminator" ) {
      printf("      conversion:subject_discriminator \"%s\";\n",subjectDiscriminator)

   }else if( $0~"conversion:dataset_version" ) {                          # TODO: DEPRECATED (but converter not recognizing it)
      printf("   conversion:dataset_version    \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:version_identifier" ) {
      printf("   conversion:version_identifier \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:dataset_identifier" && length(dataset_identifier) > 0) {
      printf("   conversion:dataset_identifier \"%s\";\n",dataset_identifier)

   }else {
      print
   }
}

# update-e-params-subject-discrim.awk
# params:
#    baseURI
#    sourceID
#    dataset_identifier
#    datasetVersion
#    layerID
#    subjectDiscriminator

{
   if( $1 == "@prefix" && $2 == ":" ) {
      # Fix up default namespace.
      printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/enhancement/%s/> .",baseURI,sourceID,dataset_identifier,datasetVersion,layerID);

   }else if( $0 == ":dataset a void:Dataset;" ) {
      # Change subject to name of layer dataset itself.
      printf("%s/source/%s/dataset/%s/version/%s/conversion/enhancement/%s",baseURI,sourceID,dataset_identifier,datasetVersion,layerID);

   }else if( $0~"conversion:subject_discriminator" ) {
      printf("      conversion:subject_discriminator \"%s\";\n",subjectDiscriminator)

   }else if( $0~"conversion:dataset_version" ) {                          # TODO: DEPRECATED (but converter not recognizing it)
      printf("   conversion:dataset_version    \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:version_identifier" ) {
      printf("   conversion:version_identifier \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:dataset_identifier" && length(dataset_identifier) > 0) {
      printf("   conversion:dataset_identifier \"%s\";\n",dataset_identifier)

   }else {
      print
   }
}


A script to convert regexps to reperf format test cases, one per line.