added name_template to bundle template

added name_template to bundle template

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
#
#   -v surrogate
#   -v sourceID
#   -v datasetID
#   -v datasetVersion
#   -v subjectDiscriminator
#   -v conversionID
#
#   -v cellDelimiter
#   -v header
#   -v dataStart
#   -v dataEnd
#
#   -v onlyIfCol
#   -v repeatAboveIfEmptyCol
#   -v interpretAsNull
#
#   -v whoami
#   -v machine_uri
#   -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix dbpediaprop:   <http://dbpedia.org/property/> ."
      print "@prefix dbpediaowl:    <http://dbpedia.org/ontology/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix bibo:          <http://purl.org/ontology/bibo/> ."
      print "@prefix prov:          <http://www.w3.org/ns/prov#> ."
      print "@prefix doap:          <http://usefulinc.com/ns/doap#> ."
      print "@prefix nfo:           <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#> ."
      print "@prefix sio:           <http://semanticscience.org/resource/> ."
      print "@prefix org:           <http://www.w3.org/ns/org#> ."
      print "@prefix vsr:           <http://purl.org/twc/vocab/vsr#> ."
      print "@prefix dcat:          <http://www.w3.org/ns/dcat#> ."
      print "@prefix qb:            <http://purl.org/linked-data/cube#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   conversion:name_template \"[/sd]company/[#2]/[r]\";\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
   if(!length(conversionID))        printf("   conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("   conversion:enhancement_identifier \"%s\";\n",conversionID);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }
                                    printf("\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   ov:csvRow 2;\n");
                                    printf("      #   a conversion:DataStartRow;\n");
                                    printf("      #];\n");
                                    printf("\n");
   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n");
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [\n",                        comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",           comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];\n",                                            comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:domain_template \"thing_[r]\";\n");
                                    printf("      #   conversion:domain_name     \"Thing\";\n");
                                    printf("      #];\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:class_name \"Thing\";\n");
                                    printf("      #   conversion:subclass_of <http://purl.org/...>;\n");
                                    printf("      #];\n");
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
#
#   -v surrogate
#   -v sourceID
#   -v datasetID
#   -v datasetVersion
#   -v subjectDiscriminator
#   -v conversionID
#
#   -v cellDelimiter
#   -v header
#   -v dataStart
#   -v dataEnd
#
#   -v onlyIfCol
#   -v repeatAboveIfEmptyCol
#   -v interpretAsNull
#
#   -v whoami
#   -v machine_uri
#   -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix dbpediaprop:   <http://dbpedia.org/property/> ."
      print "@prefix dbpediaowl:    <http://dbpedia.org/ontology/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix bibo:          <http://purl.org/ontology/bibo/> ."
      print "@prefix prov:          <http://www.w3.org/ns/prov#> ."
      print "@prefix doap:          <http://usefulinc.com/ns/doap#> ."
      print "@prefix nfo:           <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#> ."
      print "@prefix sio:           <http://semanticscience.org/resource/> ."
      print "@prefix org:           <http://www.w3.org/ns/org#> ."
      print "@prefix vsr:           <http://purl.org/twc/vocab/vsr#> ."
      print "@prefix dcat:          <http://www.w3.org/ns/dcat#> ."
      print "@prefix qb:            <http://purl.org/linked-data/cube#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
   if(!length(conversionID))        printf("   conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("   conversion:enhancement_identifier \"%s\";\n",conversionID);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }
                                    printf("\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   ov:csvRow 2;\n");
                                    printf("      #   a conversion:DataStartRow;\n");
                                    printf("      #];\n");
                                    printf("\n");
   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n");
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [\n",                        comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",           comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];\n",                                            comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:domain_template \"thing_[r]\";\n");
                                    printf("      #   conversion:domain_name     \"Thing\";\n");
                                    printf("      #];\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:class_name \"Thing\";\n");
                                    printf("      #   conversion:subclass_of <http://purl.org/...>;\n");
                                    printf("      #];\n");
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}


Fix extra header line inserted by MHz

Fix extra header line inserted by MHz

# Generate series Marquees, Descriptions, Headers from MHz series pages
#       (Headers include Genre, Country, Language, Rating)
# and return the list of Episode URLs for further processing
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/ \
#           | awk -v MARQUEE_FILE=$MARQUEE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v HEADER_FILE=$HEADER_FILE -v EPISODE_URL_FILE=$EPISODE_URL_FILE \
#           -v ERROR_FILE=$ERROR_FILE -f getMHzFrom-seriesPages.awk
#
# INPUT:
#       <title>A French Village - MHz Choice</title>
#       <meta name="description" content="DRAMA | FRANCE | FRENCH WITH ENGLISH SUBTITLES | TV-MA^M
#       This acclaimed drama is about the German...all in a day's work... its inhabitants.">
# ---
#       <select class="js-switch-season btn-dropdown-black margin-right-medium" data-switch-season >
#           <option value="https://watch.mhzchoice.com/a-french-village/season:1" selected>
#             Season 1
#           </option>
#           <option value="https://watch.mhzchoice.com/a-french-village/season:2">
#             Season 2
#           </option>
#       </select>
# ---
#       Show More
#
# OUTPUT:
#       $MARQUEE_FILE, $DESCRIPTION_FILE, $HEADER_FILE,
#       list of Episode URLs
#

/title>/ {
    seriesTitle = $0
    sub (/.*<title>/,"",seriesTitle)
    sub (/ - MHz Choice<.*/,"",seriesTitle)
    gsub (/'/,"'",seriesTitle)
    if (match (seriesTitle, /^The /)) {
        seriesTitle = substr(seriesTitle, 5) ", The"
    }
    print seriesTitle >> MARQUEE_FILE
    next
}

/meta name="description" content=/,/>/ {
    # if we're on the first line of this block ...
    if ($0 ~ /name="description" content="/) {
        headerPrinted = "no"
        sub (/.*name="description" content="/,"")
    }
    # If we find a header, clean it up and print it
    if ($0 ~ /\| TV/) {
        sub (/WITH ENGLISH SUBTITLES /,"")
        sub (/SCANDINAVIAN CRIME FICTION/,"Sweden")
        sub (/NONFICTION - DOCUMENTARY/,"Documentary")
        gsub (/ \| /,"|")
        sub (/\r/,"")
        # Split out header fields
        numFields = split ($0,fld,"\|")
        # lowercase everything but the first character in all fields but the last
        for (i = 1; i < numFields; ++i) {
            fld[i] = substr(fld[i],1,1) tolower(substr(fld[i],2))
            # uppercase the first character of any second word
            if (match (fld[i],/ /) > 0)
                fld[i] = substr(fld[i],1,RSTART) toupper(substr(fld[i],RSTART+1,1)) \
                (substr(fld[i],RSTART+2))
        }
        # print the finalized header
        if (numFields > 3 ) {
            print fld[1] "\t" fld[2] "\t" fld[(numFields-1)] "\t" fld[(numFields)] >> HEADER_FILE
            headerPrinted = "yes"
        }
    } else {
    # We found a description, clean it up and print it
    gsub (/  */," ")
    gsub (/'/,"'")
    gsub (/"/,"\"")
    gsub (/&/,"\\&")
    gsub (/‘/,"’")
    gsub (/’/,"’")
    gsub (/“/,"“")
    gsub (/”/,"”")
    gsub (/\r/," ")
    # if it's not the last line of the description, print it without a newline
    if ($0 !~ /"\>/) {
        printf ("%s", $0) >> DESCRIPTION_FILE
    } else {
    # if it's the last line of the description, print it with a newline
    sub (/"\>/,"")
    print $0 >> DESCRIPTION_FILE
    # if we didn't find a header in this block, print a blank one
    if (headerPrinted == "no") {
        print "==> No genre/country line: " seriesTitle >> ERROR_FILE
        print "\t\t\t" >> HEADER_FILE
    }
}
    }
}

# Return the list of Episode URLs for further processing
/<select class="js-switch-season/,/<\/select>/ {
    sub (/^ */,"")
    if ($0 ~ /^<option value="/) {
        split ($0,fld,"\"")
        print fld[2] >> EPISODE_URL_FILE
    }
}

# If there is more to load ...
/class="js-load-more-link/ {
    sub (/.*href="/,"https://watch.mhzchoice.com")
    sub (/">.*/,"")
    print $0 >> EPISODE_URL_FILE
}

# Generate series Marquees, Descriptions, Headers from MHz series pages
#       (Headers include Genre, Country, Language, Rating)
# and return the list of Episode URLs for further processing
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/ \
#           | awk -v MARQUEE_FILE=$MARQUEE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v HEADER_FILE=$HEADER_FILE -v EPISODE_URL_FILE=$EPISODE_URL_FILE \
#           -v ERROR_FILE=$ERROR_FILE -f getMHzFrom-seriesPages.awk
#
# INPUT:
#       <title>A French Village - MHz Choice</title>
#       <meta name="description" content="DRAMA | FRANCE | FRENCH WITH ENGLISH SUBTITLES | TV-MA^M
#       This acclaimed drama is about the German...all in a day's work... its inhabitants.">
# ---
#       <select class="js-switch-season btn-dropdown-black margin-right-medium" data-switch-season >
#           <option value="https://watch.mhzchoice.com/a-french-village/season:1" selected>
#             Season 1
#           </option>
#           <option value="https://watch.mhzchoice.com/a-french-village/season:2">
#             Season 2
#           </option>
#       </select>
# ---
#       Show More
#
# OUTPUT:
#       $MARQUEE_FILE, $DESCRIPTION_FILE, $HEADER_FILE,
#       list of Episode URLs
#

/title>/ {
    seriesTitle = $0
    sub (/.*<title>/,"",seriesTitle)
    sub (/ - MHz Choice<.*/,"",seriesTitle)
    gsub (/'/,"'",seriesTitle)
    if (match (seriesTitle, /^The /)) {
        seriesTitle = substr(seriesTitle, 5) ", The"
    }
    print seriesTitle >> MARQUEE_FILE
    next
}

/meta name="description" content=/,/>/ {
    # if we're on the first line of this block ...
    if ($0 ~ /name="description" content="/) {
        headerPrinted = "no"
        sub (/.*name="description" content="/,"")
    }
    # If we find a header, clean it up and print it
    if ($0 ~ /\| TV/) {
        sub (/WITH ENGLISH SUBTITLES /,"")
        sub (/SCANDINAVIAN CRIME FICTION/,"Sweden")
        sub (/NONFICTION - DOCUMENTARY/,"Documentary")
        gsub (/ \| /,"|")
        sub (/\r/,"")
        # Split out header fields
        numFields = split ($0,fld,"\|")
        # lowercase everything but the first character in all fields but the last
        for (i = 1; i < numFields; ++i) {
            fld[i] = substr(fld[i],1,1) tolower(substr(fld[i],2))
            # uppercase the first character of any second word
            if (match (fld[i],/ /) > 0)
                fld[i] = substr(fld[i],1,RSTART) toupper(substr(fld[i],RSTART+1,1)) \
                   (substr(fld[i],RSTART+2))
        }
        # print the finalized header
        print fld[1] "\t" fld[2] "\t" fld[(numFields-1)] "\t" fld[(numFields)] >> HEADER_FILE
        headerPrinted = "yes"
    } else {
        # We found a description, clean it up and print it
        gsub (/  */," ")
        gsub (/'/,"'")
        gsub (/"/,"\"")
        gsub (/&/,"\\&")
        gsub (/‘/,"’")
        gsub (/’/,"’")
        gsub (/“/,"“")
        gsub (/”/,"”")
        gsub (/\r/," ")
        # if it's not the last line of the description, print it without a newline
        if ($0 !~ /"\>/) {
            printf ("%s", $0) >> DESCRIPTION_FILE
        } else {
            # if it's the last line of the description, print it with a newline
            sub (/"\>/,"")
            print $0 >> DESCRIPTION_FILE
            # if we didn't find a header in this block, print a blank one
            if (headerPrinted == "no") {
                print "==> No genre/country line: " seriesTitle >> ERROR_FILE
                print "\t\t\t" >> HEADER_FILE
            }
        }
    }
}

# Return the list of Episode URLs for further processing
/<select class="js-switch-season/,/<\/select>/ {
    sub (/^ */,"")
    if ($0 ~ /^<option value="/) {
        split ($0,fld,"\"")
        print fld[2] >> EPISODE_URL_FILE
    }
}

# If there is more to load ...
/class="js-load-more-link/ {
    sub (/.*href="/,"https://watch.mhzchoice.com")
    sub (/">.*/,"")
    print $0 >> EPISODE_URL_FILE
}


-discontinuity mode

-discontinuity mode

BEGIN{
    if(!maxInterval)
        maxInterval=60
    if(!maxChunkDuration)
        maxChunkDuration=12000
    if(!minChunkDuration)
        minChunkDuration=1000
    state = "waitChunk"
    if(!verbose)
        verbose=1
}

function addStats(caption,data){
    cnts[caption] = cnts[caption] + 1
    sums[caption] = sums[caption] + data
    sums2[caption] = sums2[caption] + data*data
}

function printStats(caption){
    cnt = cnts[caption]
    if(cnt){
        sum = sums[caption]
        sum2 = sums2[caption]
        stddev = 0
        if((cnt > 1)){
         stddev = sqrt( (sum2*cnt -sum*sum) / (cnt*(cnt-1)))
        }
        printf "%s avg request interval: %.2f stddev: %.2f #: %d\n", caption,sum/cnt,stddev,cnt
    } else {
        print "couldn't find stats for "caption
    }
}

$0 ~ /\[debug\]/ && $0 ~ /GET \/hls\/(.*)\/playlist.json\/seg/ {
    split($10,b,"-");
    lastChunkPath=$10
    if(verbose){
        print $2 " "lastChunkPath;
    }

    if(length(b) == 6){
        lastChunkId = b[4]
    } else {
        lastChunkId =  b[2]
    }
    if(lastChunk && lastChunk+1 != lastChunkId){
        print $2" "lastChunkPath" error: expected chunk "lastChunk+1" got: "lastChunkId
        delete __lastDTS[0]
    }
    lastChunk=lastChunkId;
    split($2,a,":");
    time=a[3]+a[2]*60+a[1]*3600;
    if(lastTime){
        diff=time-lastTime
        if(diff>maxInterval){
            print $2" "lastChunkPath" error: requested chunk after "diff" sec."
        }
        if(diff>0){
            addStats("chunkInterval",diff)
        }
    }
    lastTime = time
    state = "checkDTS"
}

verbose && state == "checkDTS" && $0 ~ /parsed clip source/ {
    print $2" "$11" "$12" "$13
}

state == "checkDTS" && $0 ~ /first frame dts/ {
    state = "waitChunk"
    if( __lastDTS[0]){
        chunkDuration = $11 -  __lastDTS[0]

        addStats("chunkDuration",chunkDuration)
        if(chunkDuration > maxChunkDuration || chunkDuration < minChunkDuration){
            print $2" "lastChunkPath" bad chunk duration "chunkDuration
        }
    }
     __lastDTS[0] = $11
}

$0 ~ /media_set_parse_json: produced segment/ {
    tmp1 = $9
    tmp2=$11
    tmp3=$13
    if(segment_index){
        if(segment_index == tmp1 - 1){
            if(segment_start_time+segment_duration != tmp2){
                printf (" chunk %d adjucent chunks  differ: diff: %d prev={ %d  %d}  cur={%d %d}",tmp1,
                (tmp2-segment_start_time+segment_duration),
                segment_start_time,
                segment_duration,
                tmp2,
                tmp3)
            }
        } else {
            print "segment discontinuity: cur: "tmp1" prev: "segment_index
        }
    }
    segment_index = tmp1
    segment_start_time=tmp2
    segment_duration=tmp3
}

END {
    printStats("chunkInterval")
    printStats("chunkDuration")
}

BEGIN{
    if(!maxInterval)
        maxInterval=60
    if(!maxChunkDuration)
        maxChunkDuration=12000
    if(!minChunkDuration)
        minChunkDuration=1000
    state = "waitChunk"
    if(!verbose)
        verbose=1
}

function addStats(caption,data){
    cnts[caption] = cnts[caption] + 1
    sums[caption] = sums[caption] + data
    sums2[caption] = sums2[caption] + data*data
}

function printStats(caption){
    cnt = cnts[caption]
    if(cnt){
        sum = sums[caption]
        sum2 = sums2[caption]
        stddev = 0
        if((cnt > 1)){
         stddev = sqrt( (sum2*cnt -sum*sum) / (cnt*(cnt-1)))
        }
        printf "%s avg request interval: %.2f stddev: %.2f #: %d\n", caption,sum/cnt,stddev,cnt
    } else {
        print "couldn't find stats for "caption
    }
}

$0 ~ /\[debug\]/ && $0 ~ /GET \/hls\/(.*)\/playlist.json\/seg/ {
    split($10,b,"-");
    lastChunkPath=$10
    if(verbose){
        print $2 " "lastChunkPath;
    }
    if(lastChunk && lastChunk+1 != b[2]){
        print $2" "lastChunkPath" error: expected chunk "lastChunk+1" got: "b[2]
        delete __lastDTS[0]
    }
    lastChunk=b[2];
    split($2,a,":");
    time=a[3]+a[2]*60+a[1]*3600;
    if(lastTime){
        diff=time-lastTime
        if(diff>maxInterval){
            print $2" "lastChunkPath" error: requested chunk after "diff" sec."
        }
        if(diff>0){
            addStats("chunkInterval",diff)
        }
    }
    lastTime = time
    state = "checkDTS"
}

verbose && state == "checkDTS" && $0 ~ /parsed clip source/ {
    print $2" "$11" "$12" "$13
}

state == "checkDTS" && $0 ~ /first frame dts/ {
    state = "waitChunk"
    if( __lastDTS[0]){
        chunkDuration = $11 -  __lastDTS[0]

        addStats("chunkDuration",chunkDuration)
        if(chunkDuration > maxChunkDuration || chunkDuration < minChunkDuration){
            print $2" "lastChunkPath" bad chunk duration "chunkDuration
        }
    }
     __lastDTS[0] = $11
}

END {
    printStats("chunkInterval")
    printStats("chunkDuration")
}