Correction.

Correction.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@33852 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[tester],status,year

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either status is _not_ manifest or ending year is gt or eq to contract year
  current_year = 2013
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "American R")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "American S") && (substr(p[pn],1,2) <= "Bz")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Rz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,2) >= "SA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[tester],status,year

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either status is _not_ manifest or ending year is gt or eq to contract year
  current_year = 2013
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "American R")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "American S") && (substr(p[pn],1,2) <= "Bz")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Rz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,1) >= "SA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}


Fixed tagXSstrandedData.awk script.

Fixed tagXSstrandedData.awk script.

BEGIN {
    OFS="\t";
    strSym[0]="+";
    strSym[1]="-";
}

{

    if (substr($1,1,1)=="@" || $4==0)
    {# header, or unmapped read - just print
        print;
        next;
    };

    str=and($2,0x10)/0x10;

    if (and($2,0x1)==0)
    {# single end defaults to mate
        mate=1;
    } else
    {
        mate=and($2,0x40)/0x40+2*and($2,0x80)/0x80;
    };

    if (mate>0 && mate <3)
    {# mate is defined - add XS tag
       if (mate!=strType) str=1-str; #revert strand if the mate is opposite
       print $0 "\t" "XS:A:" strSym[str];
    } else
    {# mate is not defined - just print
       print;
    };
}

BEGIN {
    OFS="\t";
    strSym[0]="+";
    strSym[1]="-";
}

{

    printf $0;

    if (substr($1,1,1)=="@")
    {# header - nothing to do
        printf "\n";
        next;
    };

    str=and($2,0x10)/0x10;

    if (and($2,0x1)==0)
    {# single end defaults to mate
        mate=1;
    } else
    {
        mate=and($2,0x40)/0x40+2*and($2,0x80)/0x80;
    };

    if (mate>0 && mate <3)
    {# mate is defined
       if (mate!=strType) str=1-str; #revert strand if the mate is opposite
       printf "\t" "XS:A:" strSym[str];
    };

    printf "\n";

}


Now toloka.awk can process multiple inputs

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
    print "INPUT:hyponym", "INPUT:hypernym", "INPUT:genitive", "GOLDEN:judgement", "HINT:text";
}
FNR == 1 {
    for (i=1; i<=NF; i++) ix[$i] = i;
    next;
}
{
    print $ix["hyponym"], $ix["hypernym"], $ix["genitive"], "", "" | "sort -t \"\t\" -k1,1 -k2,2 -k3,3 -u";
}

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
    print "INPUT:hyponym", "INPUT:hypernym", "INPUT:genitive", "GOLDEN:judgement", "HINT:text";
}
NR == 1 {
    for (i=1; i<=NF; i++) ix[$i] = i;
    next;
}
$ix["found"] {
    print $ix["hyponym"], $ix["hypernym"], $ix["genitive"], "", "" | "sort -t \"\t\" -k1,1 -k2,2 -k3,3 -u";
}


fix copyright headers

fix copyright headers

#
# /**********************************************************
# *      This software is part of the graphviz package      *
# *                http://www.graphviz.org/                 *
# *                                                         *
# *            Copyright (c) 1994-2005 AT&T Corp.           *
# *                and is licensed under the                *
# *            Common Public License, Version 1.0           *
# *                      by AT&T Corp.                      *
# *                                                         *
# *        Information and Software Systems Research        *
# *              AT&T Research, Florham Park NJ             *
# **********************************************************/

BEGIN	{
			n_names = 0;
			printf("strict digraph typeref {\n");
			printf("	node [shape=record];\n");
		}

(NF == 1) {		# starts a new node in the node file
			Name[n_names++] = name = $1;
			n_fields = 0;
		}

(NF == 2) {		# enters a field in the node file
			if (($2 == "v") || (PROCS && ($2 == "p"))) {
				Field[name,n_fields] = $2 ": " $1;
				Field[name,$1] = n_fields;
				n_fields++;
			}
		}

(NF == 6) {		# an entry in the edge file. example:
				# v BinaryNode left t - Tree
			kind = $1;
			if ((kind == "v") || (PROCS && (kind == "p"))) {
				if (($2 != "-") && ($2 != $3))
					from = "\"" $2 "\".f0_1_" Field[$2,$3];
				else from = "\"" $3 "\"";

				if (($5 != "-") && ($5 != $6))
					to = "\"" $5 "\".f0_1_" Field[$5,$6];
				else to = "\"" $6 "\"";

				printf("\t%s -> %s;\n",from, to);
			}
		}

END 	{
			for (i in Name) {
				name = Name[i];
				printf("\t\"%s\" [label=\"{%s{",name,name);
				for (j = 0; Field[name,j] != ""; j++) {
					if (j > 0) printf("|");
					printf("%s",Field[name,j]);
				}
				printf("}}\"];\n");
			}
			printf("}\n");
		}

# Copyright (c) AT&T Corp. 1994, 1995.
# This code is licensed by AT&T Corp.  For the
# terms and conditions of the license, see
# http://www.research.att.com/orgs/ssr/book/reuse

BEGIN	{
			n_names = 0;
			printf("strict digraph typeref {\n");
			printf("	node [shape=record];\n");
		}

(NF == 1) {		# starts a new node in the node file
			Name[n_names++] = name = $1;
			n_fields = 0;
		}

(NF == 2) {		# enters a field in the node file
			if (($2 == "v") || (PROCS && ($2 == "p"))) {
				Field[name,n_fields] = $2 ": " $1;
				Field[name,$1] = n_fields;
				n_fields++;
			}
		}

(NF == 6) {		# an entry in the edge file. example:
				# v BinaryNode left t - Tree
			kind = $1;
			if ((kind == "v") || (PROCS && (kind == "p"))) {
				if (($2 != "-") && ($2 != $3))
					from = "\"" $2 "\".f0_1_" Field[$2,$3];
				else from = "\"" $3 "\"";

				if (($5 != "-") && ($5 != $6))
					to = "\"" $5 "\".f0_1_" Field[$5,$6];
				else to = "\"" $6 "\"";

				printf("\t%s -> %s;\n",from, to);
			}
		}

END 	{
			for (i in Name) {
				name = Name[i];
				printf("\t\"%s\" [label=\"{%s{",name,name);
				for (j = 0; Field[name,j] != ""; j++) {
					if (j > 0) printf("|");
					printf("%s",Field[name,j]);
				}
				printf("}}\"];\n");
			}
			printf("}\n");
		}


Missing newline in error message in summarize.awk

#**************************************************************************
#*                                                                        *
#*                                 OCaml                                  *
#*                                                                        *
#*         Damien Doligez, projet Gallium, INRIA Rocquencourt             *
#*                                                                        *
#*   Copyright 2013 Institut National de Recherche en Informatique et     *
#*     en Automatique.                                                    *
#*                                                                        *
#*   All rights reserved.  This file is distributed under the terms of    *
#*   the GNU Lesser General Public License version 2.1, with the          *
#*   special exception on linking described in the file LICENSE.          *
#*                                                                        *
#**************************************************************************

function check() {
    if (!in_test){
        printf("error at line %d: found test result without test start\n", NR);
        errored = 1;
    }
}

function clear() {
    curfile = "";
    in_test = 0;
}

function record_pass() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "p";
    delete SKIPPED[curdir];
    clear();
}

function record_skip() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "s";
    if (curdir in SKIPPED) SKIPPED[curdir] = 1;
    clear();
}

# The output cares only if the test passes at least once so if a test passes,
# but then fails in a re-run triggered by a different test, ignore it.
function record_fail() {
    check();
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "f";
    }
    delete SKIPPED[curdir];
    clear();
}

function record_unexp() {
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "e";
    }
    delete SKIPPED[curdir];
    clear();
}

/Running tests from '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /Running tests from '[^']*'/);
    curdir = substr($0, RSTART+20, RLENGTH - 21);
    # Use SKIPPED[curdir] as a sentinel to detect no output
    SKIPPED[curdir] = 0;
    key = curdir;
    DIRS[key] = key;
    curfile = "";
}

/ ... testing.* ... testing/ {
    printf("error at line %d: found two test results on the same line\n", NR);
    errored = 1;
}

/^ ... testing '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /... testing '[^']*'/);
    curfile = substr($0, RSTART+13, RLENGTH-14);
    if (match($0, /... testing '[^']*' with [^:=]*/)){
        curfile = substr($0, RSTART+12, RLENGTH-12);
    }
    key = sprintf ("%s/%s", curdir, curfile);
    DIRS[key] = curdir;
    in_test = 1;
}

/^ ... testing (with|[^'])/ {
    if (in_test) record_unexp();
    key = curdir;
    DIRS[key] = curdir;
    in_test = 1;
}

/=> passed/ {
    record_pass();
}

/=> skipped/ {
    record_skip();
}

/=> failed/ {
    record_fail();
}

/=> unexpected error/ {
    record_unexp();
}

/^re-ran / {
    if (in_test){
        printf("error at line %d: found re-ran inside a test\n", NR);
        errored = 1;
    }else{
        RERAN[substr($0, 8, length($0)-7)] += 1;
        ++ reran;
    }
}

END {
    if (errored){
        printf ("\n#### Some fatal error occurred during testing.\n\n");
        exit (3);
    }else{
        if (!retries){
            for (key in SKIPPED){
                if (!SKIPPED[key]){
                    ++ empty;
                    blanks[emptyidx++] = key;
                    delete SKIPPED[key];
                }
            }
            for (key in RESULTS){
                r = RESULTS[key];
                if (r == "p"){
                    ++ passed;
                }else if (r == "f"){
                    ++ failed;
                    fail[failidx++] = key;
                }else if (r == "e"){
                    ++ unexped;
                    unexp[unexpidx++] = key;
                }else if (r == "s"){
                    ++ skipped;
                    curdir = DIRS[key];
                    if (curdir in SKIPPED){
                        if (SKIPPED[curdir]){
                            SKIPPED[curdir] = 0;
                            skips[skipidx++] = curdir;
                        }
                    }else{
                        skips[skipidx++] = key;
                    }
                }
            }
            printf("\n");
            printf("Summary:\n");
            printf("  %3d tests passed\n", passed);
            printf("  %3d tests skipped\n", skipped);
            printf("  %3d tests failed\n", failed);
            printf("  %3d unexpected errors\n", unexped);
            printf("  %3d tests considered", nresults);
            if (nresults == passed + skipped + failed + unexped){
                printf ("\n");
            }else{
                printf (" (totals don't add up??)\n");
            }
            if (reran != 0){
                printf("  %3d test dir re-runs\n", reran);
            }
            if (failed != 0){
                printf("\nList of failed tests:\n");
                for (i=0; i < failed; i++) printf("    %s\n", fail[i]);
            }
            if (unexped != 0){
                printf("\nList of unexpected errors:\n");
                for (i=0; i < unexped; i++) printf("    %s\n", unexp[i]);
            }
            if (skipped != 0){
                printf("\nList of skipped tests:\n");
                for (i=0; i < skipidx; i++) printf("    %s\n", skips[i]);
            }
            if (empty != 0){
                printf("\nList of directories returning no results:\n");
                for (i=0; i < empty; i++) printf("    %s\n", blanks[i]);
            }
            printf("\n");
            if (failed || unexped){
                printf("#### Something failed. Exiting with error status.\n\n");
                exit 4;
            }
        }else{
            for (key in RESULTS){
                if (RESULTS[key] == "f" || RESULTS[key] == "e"){
                    key = DIRS[key];
                    if (!(key in RERUNS)){
                        RERUNS[key] = 1;
                        if (RERAN[key] < max_retries){
                            printf("%s\n", key);
                        }
                    }
                }
            }
        }
    }
}

#**************************************************************************
#*                                                                        *
#*                                 OCaml                                  *
#*                                                                        *
#*         Damien Doligez, projet Gallium, INRIA Rocquencourt             *
#*                                                                        *
#*   Copyright 2013 Institut National de Recherche en Informatique et     *
#*     en Automatique.                                                    *
#*                                                                        *
#*   All rights reserved.  This file is distributed under the terms of    *
#*   the GNU Lesser General Public License version 2.1, with the          *
#*   special exception on linking described in the file LICENSE.          *
#*                                                                        *
#**************************************************************************

function check() {
    if (!in_test){
        printf("error at line %d: found test result without test start\n", NR);
        errored = 1;
    }
}

function clear() {
    curfile = "";
    in_test = 0;
}

function record_pass() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "p";
    delete SKIPPED[curdir];
    clear();
}

function record_skip() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "s";
    if (curdir in SKIPPED) SKIPPED[curdir] = 1;
    clear();
}

# The output cares only if the test passes at least once so if a test passes,
# but then fails in a re-run triggered by a different test, ignore it.
function record_fail() {
    check();
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "f";
    }
    delete SKIPPED[curdir];
    clear();
}

function record_unexp() {
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "e";
    }
    delete SKIPPED[curdir];
    clear();
}

/Running tests from '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /Running tests from '[^']*'/);
    curdir = substr($0, RSTART+20, RLENGTH - 21);
    # Use SKIPPED[curdir] as a sentinel to detect no output
    SKIPPED[curdir] = 0;
    key = curdir;
    DIRS[key] = key;
    curfile = "";
}

/ ... testing.* ... testing/ {
    printf("error at line %d: found two test results on the same line\n", NR);
    errored = 1;
}

/^ ... testing '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /... testing '[^']*'/);
    curfile = substr($0, RSTART+13, RLENGTH-14);
    if (match($0, /... testing '[^']*' with [^:=]*/)){
        curfile = substr($0, RSTART+12, RLENGTH-12);
    }
    key = sprintf ("%s/%s", curdir, curfile);
    DIRS[key] = curdir;
    in_test = 1;
}

/^ ... testing (with|[^'])/ {
    if (in_test) record_unexp();
    key = curdir;
    DIRS[key] = curdir;
    in_test = 1;
}

/=> passed/ {
    record_pass();
}

/=> skipped/ {
    record_skip();
}

/=> failed/ {
    record_fail();
}

/=> unexpected error/ {
    record_unexp();
}

/^re-ran / {
    if (in_test){
        printf("error at line %d: found re-ran inside a test\n", NR);
        errored = 1;
    }else{
        RERAN[substr($0, 8, length($0)-7)] += 1;
        ++ reran;
    }
}

END {
    if (errored){
        printf ("\n#### Some fatal error occurred during testing.\n\n");
        exit (3);
    }else{
        if (!retries){
            for (key in SKIPPED){
                if (!SKIPPED[key]){
                    ++ empty;
                    blanks[emptyidx++] = key;
                    delete SKIPPED[key];
                }
            }
            for (key in RESULTS){
                r = RESULTS[key];
                if (r == "p"){
                    ++ passed;
                }else if (r == "f"){
                    ++ failed;
                    fail[failidx++] = key;
                }else if (r == "e"){
                    ++ unexped;
                    unexp[unexpidx++] = key;
                }else if (r == "s"){
                    ++ skipped;
                    curdir = DIRS[key];
                    if (curdir in SKIPPED){
                        if (SKIPPED[curdir]){
                            SKIPPED[curdir] = 0;
                            skips[skipidx++] = curdir;
                        }
                    }else{
                        skips[skipidx++] = key;
                    }
                }
            }
            printf("\n");
            printf("Summary:\n");
            printf("  %3d tests passed\n", passed);
            printf("  %3d tests skipped\n", skipped);
            printf("  %3d tests failed\n", failed);
            printf("  %3d unexpected errors\n", unexped);
            printf("  %3d tests considered", nresults);
            if (nresults == passed + skipped + failed + unexped){
                printf ("\n");
            }else{
                printf (" (totals don't add up??)");
            }
            if (reran != 0){
                printf("  %3d test dir re-runs\n", reran);
            }
            if (failed != 0){
                printf("\nList of failed tests:\n");
                for (i=0; i < failed; i++) printf("    %s\n", fail[i]);
            }
            if (unexped != 0){
                printf("\nList of unexpected errors:\n");
                for (i=0; i < unexped; i++) printf("    %s\n", unexp[i]);
            }
            if (skipped != 0){
                printf("\nList of skipped tests:\n");
                for (i=0; i < skipidx; i++) printf("    %s\n", skips[i]);
            }
            if (empty != 0){
                printf("\nList of directories returning no results:\n");
                for (i=0; i < empty; i++) printf("    %s\n", blanks[i]);
            }
            printf("\n");
            if (failed || unexped){
                printf("#### Something failed. Exiting with error status.\n\n");
                exit 4;
            }
        }else{
            for (key in RESULTS){
                if (RESULTS[key] == "f" || RESULTS[key] == "e"){
                    key = DIRS[key];
                    if (!(key in RERUNS)){
                        RERUNS[key] = 1;
                        if (RERAN[key] < max_retries){
                            printf("%s\n", key);
                        }
                    }
                }
            }
        }
    }
}


Changed ini adt to use : instead of SUBSEP

# awk library to parse config file

BEGIN {INI_SEP = ":"}
# converts string containing rules into dict
# note, does not clear out dest array
# string must be an ini file fragment
function ini_parsefrag(frag, ini, curkey,   type,l, lines, n, sep, payload, key) {

	if (frag == "") return

	type = ini_type(frag)

	if (type == "STR") {
		if (curkey="") die("no key specified for STR frag")
		ini[curkey] = ini_val(frag)
	} else if (type == "HASH") {

		split(ini_val(frag), lines, "\n")

		for (l in lines) {
			if (lines[l] == "") continue

			n = index(lines[l], "=")
			if (n <= 0) die(sprintf("no = in hash: %s", frag))

			payload = substr(lines[l], n+1)
			key = substr(lines[l], 1, n-1)

			# unescape newlines
			gsub("@n", "\n", payload)
			gsub("@@", "@", payload)
			ini[key] = payload
		}
	}
}

# converts array to a rule string. NOT an ini.
function ini_arr2frag(array,     frag, key, val, line) {
	frag = "HASH" INI_SEP
	for (key in array) {
		# escape newlines using @ and \n values
		val = array[key]
		gsub("@", "@@", val)
		gsub("\n", "@n", val)

		line = key "=" "STR" INI_SEP val "\n"
		frag = frag line
	}
	# if we have added at least one line, we need to chop off the end
	if (line != "") {
		return substr(frag, 1,length(frag)-1)
	} else {
		return frag
	}
}

# converts ini to a rule string. NOT a raw array
function ini_ini2frag(array,     frag, key, val, line) {
	frag = "HASH" INI_SEP
	for (key in array) {
		# escape newlines using @ and \n values
		val = array[key]
		gsub("@", "@@", val)
		gsub("\n", "@n", val)

		line = key "=" val "\n"
		frag = frag line
	}

	# if we have added at least one line, we need to chop off the end
	if (line != "") {
		return substr(frag, 1,length(frag)-1)
	} else {
		return frag
	}
}

# returns string representation of ini file
function ini_print(ini, name,   key, hashes, tmp_ini, retval) {
	if (name != "") retval = sprintf("[%s]\n", name)

	for (key in ini) {
		type = ini_type(ini[key])
		if (type == "STR") {
			retval = retval sprintf("%s=%s\n", key, ini_val(ini[key]))
		} else if (type == "HASH") {
			# save hashes for later
			hashes[key]=1
		} else {die(sprintf("Unknown type %s", type))}
	}

	for (key in hashes) {
		split("",tmp_ini)
		ini_parsefrag(ini[key],tmp_ini)
		if (name != "")
			retval = retval ini_print(tmp_ini, name "." key)
		else
			retval = retval ini_print(tmp_ini, key)
	}

	return retval
}

# format of a fragment: TYPE INI_SEP VALUE

# returns the type of a fragment
function ini_type(frag,   i) {
	i = index(frag, INI_SEP)
	if (i == 0) die(sprintf("Expected INI_SEP in fragment: %s", frag))
	return substr(frag, 1, i-1)
}

# returns the value of a fragment
function ini_val(frag,   i) {
	i = index(frag, INI_SEP)
	if (i == 0) die("Expected INI_SEP in fragment")
	return substr(frag, i+1)
}

# gets value of key as number, stored in array as string
function ini_num(ini, key) {
	return ini_str(ini, key)+0
}

# gets value of a key, must be a str
function ini_str(ini, key) {
	if (ini[key] == "")
		return ""

	if (ini_type(ini[key]) != "STR") die(sprintf("Expected STR key in ini: %s", ini[key]))

	return ini_val(ini[key])
}

# get value of hash as array
function ini_hash(ini, key, dest) {

	if (ini[key] == "")
		return

	if (ini_type(ini[key]) != "HASH") die(sprintf("Expected HASH key in ini: %s", ini[key]))

	ini_parsefrag(ini[key], dest)
}

function ini_add_str(ini, key, val) {
	if (key == "") die("cannot have empty key")
	ini[key] = "STR" INI_SEP val
}

# gets the rules that apply to a specific path
# this searches the adt for all rules that apply to this block,
# with more specific rules overriding more general ones
# returns in rules array
function get_rules(adt, path, rules,    path_elems, n, i, key) {
	split("", rules)

	path = add_curdir(path)

	n = split(path, path_elems, "/")
	for (i=1; i <= n; i++) {
		key = join(path_elems, 1, i, "/")
		if (key in adt)
			ini_parsefrag(adt[key], rules)
	}

	# add any rules from the frontmatter
	add_frontmatter(rules, path)
}

function has_frontmatter(filename) {
	return ! system(sprintf("head -c 3 %s 2> /dev/null | grep '+++' > /dev/null 2>&1", filename))
}

function add_frontmatter(rules, filename,    keys, n) {

	# check to see if the file has frontmatter
	# frontmatter has magic value +++ as first three bytes of file
	# if it doesn't have this, we can return
	if (! has_frontmatter(filename)) {
		return
	}

	# if we are here, the file has frontmatter, so we need to parse it.
	if ((getline < filename) <= 0 || $0 !~ /^\+\+\+[[:blank:]]*$/) {
		die(sprintf("Invalid frontmatter in %s: %s", filename, $0))
	}

	while (1) {
		if ((getline < filename) <= 0) {
			die(sprintf("Unterminated +++ in frontmatter in %s", filename))
		}
		text = text $0 "\n"
		if ($0 ~ /^\+\+\+[[:blank:]]*$/) {
			break
		}

		n = index($0,"=")
		keys[substr($0,1,n-1)] = substr($0, n+1)
	}
	close(filename)

	ini_parsefrag(ini_arr2frag(keys), rules)

	# return number of lines in frontmatter
	return gsub("\\n", "", text) + 2
}

# each time we see a new file, store as part of rule name
FNR == 1 {

	# get the directory of the current file
	len_file_rule = split(FILENAME,file_rule,"/")
	delete file_rule[len_file_rule]
	len_file_rule--
	file_rule_str = join(file_rule, 1, len_file_rule, "/")
	cur_rule_str = file_rule_str

	# special case: we get passed in the default rules.
	# we set these rules in the global scope
	if (FILENAME == (puf_lib "/puffin.ini")) {
		file_rule_str = "."
		cur_rule_str = "."
	}
}

# if we are in find_sec look for things that match brackets

# remove comment lines
/^[:space:]*#/ {next}

#change rule based on new section encountered
match($0, /\[.*\]/) {
	rulename = substr($0, RSTART+1, RLENGTH-2)
	cur_rule_str = file_rule_str "/" rulename
}

# if we detect a setting of a key value
/.*=.*/ {
	n = index($0, "=")
	key = substr($0, 1, n-1)
	val = substr($0, n+1)
	if (cur_rule_str in adt)
		adt[cur_rule_str] = adt[cur_rule_str] "\n" key "=STR" INI_SEP val
	else
		adt[cur_rule_str] = "HASH" INI_SEP key"=STR" INI_SEP val
}

# awk library to parse config file

# converts string containing rules into dict
# note, does not clear out dest array
# string must be an ini file fragment
function ini_parsefrag(frag, ini, curkey,   type,l, lines, n, sep, payload, key) {

	if (frag == "") return

	type = ini_type(frag)

	if (type == "STR") {
		if (curkey="") die("no key specified for STR frag")
		ini[curkey] = ini_val(frag)
	} else if (type == "HASH") {

		split(ini_val(frag), lines, "\n")

		for (l in lines) {
			if (lines[l] == "") continue

			n = index(lines[l], "=")
			if (n <= 0) die(sprintf("no = in hash: %s", frag))

			payload = substr(lines[l], n+1)
			key = substr(lines[l], 1, n-1)

			# unescape newlines
			gsub("@n", "\n", payload)
			gsub("@@", "@", payload)
			ini[key] = payload
		}
	}
}

# converts array to a rule string. NOT an ini.
function ini_arr2frag(array,     frag, key, val, line) {
	frag = "HASH" SUBSEP
	for (key in array) {
		# escape newlines using @ and \n values
		val = array[key]
		gsub("@", "@@", val)
		gsub("\n", "@n", val)

		line = key "=" "STR" SUBSEP val "\n"
		frag = frag line
	}
	# if we have added at least one line, we need to chop off the end
	if (line != "") {
		return substr(frag, 1,length(frag)-1)
	} else {
		return frag
	}
}

# converts ini to a rule string. NOT a raw array
function ini_ini2frag(array,     frag, key, val, line) {
	frag = "HASH" SUBSEP
	for (key in array) {
		# escape newlines using @ and \n values
		val = array[key]
		gsub("@", "@@", val)
		gsub("\n", "@n", val)

		line = key "=" val "\n"
		frag = frag line
	}

	# if we have added at least one line, we need to chop off the end
	if (line != "") {
		return substr(frag, 1,length(frag)-1)
	} else {
		return frag
	}
}

# returns string representation of ini file
function ini_print(ini, name,   key, hashes, tmp_ini, retval) {
	if (name != "") retval = sprintf("[%s]\n", name)

	for (key in ini) {
		type = ini_type(ini[key])
		if (type == "STR") {
			retval = retval sprintf("%s=%s\n", key, ini_val(ini[key]))
		} else if (type == "HASH") {
			# save hashes for later
			hashes[key]=1
		} else {die(sprintf("Unknown type %s", type))}
	}

	for (key in hashes) {
		split("",tmp_ini)
		ini_parsefrag(ini[key],tmp_ini)
		if (name != "")
			retval = retval ini_print(tmp_ini, name "." key)
		else
			retval = retval ini_print(tmp_ini, key)
	}

	return retval
}

# format of a fragment: TYPE SUBSEP VALUE

# returns the type of a fragment
function ini_type(frag,   i) {
	i = index(frag, SUBSEP)
	if (i == 0) die(sprintf("Expected SUBSEP in fragment: %s", frag))
	return substr(frag, 1, i-1)
}

# returns the value of a fragment
function ini_val(frag,   i) {
	i = index(frag, SUBSEP)
	if (i == 0) die("Expected SUBSEP in fragment")
	return substr(frag, i+1)
}

# gets value of key as number, stored in array as string
function ini_num(ini, key) {
	return ini_str(ini, key)+0
}

# gets value of a key, must be a str
function ini_str(ini, key) {
	if (ini[key] == "")
		return ""

	if (ini_type(ini[key]) != "STR") die(sprintf("Expected STR key in ini: %s", ini[key]))

	return ini_val(ini[key])
}

# get value of hash as array
function ini_hash(ini, key, dest) {

	if (ini[key] == "")
		return

	if (ini_type(ini[key]) != "HASH") die(sprintf("Expected HASH key in ini: %s", ini[key]))

	ini_parsefrag(ini[key], dest)
}

function ini_add_str(ini, key, val) {
	if (key == "") die("cannot have empty key")
	ini[key] = "STR" SUBSEP val
}

# gets the rules that apply to a specific path
# this searches the adt for all rules that apply to this block,
# with more specific rules overriding more general ones
# returns in rules array
function get_rules(adt, path, rules,    path_elems, n, i, key) {
	split("", rules)

	path = add_curdir(path)

	n = split(path, path_elems, "/")
	for (i=1; i <= n; i++) {
		key = join(path_elems, 1, i, "/")
		if (key in adt)
			ini_parsefrag(adt[key], rules)
	}

	# add any rules from the frontmatter
	add_frontmatter(rules, path)
}

function has_frontmatter(filename) {
	return ! system(sprintf("head -c 3 %s 2> /dev/null | grep '+++' > /dev/null 2>&1", filename))
}

function add_frontmatter(rules, filename,    keys, n) {

	# check to see if the file has frontmatter
	# frontmatter has magic value +++ as first three bytes of file
	# if it doesn't have this, we can return
	if (! has_frontmatter(filename)) {
		return
	}

	# if we are here, the file has frontmatter, so we need to parse it.
	if ((getline < filename) <= 0 || $0 !~ /^\+\+\+[[:blank:]]*$/) {
		die(sprintf("Invalid frontmatter in %s: %s", filename, $0))
	}

	while (1) {
		if ((getline < filename) <= 0) {
			die(sprintf("Unterminated +++ in frontmatter in %s", filename))
		}
		text = text $0 "\n"
		if ($0 ~ /^\+\+\+[[:blank:]]*$/) {
			break
		}

		n = index($0,"=")
		keys[substr($0,1,n-1)] = substr($0, n+1)
	}
	close(filename)

	ini_parsefrag(ini_arr2frag(keys), rules)

	# return number of lines in frontmatter
	return gsub("\\n", "", text) + 2
}

# each time we see a new file, store as part of rule name
FNR == 1 {

	# get the directory of the current file
	len_file_rule = split(FILENAME,file_rule,"/")
	delete file_rule[len_file_rule]
	len_file_rule--
	file_rule_str = join(file_rule, 1, len_file_rule, "/")
	cur_rule_str = file_rule_str

	# special case: we get passed in the default rules.
	# we set these rules in the global scope
	if (FILENAME == (puf_lib "/puffin.ini")) {
		file_rule_str = "."
		cur_rule_str = "."
	}
}

# if we are in find_sec look for things that match brackets

# remove comment lines
/^[:space:]*#/ {next}

#change rule based on new section encountered
match($0, /\[.*\]/) {
	rulename = substr($0, RSTART+1, RLENGTH-2)
	cur_rule_str = file_rule_str "/" rulename
}

# if we detect a setting of a key value
/.*=.*/ {
	n = index($0, "=")
	key = substr($0, 1, n-1)
	val = substr($0, n+1)
	if (cur_rule_str in adt)
		adt[cur_rule_str] = adt[cur_rule_str] "\n" key "=STR" SUBSEP val
	else
		adt[cur_rule_str] = "HASH" SUBSEP key"=STR" SUBSEP val
}


rnc2rng.awk: Better handling of data types and params

#!/bin/awk
# -*- indent-tabs-mode: nil; c-basic-offset: 2 -*-
# rnc2rng.awk - Convert RELAX NG Compact Syntax to XML Syntax
# Copyright (C) 2007 Shaun McCance <shaunm@gnome.org>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

# This program is free software, but that doesn't mean you should use it.
# I wanted to write and maintain the Mallard schema in RNC in code blocks
# within the specification, but I needed to distribute the schema in RNG.
# Since xmllint (libxml2) does not currently support RNC, and since I'm
# not willing to introduce more dependencies, awk was my only option.
#
# This awk script is *NOT* a complete implementation, and probably never
# will be.  Its sole purpose is to handle the Mallard schema, and it is
# sufficient for that task.
#
# You are, of course, free to use any of this.  If you do proliferate this
# hack, it is requested (though not required, that would be non-free) that
# you atone for your actions.  A good atonement would be contributing to
# free software.

function parse_define (line) {
  nameix = index(line, "=");
  combine = "";
  if (nameix < 1) next;
  if (substr(line, nameix - 1, 1) == "|") {
    combine = "choice";
    name = substr(line, 1, nameix - 2);
  }
  else if (substr(line, nameix - 1, 1) == "&") {
    combine = "interleave";
    name = substr(line, 1, nameix - 2);
  }
  else {
    name = substr(line, 1, nameix - 1);
  }
  sub(/^ */, "", name);
  sub(/ *$/, "", name);
  if (name == "")
    name = maybe_define;
  if (name == "start")
    define = "<start"
  else
    define = sprintf("<define name='%s'", name);
  if (combine != "")
    define = define " combine='" combine "'"
  define = define ">"
  stack[++stack_i] = define;
  mode = "pattern";
  maybe_define = "";
  if (length(line) >= nameix + 1)
    parse_pattern(substr(line, nameix + 1))
}
function parse_pattern (line) {
  sub(/^ */, "", line)
  if (length(line) == 0) return;
  c = substr(line, 1, 1);
  if (c == "(" || c == "{") {
    stack[++stack_i] = substr(line, 1, 1);
    paren[++paren_i] = stack_i;
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == ")" || c == "}") {
    open = stack[paren[paren_i]];
    oc = substr(open, 1, 1) c;
    if (oc != "()" && oc != "{}") {
      print "Mismatched parentheses on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }

    tag = "";
    if (length(open) > 1 && substr(open, 2, 1) == "&") {
      tag = "interleave";
    }
    else if (length(open) > 1 && substr(open, 2, 1) == "|") {
      tag = "choice";
    }
    else if (oc == "()") {
      tag = "group";
    }

    tmp = "";
    if (tag != "") {
      tmp = "<" tag ">";
    }
    for (i = paren[paren_i] + 1; i <= stack_i; i++) {
      tmp = tmp stack[i] "\n";
    }
    if (tag != "") {
      tmp = tmp "</" tag ">";
    }
    stack_i = paren[paren_i];
    stack[stack_i] = tmp;
    paren_i--;

    if (oc == "{}") {
      if (match(stack[stack_i - 1], "^<element")) {
        tmp = stack[stack_i - 1] "\n";
        if (stack[stack_i] != "") {
          tmp = tmp stack[stack_i] "\n";
        } else {
          tmp = tmp "<empty/>\n";
        }
        tmp = tmp "</element>";
        stack[--stack_i] = tmp;
      }
      else if (match(stack[stack_i - 1], "^<attribute")) {
        tmp = stack[stack_i - 1] "\n";
        if (stack[stack_i] != "") {
          tmp = tmp stack[stack_i] "\n";
        } else {
          tmp = tmp "<empty/>\n";
        }
        tmp = tmp "</attribute>";
        stack[--stack_i] = tmp;
      }
      else if (stack[stack_i - 1] == "<list>") {
        tmp = stack[stack_i - 1] "\n";
        tmp = tmp stack[stack_i] "\n";
        tmp = tmp "</list>";
        stack[--stack_i] = tmp;
      }
      else if (stack[stack_i - 1] == "<mixed>") {
        tmp = stack[stack_i - 1] "\n";
        tmp = tmp stack[stack_i] "\n";
        tmp = tmp "</mixed>";
        stack[--stack_i] = tmp;
      }
    }
    if (paren_i == 0) {
      mode = "grammar";
    }
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "|" || c == "&" || c == ",") {
    if (length(stack[paren[paren_i]]) == 1) {
      stack[paren[paren_i]] = stack[paren[paren_i]] c;
    }
    else if (length(stack[paren[paren_i]]) < 2 || substr(stack[paren[paren_i]], 2) != c) {
      if (paren_i == 0) {
        tmp = stack[stack_i];
        stack[stack_i] = c c;
        paren[++paren_i] = stack_i;
        stack[++stack_i] = tmp;
      }
      else {
        print "Mismatched infix operators on line " FNR | "cat 1>&2";
        error = 1;
        exit 1
      }
    }
    postop = 1;
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "?") {
    stack[stack_i] = "<optional>" stack[stack_i] "</optional>"
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "*") {
    stack[stack_i] = "<zeroOrMore>" stack[stack_i] "</zeroOrMore>"
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "+") {
    stack[stack_i] = "<oneOrMore>" stack[stack_i] "</oneOrMore>"
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "\"") {
    txt = substr(line, 2);
    sub(/".*/, "", txt)
    stack[++stack_i] = "<value>" txt "</value>";
    postop = 0;
    if (length(line) >= length(txt) + 3)
      parse_pattern(substr(line, length(txt) + 3));
  }
  else if (match(line, "^element ")) {
    aft = substr(line, 8);
    sub(/^ */, "", aft);
    name = aft;
    stack[++stack_i] = "<element";
    mode = "name_class";
    name_class_i = stack_i;
    postop = 0;
    parse_name_class(name);
  }
  else if (match(line, "^attribute ")) {
    aft = substr(line, 10);
    sub(/^ */, "", aft);
    name = aft;
    stack[++stack_i] = "<attribute";
    mode = "name_class";
    name_class_i = stack_i;
    postop = 0;
    parse_name_class(name);
  }
  else if (match(line, "^list ")) {
    aft = substr(line, 5);
    sub(/^ */, "", aft);
    stack[++stack_i] = "<list>";
    if (aft != "") { parse_pattern(aft); }
  }
  else if (match(line, "^mixed ")) {
    aft = substr(line, 6);
    sub(/^ */, "", aft);
    stack[++stack_i] = "<mixed>";
    if (aft != "") { parse_pattern(aft); }
  }
  else if (match(line, /^text[^A-Za-z]/) || match(line, /^text$/)) {
    stack[++stack_i] = "<text/>";
    aft = substr(line, 5);
    sub(/^ */, "", aft);
    postop = 0;
    if (aft != "") { parse_pattern(aft); }
  }
  else if (match(line, "^default namespace ")) {
    print "default namespace appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, "^namespace ")) {
    print "namespace appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, "^start ")) {
    print "start appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, "^include ")) {
    print "include appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, /^xsd:[A-Za-z_.-]/)) {
    name = substr(line, 1);
    sub(/^xsd:/, "", name);
    sub(/[^A-Za-z_.-]+.*/, "", name);
    stack[++stack_i] = sprintf("<data type='%s' datatypeLibrary='http://www.w3.org/2001/XMLSchema-datatypes'",
                               name);
    postop = 0;
    aft = "";
    if (length(line) >= length(name) + 5) {
      aft = substr(line, length(name) + 5);
      sub(/^ */, "", aft)
    }
    if (substr(aft, 1, 1) == "{") {
      stack[stack_i] = stack[stack_i] ">";
      aft = substr(aft, 2);
      while (aft != "" && substr(aft, 1, 1) != "}") {
        sub(/^ */, "", aft);
        subpos = index(aft, "=");
        paramname = substr(aft, 1, subpos - 1);
        sub(/ *$/, "", paramname);
        aft = substr(aft, subpos);
        subpos = index(aft, "\"");
        aft = substr(aft, subpos + 1);
        subpos = index(aft, "\"");
        paramval = substr(aft, 1, subpos - 1);
        aft = substr(aft, subpos + 1);
        sub(/^ */, "", aft);
        stack[stack_i] = stack[stack_i] sprintf("<param name='%s'>%s</param>", paramname, paramval);
      }
      stack[stack_i] = stack[stack_i] "</data>";
      if (aft != "")
        aft = substr(aft, 2);
    }
    else {
      stack[stack_i] = stack[stack_i] "/>";
    }
    if (aft != "") {
      parse_pattern(aft);
    }
  }
  else if (match(line, /^[A-Za-z_.-]/)) {
    if (popop(line)) {
      next;
    }
    name = substr(line, 1);
    sub(/[^A-Za-z_.-]+.*/, "", name);
    if (name == "string") {
      stack[++stack_i] = "<data type='string'/>";
    }
    else if (name == "empty") {
      stack[++stack_i] = "<empty/>";
    }
    else if (name == "notAllowed") {
      stack[++stack_i] = "<notAllowed/>";
    }
    else {
      stack[++stack_i] = sprintf("<ref name='%s'/>", name);
    }
    postop = 0;
    if (length(line) >= length(name) + 1) {
      aft = substr(line, length(name) + 1);
      parse_pattern(aft);
    }
  }
}
function popop (line) {
  op = stack[paren[paren_i]];
  if (postop == 0 && (op == "||" || op == "&&" || op == ",,")) {
    stack[paren[paren_i]] = "(" substr(op, 1, 1);
    parse_pattern(")");
    mode = "grammar";
    if (line != "") {
      parse_define(line);
    }
    return 1;
  }
  return 0;
}
function parse_name_class (line) {
  sub(/^ */, "", line)
  if (length(line) == 0) return;
  c = substr(line, 1, 1);
  if (c == "{") {
    if (stack_i != name_class_i) {
      tmp = "";
      for (i = stack_i; i >= name_class_i; i--) {
        if (stack[i] == "<except>") {
          tmp = "<except>" tmp "</except>";
        }
        else if (stack[i] == "<anyName>") {
          tmp = "<anyName>" tmp "</anyName>";
        }
        else {
          tmp = stack[i] tmp
        }
      }
      stack[name_class_i] = tmp;
      stack_i = name_class_i;
    }
    mode = "pattern";
    parse_pattern(line);
  }
  else if (c == "*") {
    if (stack[stack_i] == "<element" || stack[stack_i] == "<attribute") {
      stack[stack_i] = stack[stack_i] ">";
    }
    stack[++stack_i] = "<anyName>";
    parse_name_class(substr(line, 2));
  }
  else if (c == "-" && stack[stack_i] == "<anyName>") {
    stack[++stack_i] = "<except>"
    parse_name_class(substr(line, 2));
  }
  else if (c == "|") {
    if (length(stack[paren[paren_i]]) == 1) {
      stack[paren[paren_i]] = stack[paren[paren_i]] c;
    }
    else if (substr(stack[paren[paren_i]], 2) != "|") {
      print "Mismatched infix operators on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }
    parse_name_class(substr(line, 2));
  }
  else if (c == "(") {
    stack[++stack_i] = "(";
    paren[++paren_i] = stack_i;
    parse_name_class(substr(line, 2));
  }
  else if (c == ")") {
    open = stack[paren[paren_i]];
    if (substr(open, 1, 1) != "(") {
      print "Mismatched parentheses on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }
    if (length(open) == 2 && substr(open, 2, 1) != "|") {
      print "Unknown name class pattern on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }
    tmp = ""
    for (i = paren[paren_i] + 1; i <= stack_i; i++) {
      tmp = tmp stack[i] "\n";
    }
    stack_i = paren[paren_i];
    stack[stack_i] = tmp;
    paren_i--;
    parse_name_class(substr(line, 2));
  }
  else if (match(line, /^[A-Za-z_.-]/)) {
    name = substr(line, 1);
    sub(/[^A-Za-z_.-]+.*/, "", name);
    if (length(line) >= length(name) + 1)
      aft = substr(line, length(name) + 1);
    else
      aft = "";
    if (length(aft) >= 2 && substr(aft, 1, 2) == ":*") {
      namespace_uri = "";
      if (name == "xml") {
        namespace_uri = "http://www.w3.org/XML/1998/namespace";
      }
      else {
        for (i = 1; i <= namespaces_i; i++) {
          if (nsnames[i] == name) {
            namespace_uri = namespaces[i];
            break;
          }
        }
      }
      if (namespace_uri == "")
        stack[++stack_i] = "<nsName/>";
      else
        stack[++stack_i] = sprintf("<nsName ns='%s'/>", namespace_uri);
      if (length(aft) >= 3)
        aft = substr(aft, 3);
      else
        aft = "";
    }
    else {
      if (length(aft) >= 2 && substr(aft, 1, 1) == ":") {
        checkns = "";
        pos = 1;
        while (pos <= namespaces_i) {
          if (namespaces[pos] != "" && nsnames[pos] == name) {
            checkns = namespaces[pos];
            break;
          }
          pos++;
        }
        if (name != "xml" && checkns == "") {
          print "Unknown namespace prefix " name " on line " FNR | "cat 1>&2";
          error = 1;
          exit 1
        }
        localname = substr(aft, 2);
        sub(/[^A-Za-z_.-]+.*/, "", localname);
        if (length(aft) >= length(localname) + 2)
          aft = substr(aft, length(localname) + 2);
        else
          aft = "";
        name = name ":" localname;
      }
      if (stack[stack_i] == "<element" || stack[stack_i] == "<attribute") {
        stack[stack_i] = stack[stack_i] sprintf(" name='%s'>", name);
      }
      else {
        stack[++stack_i] = sprintf("<name>%s</name>", name);
      }
    }
    parse_name_class(aft);
  }
}

function printstack () {
  pos = 1;
  while (pos <= stack_i) {
    printstackone();
  }
}
function printstackone () {
  if (substr(stack[pos], 1, 1) == "#") {
    print "<!--";
    while (substr(stack[pos], 1, 1) == "#") {
      cmt = substr(stack[pos], 2);
      sub(/^ */, "", cmt);
      print cmt;
      pos++;
    }
    print "-->";
  }
  else if (substr(stack[pos], 1, 6) == "<start") {
    print stack[pos];
    pos++;
    printstackone();
    print "</start>"
  }
  else if (substr(stack[pos], 1, 7) == "<define") {
    print stack[pos];
    pos++;
    printstackone();
    print "</define>"
  }
  else {
    print stack[pos];
    pos++;
  }
}

BEGIN {
  mode = "grammar";
  stack_i = 0;
  paren_i = 0;
  namespaces_i = 0;
  include_i = 0;
  postop = 0;
  error = 0;
  maybe_define = "";
}

END {
  popop("");
  if (!error) {
    print "<grammar xmlns='http://relaxng.org/ns/structure/1.0'";
    pos = 1;
    while (pos <= namespaces_i) {
      if (namespaces[pos] != "") {
        printf " xmlns:%s='%s'", nsnames[pos], namespaces[pos];
      }
      pos++;
    }
    if (default_namespace != "") {
      printf " ns='%s'>\n", default_namespace;
    }
    else {
      print ">";
    }
    printstack()
    print "</grammar>"
  }
}
/^[^#]/ || /^$/ {
  if (substr(stack[stack_i], 1, 1) == "#") {
    stack[++stack_i] = " "
  }
}

mode == "pattern" && paren_i == 0 && /^[^{(]*=/ { mode = "grammar"; }
mode == "grammar" && /^[^{(]*=/ {
  if (match($0, "^default namespace ")) {
    namespace = substr($0, index($0, "=") + 1);
    nsname = substr($0, 19, index($0, "=") - 19);
    sub(/^ */, "", nsname);
    sub(/ *$/, "", nsname);
    sub(/^ *"/, "", namespace);
    sub(/" *$/, "", namespace);
    if (nsname != "") {
      nsnames[++namespaces_i] = nsname;
      namespaces[namespaces_i] = namespace;
    }
    default_namespace = namespace
  }
  else if (match($0, "^namespace ")) {
    namespace = substr($0, index($0, "=") + 1);
    nsname = substr($0, 11, index($0, "=") - 11);
    sub(/^ */, "", nsname);
    sub(/ *$/, "", nsname);
    sub(/^ *"/, "", namespace);
    sub(/" *$/, "", namespace);
    if (nsname != "") {
      nsnames[++namespaces_i] = nsname;
      namespaces[namespaces_i] = namespace;
    }
  }
  else {
    parse_define($0);
  }
  next;
}
mode == "grammar" && /^include / {
  href = substr($0, index($0, "\"") + 1);
  brace = substr(href, index(href, "\"") + 1);
  href = substr(href, 1, index(href, "\"") - 1);
  sub(/^ */, "", brace);
  sub(/ *$/, "", brace);
  sub(/\.rnc$/, ".rng", href);
  if (brace == "{") {
    stack[++stack_i] = "<include href=\"" href "\">";
    include_i = stack_i;
  }
  else {
    stack[++stack_i] = "<include href=\"" href "\"/>";
  }
}
mode == "grammar" && /^ *[A-Za-z_.-]/ {
  maybe_define = substr($0, 1, length($0));
  sub(/^ */, "", maybe_define);
  sub(/ *$/, "", maybe_define);
}
include_i != 0 && paren_i == 0 && /}/ {
  if (substr(stack[include_i], 1, 8) == "<include") {
    stack[++stack_i] = "</include>";
  }
  else {
    print "Mismatched parentheses on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  mode = "grammar";
  include_i = 0;
  next;
}
mode == "pattern" && /^[^#]/ {
  parse_pattern($0);
  next;
}
mode == "name_class" && /^[^#]/ {
  parse_name_class($0);
  next;
}
# Doesn't handle all comments. That would require modifying
# parse_pattern and parse_name_class and generally being a lot
# more clever about output. But it handles comments that start
# a line outside a pattern. Enough for me.
/#.*/ {
  popop("");
  stack[++stack_i] = $0
  next;
}

#!/bin/awk
# -*- indent-tabs-mode: nil; c-basic-offset: 2 -*-
# rnc2rng.awk - Convert RELAX NG Compact Syntax to XML Syntax
# Copyright (C) 2007 Shaun McCance <shaunm@gnome.org>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

# This program is free software, but that doesn't mean you should use it.
# I wanted to write and maintain the Mallard schema in RNC in code blocks
# within the specification, but I needed to distribute the schema in RNG.
# Since xmllint (libxml2) does not currently support RNC, and since I'm
# not willing to introduce more dependencies, awk was my only option.
#
# This awk script is *NOT* a complete implementation, and probably never
# will be.  Its sole purpose is to handle the Mallard schema, and it is
# sufficient for that task.
#
# You are, of course, free to use any of this.  If you do proliferate this
# hack, it is requested (though not required, that would be non-free) that
# you atone for your actions.  A good atonement would be contributing to
# free software.

function parse_define (line) {
  nameix = index(line, "=");
  combine = "";
  if (nameix < 1) next;
  if (substr(line, nameix - 1, 1) == "|") {
    combine = "choice";
    name = substr(line, 1, nameix - 2);
  }
  else if (substr(line, nameix - 1, 1) == "&") {
    combine = "interleave";
    name = substr(line, 1, nameix - 2);
  }
  else {
    name = substr(line, 1, nameix - 1);
  }
  sub(/^ */, "", name);
  sub(/ *$/, "", name);
  if (name == "")
    name = maybe_define;
  if (name == "start")
    define = "<start"
  else
    define = sprintf("<define name='%s'", name);
  if (combine != "")
    define = define " combine='" combine "'"
  define = define ">"
  stack[++stack_i] = define;
  mode = "pattern";
  maybe_define = "";
  if (length(line) >= nameix + 1)
    parse_pattern(substr(line, nameix + 1))
}
function parse_pattern (line) {
  sub(/^ */, "", line)
  if (length(line) == 0) return;
  c = substr(line, 1, 1);
  if (c == "(" || c == "{") {
    stack[++stack_i] = substr(line, 1, 1);
    paren[++paren_i] = stack_i;
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == ")" || c == "}") {
    open = stack[paren[paren_i]];
    oc = substr(open, 1, 1) c;
    if (oc != "()" && oc != "{}") {
      print "Mismatched parentheses on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }

    tag = "";
    if (length(open) > 1 && substr(open, 2, 1) == "&") {
      tag = "interleave";
    }
    else if (length(open) > 1 && substr(open, 2, 1) == "|") {
      tag = "choice";
    }
    else if (oc == "()") {
      tag = "group";
    }

    tmp = "";
    if (tag != "") {
      tmp = "<" tag ">";
    }
    for (i = paren[paren_i] + 1; i <= stack_i; i++) {
      tmp = tmp stack[i] "\n";
    }
    if (tag != "") {
      tmp = tmp "</" tag ">";
    }
    stack_i = paren[paren_i];
    stack[stack_i] = tmp;
    paren_i--;

    if (oc == "{}") {
      if (match(stack[stack_i - 1], "^<element")) {
        tmp = stack[stack_i - 1] "\n";
        if (stack[stack_i] != "") {
          tmp = tmp stack[stack_i] "\n";
        } else {
          tmp = tmp "<empty/>\n";
        }
        tmp = tmp "</element>";
        stack[--stack_i] = tmp;
      }
      else if (match(stack[stack_i - 1], "^<attribute")) {
        tmp = stack[stack_i - 1] "\n";
        if (stack[stack_i] != "") {
          tmp = tmp stack[stack_i] "\n";
        } else {
          tmp = tmp "<empty/>\n";
        }
        tmp = tmp "</attribute>";
        stack[--stack_i] = tmp;
      }
      else if (stack[stack_i - 1] == "<list>") {
        tmp = stack[stack_i - 1] "\n";
        tmp = tmp stack[stack_i] "\n";
        tmp = tmp "</list>";
        stack[--stack_i] = tmp;
      }
      else if (stack[stack_i - 1] == "<mixed>") {
        tmp = stack[stack_i - 1] "\n";
        tmp = tmp stack[stack_i] "\n";
        tmp = tmp "</mixed>";
        stack[--stack_i] = tmp;
      }
    }
    if (paren_i == 0) {
      mode = "grammar";
    }
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "|" || c == "&" || c == ",") {
    if (length(stack[paren[paren_i]]) == 1) {
      stack[paren[paren_i]] = stack[paren[paren_i]] c;
    }
    else if (length(stack[paren[paren_i]]) < 2 || substr(stack[paren[paren_i]], 2) != c) {
      if (paren_i == 0) {
        tmp = stack[stack_i];
        stack[stack_i] = c c;
        paren[++paren_i] = stack_i;
        stack[++stack_i] = tmp;
      }
      else {
        print "Mismatched infix operators on line " FNR | "cat 1>&2";
        error = 1;
        exit 1
      }
    }
    postop = 1;
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "?") {
    stack[stack_i] = "<optional>" stack[stack_i] "</optional>"
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "*") {
    stack[stack_i] = "<zeroOrMore>" stack[stack_i] "</zeroOrMore>"
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "+") {
    stack[stack_i] = "<oneOrMore>" stack[stack_i] "</oneOrMore>"
    if (length(line) >= 2)
      parse_pattern(substr(line, 2));
  }
  else if (c == "\"") {
    txt = substr(line, 2);
    sub(/".*/, "", txt)
    stack[++stack_i] = "<value>" txt "</value>";
    if (length(line) >= length(txt) + 3)
      parse_pattern(substr(line, length(txt) + 3));
  }
  else if (match(line, "^element ")) {
    aft = substr(line, 8);
    sub(/^ */, "", aft);
    name = aft;
    stack[++stack_i] = "<element";
    mode = "name_class";
    name_class_i = stack_i;
    postop = 0;
    parse_name_class(name);
  }
  else if (match(line, "^attribute ")) {
    aft = substr(line, 10);
    sub(/^ */, "", aft);
    name = aft;
    stack[++stack_i] = "<attribute";
    mode = "name_class";
    name_class_i = stack_i;
    postop = 0;
    parse_name_class(name);
  }
  else if (match(line, "^list ")) {
    aft = substr(line, 5);
    sub(/^ */, "", aft);
    stack[++stack_i] = "<list>";
    if (aft != "") { parse_pattern(aft); }
  }
  else if (match(line, "^mixed ")) {
    aft = substr(line, 6);
    sub(/^ */, "", aft);
    stack[++stack_i] = "<mixed>";
    if (aft != "") { parse_pattern(aft); }
  }
  else if (match(line, /^text[^A-Za-z]/) || match(line, /^text$/)) {
    stack[++stack_i] = "<text/>";
    aft = substr(line, 5);
    sub(/^ */, "", aft);
    if (aft != "") { parse_pattern(aft); }
  }
  else if (match(line, "^default namespace ")) {
    print "default namespace appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, "^namespace ")) {
    print "namespace appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, "^start ")) {
    print "start appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, "^include ")) {
    print "include appeared out of context on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  else if (match(line, /^xsd:[A-Za-z_.-]/)) {
    name = substr(line, 1);
    sub(/^xsd:/, "", name);
    sub(/[^A-Za-z_.-]+.*/, "", name);
    stack[++stack_i] = sprintf("<data type='%s' datatypeLibrary='http://www.w3.org/2001/XMLSchema-datatypes'/>",
                               name);
    postop = 0;
    if (length(line) >= length(name) + 5) {
      aft = substr(line, length(name) + 5);
      parse_pattern(aft);
    }
  }
  else if (match(line, /^[A-Za-z_.-]/)) {
    if (popop(line)) {
      next;
    }
    name = substr(line, 1);
    sub(/[^A-Za-z_.-]+.*/, "", name);
    if (name == "empty") {
      stack[++stack_i] = "<empty/>";
    }
    else if (name == "notAllowed") {
      stack[++stack_i] = "<notAllowed/>";
    }
    else {
      stack[++stack_i] = sprintf("<ref name='%s'/>", name);
    }
    postop = 0;
    if (length(line) >= length(name) + 1) {
      aft = substr(line, length(name) + 1);
      parse_pattern(aft);
    }
  }
}
function popop (line) {
  op = stack[paren[paren_i]];
  if (postop == 0 && (op == "||" || op == "&&" || op == ",,")) {
    stack[paren[paren_i]] = "(" substr(op, 1, 1);
    parse_pattern(")");
    mode = "grammar";
    if (line != "") {
      parse_define(line);
    }
    return 1;
  }
  return 0;
}
function parse_name_class (line) {
  sub(/^ */, "", line)
  if (length(line) == 0) return;
  c = substr(line, 1, 1);
  if (c == "{") {
    if (stack_i != name_class_i) {
      tmp = "";
      for (i = stack_i; i >= name_class_i; i--) {
        if (stack[i] == "<except>") {
          tmp = "<except>" tmp "</except>";
        }
        else if (stack[i] == "<anyName>") {
          tmp = "<anyName>" tmp "</anyName>";
        }
        else {
          tmp = stack[i] tmp
        }
      }
      stack[name_class_i] = tmp;
      stack_i = name_class_i;
    }
    mode = "pattern";
    parse_pattern(line);
  }
  else if (c == "*") {
    if (stack[stack_i] == "<element" || stack[stack_i] == "<attribute") {
      stack[stack_i] = stack[stack_i] ">";
    }
    stack[++stack_i] = "<anyName>";
    parse_name_class(substr(line, 2));
  }
  else if (c == "-" && stack[stack_i] == "<anyName>") {
    stack[++stack_i] = "<except>"
    parse_name_class(substr(line, 2));
  }
  else if (c == "|") {
    if (length(stack[paren[paren_i]]) == 1) {
      stack[paren[paren_i]] = stack[paren[paren_i]] c;
    }
    else if (substr(stack[paren[paren_i]], 2) != "|") {
      print "Mismatched infix operators on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }
    parse_name_class(substr(line, 2));
  }
  else if (c == "(") {
    stack[++stack_i] = "(";
    paren[++paren_i] = stack_i;
    parse_name_class(substr(line, 2));
  }
  else if (c == ")") {
    open = stack[paren[paren_i]];
    if (substr(open, 1, 1) != "(") {
      print "Mismatched parentheses on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }
    if (length(open) == 1 || substr(open, 2, 1) != "|") {
      print "Unknown name class pattern on line " FNR | "cat 1>&2";
      error = 1;
      exit 1
    }
    tmp = ""
    for (i = paren[paren_i] + 1; i <= stack_i; i++) {
      tmp = tmp stack[i] "\n";
    }
    stack_i = paren[paren_i];
    stack[stack_i] = tmp;
    paren_i--;
    parse_name_class(substr(line, 2));
  }
  else if (match(line, /^[A-Za-z_.-]/)) {
    name = substr(line, 1);
    sub(/[^A-Za-z_.-]+.*/, "", name);
    if (length(line) >= length(name) + 1)
      aft = substr(line, length(name) + 1);
    else
      aft = "";
    if (length(aft) >= 2 && substr(aft, 1, 2) == ":*") {
      namespace_uri = "";
      if (name == "xml") {
        namespace_uri = "http://www.w3.org/XML/1998/namespace";
      }
      else {
        for (i = 1; i <= namespaces_i; i++) {
          if (nsnames[i] == name) {
            namespace_uri = namespaces[i];
            break;
          }
        }
      }
      if (namespace_uri == "")
        stack[++stack_i] = "<nsName/>";
      else
        stack[++stack_i] = sprintf("<nsName ns='%s'/>", namespace_uri);
      if (length(aft) >= 3)
        aft = substr(aft, 3);
      else
        aft = "";
    }
    else {
      if (length(aft) >= 2 && substr(aft, 1, 1) == ":") {
        checkns = "";
        pos = 1;
        while (pos <= namespaces_i) {
          if (namespaces[pos] != "" && nsnames[pos] == name) {
            checkns = namespaces[pos];
            break;
          }
          pos++;
        }
        if (name != "xml" && checkns == "") {
          print "Unknown namespace prefix " name " on line " FNR | "cat 1>&2";
          error = 1;
          exit 1
        }
        localname = substr(aft, 2);
        sub(/[^A-Za-z_.-]+.*/, "", localname);
        if (length(aft) >= length(localname) + 2)
          aft = substr(aft, length(localname) + 2);
        else
          aft = "";
        name = name ":" localname;
      }
      if (stack[stack_i] == "<element" || stack[stack_i] == "<attribute") {
        stack[stack_i] = stack[stack_i] sprintf(" name='%s'>", name);
      }
      else {
        stack[++stack_i] = sprintf("<name>%s</name>", name);
      }
    }
    parse_name_class(aft);
  }
}

function printstack () {
  pos = 1;
  while (pos <= stack_i) {
    printstackone();
  }
}
function printstackone () {
  if (substr(stack[pos], 1, 1) == "#") {
    print "<!--";
    while (substr(stack[pos], 1, 1) == "#") {
      cmt = substr(stack[pos], 2);
      sub(/^ */, "", cmt);
      print cmt;
      pos++;
    }
    print "-->";
  }
  else if (substr(stack[pos], 1, 6) == "<start") {
    print stack[pos];
    pos++;
    printstackone();
    print "</start>"
  }
  else if (substr(stack[pos], 1, 7) == "<define") {
    print stack[pos];
    pos++;
    printstackone();
    print "</define>"
  }
  else {
    print stack[pos];
    pos++;
  }
}

BEGIN {
  mode = "grammar";
  stack_i = 0;
  paren_i = 0;
  namespaces_i = 0;
  include_i = 0;
  postop = 0;
  error = 0;
  maybe_define = "";
}

END {
  popop("");
  if (!error) {
    print "<grammar xmlns='http://relaxng.org/ns/structure/1.0'";
    pos = 1;
    while (pos <= namespaces_i) {
      if (namespaces[pos] != "") {
        printf " xmlns:%s='%s'", nsnames[pos], namespaces[pos];
      }
      pos++;
    }
    if (default_namespace != "") {
      printf " ns='%s'>\n", default_namespace;
    }
    else {
      print ">";
    }
    printstack()
    print "</grammar>"
  }
}
/^[^#]/ || /^$/ {
  if (substr(stack[stack_i], 1, 1) == "#") {
    stack[++stack_i] = " "
  }
}

mode == "pattern" && paren_i == 0 && /.*=/ { mode = "grammar"; }
mode == "grammar" && /.*=/ {
  if (match($0, "^default namespace ")) {
    namespace = substr($0, index($0, "=") + 1);
    nsname = substr($0, 19, index($0, "=") - 19);
    sub(/^ */, "", nsname);
    sub(/ *$/, "", nsname);
    sub(/^ *"/, "", namespace);
    sub(/" *$/, "", namespace);
    if (nsname != "") {
      nsnames[++namespaces_i] = nsname;
      namespaces[namespaces_i] = namespace;
    }
    default_namespace = namespace
  }
  else if (match($0, "^namespace ")) {
    namespace = substr($0, index($0, "=") + 1);
    nsname = substr($0, 11, index($0, "=") - 11);
    sub(/^ */, "", nsname);
    sub(/ *$/, "", nsname);
    sub(/^ *"/, "", namespace);
    sub(/" *$/, "", namespace);
    if (nsname != "") {
      nsnames[++namespaces_i] = nsname;
      namespaces[namespaces_i] = namespace;
    }
  }
  else {
    parse_define($0);
  }
  next;
}
mode == "grammar" && /^include / {
  href = substr($0, index($0, "\"") + 1);
  brace = substr(href, index(href, "\"") + 1);
  href = substr(href, 1, index(href, "\"") - 1);
  sub(/^ */, "", brace);
  sub(/ *$/, "", brace);
  sub(/\.rnc$/, ".rng", href);
  if (brace == "{") {
    stack[++stack_i] = "<include href=\"" href "\">";
    include_i = stack_i;
  }
  else {
    stack[++stack_i] = "<include href=\"" href "\"/>";
  }
}
mode == "grammar" && /^ *[A-Za-z_.-]/ {
  maybe_define = substr($0, 1, length($0));
  sub(/^ */, "", maybe_define);
  sub(/ *$/, "", maybe_define);
}
include_i != 0 && paren_i == 0 && /}/ {
  if (substr(stack[include_i], 1, 8) == "<include") {
    stack[++stack_i] = "</include>";
  }
  else {
    print "Mismatched parentheses on line " FNR | "cat 1>&2";
    error = 1;
    exit 1
  }
  mode = "grammar";
  include_i = 0;
  next;
}
mode == "pattern" && /^[^#]/ {
  parse_pattern($0);
  next;
}
mode == "name_class" && /^[^#]/ {
  parse_name_class($0);
  next;
}
# Doesn't handle all comments. That would require modifying
# parse_pattern and parse_name_class and generally being a lot
# more clever about output. But it handles comments that start
# a line outside a pattern. Enough for me.
/#.*/ {
  popop("");
  stack[++stack_i] = $0
  next;
}


Fix: bracket expression matches a character at gawk

This is a trivial patch. However, It's must be modified by gawk.
Check it exactly as a plain character (e.g., space, alnum)
within the bracket expression at ./coreclr/generateversionscript.awk

Signed-off-by: Geunsik Lim <acd8f4539166019ed6425de5f67e6d834f46e5a4@samsung.com>

BEGIN {
	print "V1.0 {";
	print "    global:";
}
{
	# Remove the CR character in case the sources are mapped from
	# a Windows share and contain CRLF line endings
	gsub(/\r/,"", $0);

	# Skip empty lines and comment lines starting with semicolon
	if (NF && !match($0, /^[[:space:]]*;/))
	{
		print "        "  $0 ";";
	}
}
END {
	print "    local: *;"
	print "};";
}

BEGIN {
	print "V1.0 {";
	print "    global:";
}
{
	# Remove the CR character in case the sources are mapped from
	# a Windows share and contain CRLF line endings
	gsub(/\r/,"", $0);

	# Skip empty lines and comment lines starting with semicolon
	if (NF && !match($0, /^[:space:]*;/))
	{
		print "        "  $0 ";";
	}
}
END {
	print "    local: *;"
	print "};";
}


test: reimplement assertEqual() and assertNotEqual()

#!/usr/bin/gawk -f
@include "include/Commons"

function pass(name, message, ansiCode,    string) {
    if (!name) name = TEST
    if (!ansiCode) ansiCode = "green"

    G_COUNTER++; COUNTER++; G_SUCCESS++
    string = sprintf("(%2s/%-2s) TESTING %s", COUNTER, TOTAL, name)
    if (message) string = string "\n       " message
    print(ansi(ansiCode, ansi("bold", "[PASS] " string)))
}

function fail(name, message, ansiCode,    string) {
    if (!name) name = TEST
    if (!ansiCode) ansiCode = "red"

    G_COUNTER++; COUNTER++
    string = sprintf("(%2s/%-2s) TESTING %s", COUNTER, TOTAL, name)
    if (message) string = string "\n       " message
    print(ansi(ansiCode, ansi("bold", "[FAIL] " string)))
}

function assertTrue(x, name, message, ansiCode) {
    if (x)
        pass(name, "", ansiCode)
    else {
        if (!message) message = "assertTrue: FALSE"
        fail(name, message, ansiCode)
    }
}

function assertFalse(x, name, message, ansiCode) {
    if (!message) message = "assertFalse: TRUE"
    assertTrue(!x, name, message, ansiCode)
}

function assertEqual(x, y, name, message, ansiCode,    i) {
    if (!message)
        message = "assertEqual: " ansi("underline", toString(x, 1))     \
            " IS NOT EQUAL TO " ansi("underline", toString(y, 1))
    assertTrue(identical(x, y), name, message, ansiCode)
}

function assertNotEqual(x, y, name, message, ansiCode,    i) {
    if (!message)
        message = "assertNotEqual: " ansi("underline", toString(x, 1))  \
            " IS EQUAL TO " ansi("underline", toString(y, 1))
    assertFalse(identical(x, y), name, message, ansiCode)
}

function T(test, total) {
    TEST = test
    TOTAL = total
    COUNTER = 0
}

function START_TEST(name) {
    print(ansi("negative", ansi("bold", "====== TESTING FOR "           \
                                ansi("underline", name) " STARTED")))
    G_COUNTER = G_SUCCESS = 0
}

function END_TEST() {
    TOTAL = G_COUNTER; COUNTER = G_SUCCESS - 1
    G_FAILURE = G_COUNTER - G_SUCCESS
    if (G_FAILURE) {
        assertEqual(G_COUNTER, G_SUCCESS, "COMPLETED",                  \
                    G_SUCCESS " PASSED, " G_FAILURE " FAILED", "yellow")
        exit 1
    } else
        assertEqual(G_COUNTER, G_SUCCESS, "COMPLETED. PERFECT!")
}

@include "test/Test"

#!/usr/bin/gawk -f
@include "include/Commons"

function pass(name, message, ansiCode,    string) {
    if (!name) name = TEST
    if (!ansiCode) ansiCode = "green"

    G_COUNTER++; COUNTER++; G_SUCCESS++
    string = sprintf("(%2s/%-2s) TESTING %s", COUNTER, TOTAL, name)
    if (message) string = string "\n       " message
    print(ansi(ansiCode, ansi("bold", "[PASS] " string)))
}

function fail(name, message, ansiCode,    string) {
    if (!name) name = TEST
    if (!ansiCode) ansiCode = "red"

    G_COUNTER++; COUNTER++
    string = sprintf("(%2s/%-2s) TESTING %s", COUNTER, TOTAL, name)
    if (message) string = string "\n       " message
    print(ansi(ansiCode, ansi("bold", "[FAIL] " string)))
}

function assertTrue(x, name, message, ansiCode) {
    if (x)
        pass(name, "", ansiCode)
    else {
        if (!message) message = "assertTrue: FALSE"
        fail(name, message, ansiCode)
    }
}

function assertFalse(x, name, message, ansiCode) {
    if (!message) message = "assertFalse: TRUE"
    assertTrue(!x, name, message, ansiCode)
}

function assertEqual(x, y, name, message, ansiCode,    i) {
    if (isarray(x) && !isarray(y)) {
        if (!message)
            message = "assertEqual: EXPECTED SCALAR VALUE " ansi("underline", y) ", GOT:"
        fail(name, message, ansiCode)
        da(x)

    } else if (!isarray(x) && isarray(y)) {
        if (!message)
            message = "assertEqual: GOT SCALAR VALUE " ansi("underline", x) ", EXPECTED:"
        fail(name, message, ansiCode)
        da(y)

    } else if (isarray(x) && isarray(y)) {
        if (length(x) != length(y)) {
            if (!message)
                message = "assertEqual: ARRAY LENGTH "                  \
                    ansi("underline", length(x)) " != " ansi("underline", length(y))
            fail(name, message, ansiCode)

        } else {
            for (i in x) {
                if (x[i] != y[i]) {
                    if (!message)
                        message = "assertEqual: ARRAY[" i "] "          \
                            ansi("underline", x[i]) " != " ansi("underline", y[i])
                    fail(name, message, ansiCode)
                    return
                }
            }
            pass(name, "", ansiCode)
        }

    } else {
        if (!message)
            message = "assertEqual: " ansi("underline", x)      \
                " IS NOT EQUAL TO " ansi("underline", y)
        assertTrue(x == y, name, message, ansiCode)
    }
}

function assertNotEqual(x, y, name, message, ansiCode,    i) {
    if (isarray(x) && isarray(y)) {
        if (length(x) != length(y))
            pass(name, "", ansiCode)
        else {
            for (i in x) {
                if (x[i] != y[i]) {
                    pass(name, "", ansiCode)
                    return
                }
            }
            if (!message)
                message = "assertNotEqual: TWO ARRAYS ARE IDENTICAL"
            fail(name, message, ansiCode)
        }

    } else if (!isarray(x) && !isarray(y)) {
        if (!message)
            message = "assertNotEqual: " ansi("underline", x)   \
                " IS EQUAL TO " ansi("underline", y)
        assertFalse(x == y, name, message, ansiCode)

    } else
        pass(name, "", ansiCode)
}

function T(test, total) {
    TEST = test
    TOTAL = total
    COUNTER = 0
}

function START_TEST(name) {
    print(ansi("negative", ansi("bold", "====== TESTING FOR "           \
                                ansi("underline", name) " STARTED")))
    G_COUNTER = G_SUCCESS = 0
}

function END_TEST() {
    TOTAL = G_COUNTER; COUNTER = G_SUCCESS - 1
    G_FAILURE = G_COUNTER - G_SUCCESS
    if (G_FAILURE) {
        assertEqual(G_COUNTER, G_SUCCESS, "COMPLETED",                  \
                    G_SUCCESS " PASSED, " G_FAILURE " FAILED", "yellow")
        exit 1
    } else
        assertEqual(G_COUNTER, G_SUCCESS, "COMPLETED. PERFECT!")
}

@include "test/Test"


Add Newton's Law bonus

Add Newton's Law bonus

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Acorn TV | 19-2</title>
#  ---
#        19-2
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/<title>/ {
    sub (/.*<title>Acorn TV \| /,"")
    sub (/<.*/,"")
    gsub (/&/,"\\&")
    # gsub (/'/,"'")
    # gsub (/"/,"\"\"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    numEpisodesStr = ""
    seriesTitle = $0
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    split ($0,fld,"\"")
    episodeLinesFound += 1
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if (numSeasons == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/'/,"'",description)
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, seriesTitle, episodeURL)  >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default bonus, parttwo, and christmasspecials episodeType to "X"
    if (episodeURL ~ /bonus\/|[0-9]{1,2}parttwo\/|christmasspecial\//)
        episodeType = "X"
    # jamaicainn, newworlds & newtonslaw bonus seasonNumber should be 1
    if (episodeURL ~ /\/jamaicainn\/bonus\/|\/newworlds\/bonus\/|\/newtonslaw\/bonus\// && seasonNumber != 1) {
        split (episodeURL, part, "/")
        printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
        seasonNumber = 1
    }
    # Plain christmasspecial, seriesfinale don't increment seasonNumber
    if (episodeURL ~ /\/christmasspecial\/|\/seriesfinale\//) {
        split (episodeURL, part, "/")
        printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, Vera, and others have problems
    if (episodeURL ~ \
           /\/series[0-9]{1,2}\/|[0-9]{1,2}bonus\/|[0-9]{1,2}parttwo\/|\/murdoch\/season[0-9]{1,2}/) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/bonus/,"",URLseasonNumber)
        # Foyle's War has sets: foyleswar/series9set8bonus/
        sub (/set[0-9]/,"",URLseasonNumber)
        # Midsomer Murders has two parts: midsomermurders/series19parttwo/
        sub (/parttwo/,"",URLseasonNumber)
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

/-- Viewers Also Watched --/ {
    print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    if (description == "")
        printf ("==> No description: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    description = ""
    #
    seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
    print seriesDuration >> DURATION_FILE
    if (seriesDuration == "00:00:00")
        printf ("==> No duration: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Acorn TV | 19-2</title>
#  ---
#        19-2
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/<title>/ {
    sub (/.*<title>Acorn TV \| /,"")
    sub (/<.*/,"")
    gsub (/&/,"\\&")
    # gsub (/'/,"'")
    # gsub (/"/,"\"\"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    numEpisodesStr = ""
    seriesTitle = $0
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    split ($0,fld,"\"")
    episodeLinesFound += 1
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if (numSeasons == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/'/,"'",description)
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, seriesTitle, episodeURL)  >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default bonus, parttwo, and christmasspecials episodeType to "X"
    if (episodeURL ~ /bonus\/|[0-9]{1,2}parttwo\/|christmasspecial\//)
        episodeType = "X"
    # jamaicainn & newworlds bonus seasonNumber should be 1
    if (episodeURL ~ /\/jamaicainn\/bonus\/|\/newworlds\/bonus\// && seasonNumber != 1) {
        split (episodeURL, part, "/")
        printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
        seasonNumber = 1
    }
    # Plain christmasspecial, seriesfinale don't increment seasonNumber
    if (episodeURL ~ /\/christmasspecial\/|\/seriesfinale\//) {
        split (episodeURL, part, "/")
        printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, Vera, and others have problems
    if (episodeURL ~ \
           /\/series[0-9]{1,2}\/|[0-9]{1,2}bonus\/|[0-9]{1,2}parttwo\/|\/murdoch\/season[0-9]{1,2}/) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/bonus/,"",URLseasonNumber)
        # Foyle's War has sets: foyleswar/series9set8bonus/
        sub (/set[0-9]/,"",URLseasonNumber)
        # Midsomer Murders has two parts: midsomermurders/series19parttwo/
        sub (/parttwo/,"",URLseasonNumber)
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

/-- Viewers Also Watched --/ {
    print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    if (description == "")
        printf ("==> No description: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    description = ""
    #
    seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
    print seriesDuration >> DURATION_FILE
    if (seriesDuration == "00:00:00")
        printf ("==> No duration: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}


looser pattern for shell scripts and some other C files

BEGIN {
	IN_BLOCK = 0;

	PRE_LINE = "^[ \t]*";
	POST_LINE = "[ \t]*$";

	# tmp variables
	X = "((\\*|#( *\\*)?) *)?";
	PRE_BLOCK = "(/\\* *|# *(/\\* *)?)?";
	POST_BLOCK = " *(\\*/)?";

	BLOCK[ 0] = PRE_BLOCK "\\*\\*\\*\\*\\* BEGIN LICENSE BLOCK \\*\\*\\*\\*\\*";
	BLOCK[ 1] = X;
	BLOCK[ 2] = X "BBN Address and AS Number PKI Database/repository software";
	BLOCK[ 3] = X "Version [^ ]*";
	BLOCK[ 4] = X;
	BLOCK[ 5] = X "US government users are permitted unrestricted rights as";
	BLOCK[ 6] = X "defined in the FAR.";
	BLOCK[ 7] = X;
	BLOCK[ 8] = X "This software is distributed on an \"AS IS\" basis, WITHOUT";
	BLOCK[ 9] = X "WARRANTY OF ANY KIND, either express or implied.";
	BLOCK[10] = X;
	BLOCK[11] = X "Copyright .*";
	BLOCK[12] = X;
	BLOCK[13] = X "Contributor(\\(s\\)|s)?: .*";
	BLOCK[14] = X;
	BLOCK[15] = X "\\*\\*\\*\\*\\* END LICENSE BLOCK \\*\\*\\*\\*\\*" POST_BLOCK;

	BLOCK_END = 15;
}

{
	if (!IN_BLOCK) {
		if ($0 ~ PRE_LINE BLOCK[0] POST_LINE) {
			IN_BLOCK = 1;
			BLOCK_LINENO = 0;
			SAVED_BLOCK = $0;
		} else {
			print;
		}
	} else {
		if ($0 ~ PRE_LINE BLOCK[BLOCK_LINENO + 1] POST_LINE) {
			SAVED_BLOCK = SAVED_BLOCK RS $0;
			BLOCK_LINENO += 1;
			if (BLOCK_LINENO == BLOCK_END) {
				IN_BLOCK = 0;
			}
		} else {
			print SAVED_BLOCK;
			print;
			IN_BLOCK = 0;
		}
	}
}

END {
	if (IN_BLOCK) {
		print SAVED_BLOCK;
	}
}

BEGIN {
	IN_BLOCK = 0;

	PRE_LINE = "^[ \t]*";
	POST_LINE = "[ \t]*$";

	# tmp variable
	X = "(\\* *)?";

	BLOCK[ 0] = "/\\* \\*\\*\\*\\*\\* BEGIN LICENSE BLOCK \\*\\*\\*\\*\\*";
	BLOCK[ 1] = X;
	BLOCK[ 2] = X "BBN Address and AS Number PKI Database/repository software";
	BLOCK[ 3] = X "Version [^ ]*";
	BLOCK[ 4] = X;
	BLOCK[ 5] = X "US government users are permitted unrestricted rights as";
	BLOCK[ 6] = X "defined in the FAR.";
	BLOCK[ 7] = X;
	BLOCK[ 8] = X "This software is distributed on an \"AS IS\" basis, WITHOUT";
	BLOCK[ 9] = X "WARRANTY OF ANY KIND, either express or implied.";
	BLOCK[10] = X;
	BLOCK[11] = X "Copyright .*";
	BLOCK[12] = X;
	BLOCK[13] = X "Contributor(\\(s\\)|s)?: .*";
	BLOCK[14] = X;
	BLOCK[15] = X "\\*\\*\\*\\*\\* END LICENSE BLOCK \\*\\*\\*\\*\\* \\*/";

	BLOCK_END = 15;
}

{
	if (!IN_BLOCK) {
		if ($0 ~ PRE_LINE BLOCK[0] POST_LINE) {
			IN_BLOCK = 1;
			BLOCK_LINENO = 0;
			SAVED_BLOCK = $0;
		} else {
			print;
		}
	} else {
		if ($0 ~ PRE_LINE BLOCK[BLOCK_LINENO + 1] POST_LINE) {
			SAVED_BLOCK = SAVED_BLOCK RS $0;
			BLOCK_LINENO += 1;
			if (BLOCK_LINENO == BLOCK_END) {
				IN_BLOCK = 0;
			}
		} else {
			print SAVED_BLOCK;
			print;
			IN_BLOCK = 0;
		}
	}
}

END {
	if (IN_BLOCK) {
		print SAVED_BLOCK;
	}
}