Enhanced to be usable for python objects in addition to C++ objects.

# Qick hack to convert pydoc to something decent looking in html.
# Author: Thomas Moeller
# Usage: pydoc3.3 inventor | awk -f pydoc2html.awk > inventor.html
#
# Example for updating a man directory:
# for f in *.html; do pydoc3.3 ${f%%.html} | awk -f ../../PyInventor/utils/pydoc2html.awk > $f; done
#
# Copyright (C) the PyInventor contributors. All rights reserved.
# This file is part of PyInventor, distributed under the BSD 3-Clause
# License. For full terms see the included COPYING file.
#

BEGIN   {
          eatEmptyLines = 0;
          indendet = 0;
          dataDesc = 0;
          isBullet = 0;
          print "<!DOCTYPE html>";
          print "<meta name='description' content='PyInventor : 3D Graphics in Python with Open Inventor' />";
          print "<link rel='stylesheet' type='text/css' media='screen' href='../stylesheets/stylesheet.css'>";
          print "<title>PyInventor</title>";
          print "
";
          print "View on GitHub";
          print "
PyInventor

3D Graphics in Python with Open Inventor
";
          print "
";

          print "

";
        }

END     {
          print "
";
          print "
";
          print "
PyInventor maintained by TheHubbit
";
          print "
Published with GitHub Pages

";
          print "";
        }

        { sub(/^[ \t]\|  /, ""); gsub(/[\<\>]/, ""); }
/^Help on/ { getline; next; }
/ = class / { print "
" $1 "

" $3 " " $4 "
"; module = substr($1, 0, index($1, ".") - 1); next; }
/^[ ]*FILE$/ { getline; next; }
/^[ ]*NAME$/ { getline; print "
" $1 "

module " $0 "
"; module = $1; next; }
/^[ ]*DESCRIPTION$/ { next; }
/^[ ]*FUNCTIONS$/ { print "
Functions:
"; next; }
/^[ ]*[A-Z]+$/ { print "
" $0 "
"; next; }
/Method resolution order:/ {
            print "
" $0 "

";
            while (!match($0, /^[ \|\t]*$/))
            {
              getline;
              if (match($2, /PySide\./)) { link = $2; gsub(/\./, "/", link); print "
" $2 ""; } else
              if (match($2, /FieldContainer/)) print "
" $2 ""; else
              if (length($2) > 0) print "
" $2;
            }
            print "
";
            isBullet = 0; next;
          }
/Methods inherited from PySide/ { exit; }
/^[ ]*[A-Z][a-z]+ [A-Za-z \.]+:$/ { header = "
 " $0 "
"; eatEmptyLines = 1; if (match($0, /[ ]*Data /)) dataDesc = 1; next; }
/__/    { getline; next; }
/^[ \t]*[a-zA-Z0-9_]+\([A-Za-z0-9_\., ]+\)$/ {
          sub(/^[ \t]*/, "");
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          if (length(header) > 0)
          {
            print header;
            header = "";
          }
          print "
" $0 "
"; next;
        }
/Args:|Returns:|Note:/ {
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          print "
" $0 "
"; print "
"; indendet = 1; next;
        }
/^[ ]*\- [a-zA-Z0-9_, ]+: / {
          if (!isBullet) print "
";
          print "
" $2 "";
          sub(/^[ ]*\- [a-zA-Z0-9_, ]+: /, "");
          print;
          isBullet = 1; next;
        }
/^[ -]+[a-zA-Z0-9_, ]+: / { sub(/^[ -]+/, ""); print "
" $0; isBullet = 1; next; }
/-----/ { print ""; dataDesc = 0; next; }
        {
          sub(/^[ \t]*/, "");
          if (dataDesc)
          {
            if (indendet)
            {
              indendet = 0;
              print "
";
            }
            if ((length(header) > 0) && (length($0) > 0))
            {
              print header;
              header = "";
            }
            if (match($0, /^[ \t]*[a-zA-Z0-9_]+$/))
            {
              print "
" $0 "
";
            }
            else if (length($0) > 0)
            {
              if (index($0, " = ") > 0)
              {
                print $0 "
";
              }
              else print $0;
            }
          }
          else if (!eatEmptyLines || length($0) > 0)
          {
            print;
          }
          if (length($0) > 0) eatEmptyLines = 0; else if (isBullet)
          {
            print "
";
            isBullet = 0;
          }
        }

# Qick hack to convert pydoc to something decent looking in html.
# Author: Thomas Moeller
# Usage: pydoc3.3 inventor | awk -f pydoc2html.awk > inventor.html
#
# Example for updating a man directory:
# for f in *.html; do pydoc3.3 ${f%%.html} | awk -f ../../PyInventor/utils/pydoc2html.awk > $f; done
#
# Copyright (C) the PyInventor contributors. All rights reserved.
# This file is part of PyInventor, distributed under the BSD 3-Clause
# License. For full terms see the included COPYING file.
#

BEGIN   {
          eatEmptyLines = 0;
          indendet = 0;
          dataDesc = 0;
          isBullet = 0;
          print "<!DOCTYPE html>";
          print "<meta name='description' content='PyInventor : 3D Graphics in Python with Open Inventor' />";
          print "<link rel='stylesheet' type='text/css' media='screen' href='../stylesheets/stylesheet.css'>";
          print "<title>PyInventor</title>";
          print "
";
          print "View on GitHub";
          print "
PyInventor

3D Graphics in Python with Open Inventor
";
          print "
";

          print "

";
        }

END     {
          print "
";
          print "
";
          print "
PyInventor maintained by TheHubbit
";
          print "
Published with GitHub Pages

";
          print "";
        }

        { sub(/^[ \t]\|  /, ""); }
/^Help on/ { getline; next; }
/ = class / { print "
" $1 "

" $3 " " $4 "
"; module = substr($1, 0, index($1, ".") - 1); next; }
/^[ ]*FILE$/ { getline; next; }
/^[ ]*NAME$/ { getline; print "
" $1 "

module " $0 "
"; module = $1; next; }
/^[ ]*DESCRIPTION$/ { next; }
/^[ ]*FUNCTIONS$/ { print "
Functions:
"; next; }
/^[ ]*[A-Z]+$/ { print "
" $0 "
"; next; }
/Method resolution order:/ {
            print "
" $0 "

";
            while (!match($0, /^[ \|\t]*$/))
            {
              getline;
              #if (match($2, /^object/)) print "
" $2; else if (length($2) > 0) print "
" $2 "";
              if (length($2) > 0) print "
" $2;
            }
            print "
";
            isBullet = 0; next;
          }
/Data descriptors defined here:$/ { header = "
 " $0 "
"; eatEmptyLines = 1; dataDesc = 1; next; }
/^[ ]*[A-Z][a-z]+ [A-Za-z ]+:$/ { header = "
 " $0 "
"; eatEmptyLines = 1; next; }
/__/    { getline; next; }
/\(\.\.\.\)/ {
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          if (length(header) > 0)
          {
            print header;
            header = "";
          }
          print "
" $1 "
"; next;
        }
/Args:|Returns:|Note:/ {
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          print "
" $0 "
"; print "
"; indendet = 1; next;
        }
/^[ ]*\- [a-zA-Z0-9_, ]+: / {
          if (!isBullet) print "
";
          print "
" $2 "";
          sub(/^[ ]*\- [a-zA-Z0-9_, ]+: /, "");
          print;
          isBullet = 1; next;
        }
/^[ -]+[a-zA-Z0-9_, ]+: / { sub(/^[ -]+/, ""); print "
" $0; isBullet = 1; next; }
/-----/ { print ""; dataDesc = 0; next; }
        {
          sub(/^[ \t]*/, "");
          if (dataDesc)
          {
            if (indendet)
            {
              indendet = 0;
              print "
";
            }
            if (length(header) > 0)
            {
              print header;
              header = "";
            }
            if (match($0, /^[ \t]*[a-zA-Z0-9_]+$/))
            {
              print "
" $0 "
";
            }
            else if (length($0) > 0)
            {
              print $0;
            }
          }
          else if (!eatEmptyLines || length($0) > 0)
          {
            print;
          }
          if (length($0) > 0) eatEmptyLines = 0; else if (isBullet)
          {
            print "
";
            isBullet = 0;
            #print "

";
          }
        }


addr_utils.awk	-- oops used = instead of ==

function AU_parse(rply, do_subs, addr, result, towns, st_types, st_quals, dirs, st_ords,   nf, ary, i, f_st, nf2, ary2, i1, name, street, quals, town, k) {

	result["status"] = "B"
	result["emsg"  ] = ""
	result["name"  ] = ""
	result["street"] = ""
	result["quals" ] = ""
	result["town"  ] = ""
	result["state" ] = ""

	nf = split(addr, ary, ",")
	for(i = 1; i <= nf; i++){
		sub(/^ */, "", ary[i])
		sub(/ *$/, "", ary[i])
	}

	if(rply){
		# TODO: generalize to all states?
		if(ary[nf] != "United States of America"){
			result["emsg"] = "not.usa"
			return 1
		}else
			nf--
		if(ary[nf] == "CA")
			nf--
		else if(ary[nf] ~ /CA [0-9]{5}$/)
			nf--
		else if(ary[nf] ~ /CA [0-9]{5}-[0-9]{4}$/)
			nf--
		else if(ary[nf] == "California")
			nf--
		else{
			result["emsg"] = "not.CA"
			return 1
		}
	}

	# minimal address has 2 fields: street, town
	if(nf < 2){
		result["emsg"] = "short.addr"
		return 1
	}

	# check that we have a known town
	if(!(ary[nf] in towns)){
		result["emsg"] = "bad.town"
		return 1
	}
	town = do_subs ? towns[ary[nf]] : ary[nf]

	# find the street.
	# street is 1) 1st elt of ary[1:nf-1] that begins w/number & ends w/st_type or 2) last elt of ary[1:nf-1] that begins w/number
	f_st = 0
	for(i = 1; i < nf; i++){
		if(ary[i] ~ /^[1-9]/){
			nf2 = split(ary[i], ary2, /  */)
			if(ary2[nf2] in st_types){
				f_st = i
				break
			}
			f_st = i
		}
	}
	if(f_st == 0){
		result["emsg"] = "no.street"
		return 1
	}

	# The name is what precedes f_st.  if f_st == 1, set name to "Residence"
	if(f_st == 1)
		name = "Residence"
	else{
		name = ary[1]
		for(i = 2; i < f_st; i++)
			name = name ", " ary[i]
	}

	# Street qualifiers are those fields, if any, that follow f_st and precede nf
	if(nf - f_st > 1){
		quals = ary[f_st+1]
		for(i = f_st+2; i < nf; i++)
			quals = quals ", " ary[i]
	}else
		quals = ""

	# clean up street
	nf2 = split(ary[f_st], ary2, /  */)
	if(nf2 < 2){
		result["emsg"] = "short.street"
		return 1
	}

	# Street Hacks: Probably should just compare ignore case but not ready for that step yet
	# change el Camino X to El Camino X
	# change el Monte X  to El Monte X
	if(do_subs){
		if(nf2 > 3){
			if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Camino")
				ary2[nf2-2] = "El"
			else if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Monte")
				ary2[nf2-2] =  "El"
		}
	}

	# Street should be num [ dir ] str [ st ]
	street = ary2[1]
	if(do_subs){
		# This is amusing.  What is South Court? Is is S. Court or South Ct.  No idea so, leave such streets in long form
		if(nf2 == 3 && (ary2[2] in dirs) && (ary2[3] in st_types)){
			for(i = 2; i <= nf2; i++)
				street = street " " ary2[i]
		}else{
			if(ary2[2] in dirs){
				street = street " " dirs[ary2[2]]
				i1 = 3
			}else
				i1 = 2
			for(i = i1; i < nf2; i++){
				if(i == i1)
					street = street ((ary2[i] in st_ords) ? (" " st_ords[ary2[i]]) : (" "ary2[i]))
				else
					street = street " " ary2[i]
			}
			street = street ((ary2[nf2] in st_types) ? (" " st_types[ary2[nf2]]) : (" " ary2[nf2]))
		}
	}else{
		for(i = 2; i <= nf2; i++)
			street = street " " ary2[i]
	}

	result["status"] = "G"
	result["name"  ] = name
	result["street"] = street
	result["quals" ] = quals
	result["town"  ] = town
	result["state" ] = "CA"

	return 0
}
function AU_get_addr_data(addr_info, key, data,   k, keys, nk, i) {

	n_data = 0
	for(k in addr_info){
		nk = split(k, keys, SUBSEP)
		if(keys[1] == key){
			data[keys[2]] = addr_info[k]
			n_data++
		}
	}
	return n_data
}
function AU_match(cand, ref,   is_mat, i, n_cand_st, cand_st, cand_odd, n_ref_st, ref_st, ref_odd) {

	is_mat = 0

	# deal with street ranges
	n_cand_st = split(cand["street"], cand_st, /-/)
	n_ref_st = split(ref["street"], ref_st, /-/)

	# US rule: require all numbers to be either odd or even
	cand_odd = cand_st[1] % 2
	for(i = 2; i <= n_cand_st; i++){
		if(cand_st[i] % 2 != cand_odd){
			printf("ERROR: odd/even candidate address range: %s\n", cand["street"]) > "/dev/stderr"
			return 0
		}
	}
	n_ref_st = split(ref["street"], ref_st, /-/)
	ref_odd = ref_st[1] % 2
	for(i = 2; i <= n_ref_st; i++){
		if(ref_st[i] % 2 != ref_odd){
			printf("ERROR: odd/even reference address range: %s\n", ref["street"]) > "/dev/stderr"
			return 0
		}
	}
	if(cand_odd != ref_odd)
		return 0

	# odd/even good, check the actual numbers/ranges
	if(n_cand_st == 1){
		if(n_ref_st == 1){
			is_mat = cand_st[1] == ref_st[1]
		}else{
			is_mat = cand_st[1] >= ref_st[1] && cand_st[1] <= ref_st[2]
		}
	}else if(n_ref_st == 1){
		is_mat = ref_st[1] >= cand_st[1] && ref_st[1] <= cand_st[2]
	}else{
		is_mat = !(cand_st[2] < ref_st[1] || cand_st[1] > ref_st[2])
	}

	# streets are good, deal with towns
	if(is_mat){
		is_mat = cand["town"] == ref["town"]
	}

	return is_mat
}
function AU_match_2(cand, ref,   n_cand_fields, cand_fields, n_ref_fields, ref_fields, i, n_cand_rtab, cand_rtab, n_ref_rtab, ref_rtab) {

	# street numbers and/or ranges, etc is 1st field
	n_cand_fields = split(cand["street"], cand_fields, /  */)
	n_ref_fields = split(ref["street"], ref_fields, /  */)

	# check that non-number parts of streets agree
	if(n_cand_fields != n_ref_fields)
		return 0
	for(i = 2; i <= n_cand_fields; i++){
		if(cand_fields[i] != ref_fields[i])
			return 0
	}

	n_cand_rtab = AU_get_rtab(cand_fields[1], cand_rtab)
	if(n_cand_rtab == 0)
		return 0

	n_ref_rtab = AU_get_rtab(ref_fields[1], ref_rtab)
	if(n_ref_rtab == 0)
		return 0

	if(!AU_rtabs_intersect(n_cand_rtab, cand_rtab, n_ref_rtab, ref_rtab))
		return 0

	return cand["town"] == ref["town"]
}
function AU_get_rtab(street, rtab,   n_rtab, i, nw, work) {

	n_rtab = split(street, rtab, ";")
	for(i = 1; i <= n_rtab; i++){
		nw = split(rtab[i], work, "-")
		if(nw > 1){
			rtab[i, 1] = work[1] + 0	# force number
			rtab[i, 2] = work[2] + 0	# force number
			rtab[i, "rng"] = 1
			if(rtab[i, 1] % 2 != rtab[i, 2] % 2){
				printf("ERROR: bad range: odd/even %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
				return 0
			}
		}else{
			rtab[i] = work[1] + 0	# force number
			rtab[i, "rng"] = 0
		}
	}
	return n_rtab
}
function AU_rtabs_intersect(n_rtab1, rtab1, n_rtab2, rtab2,   i, j) {

	# the most common case: number v number
	if(n_rtab1 == 1 && !rtab1[1, "rng"] && n_rtab2 == 1 && !rtab2[1, "rng"])
		return rtab1[1] == rtab2[1]

	for(i = 1; i <= n_rtab1; i++){
		for(j = 1; j <= n_rtab2; j++){
			if(!rtab1[i, "rng"]){
				if(!rtab2[j, "rng"]){	# number v number
					if(rtab1[i] == rtab2[j])
						return 1
				}else{			# number v range
					if(rtab1[i] >= rtab2[j, 1] && rtab1[i] <= rtab2[j, 2])
						return 1
				}
			}else if(!rtab2[j, "rng"]){	# range v number
				if(rtab2[j] >= rtab1[i, 1] && rtab2[j] <= rtab1[i, 2])
					return 1
			}else{				# range v range
				if(!(rtab1[i, 2] < rtab2[j, 1] || rtab1[i, 1] > rtab2[j, 2]))
					return 1
			}
		}
	}
	return 0
}
function AU_rtab_dump(file, n_rtab, rtab,   i) {

	printf("n_rtab = %d {\n", n_rtab) > file
	for(i = 1; i <= n_rtab; i++){
		if(rtab[i, "rng"])
			printf("\trtab[%d, 1-2] = %d-%d\n", i, rtab[i, 1], rtab[i, 2]) > file
		else
			printf("\trtab[%d     ] = %d\n", i, rtab[i]) > file
	}
	printf("}\n") > file
}

function AU_parse(rply, do_subs, addr, result, towns, st_types, st_quals, dirs, st_ords,   nf, ary, i, f_st, nf2, ary2, i1, name, street, quals, town, k) {

	result["status"] = "B"
	result["emsg"  ] = ""
	result["name"  ] = ""
	result["street"] = ""
	result["quals" ] = ""
	result["town"  ] = ""
	result["state" ] = ""

	nf = split(addr, ary, ",")
	for(i = 1; i <= nf; i++){
		sub(/^ */, "", ary[i])
		sub(/ *$/, "", ary[i])
	}

	if(rply){
		# TODO: generalize to all states?
		if(ary[nf] != "United States of America"){
			result["emsg"] = "not.usa"
			return 1
		}else
			nf--
		if(ary[nf] == "CA")
			nf--
		else if(ary[nf] ~ /CA [0-9]{5}$/)
			nf--
		else if(ary[nf] ~ /CA [0-9]{5}-[0-9]{4}$/)
			nf--
		else if(ary[nf] == "California")
			nf--
		else{
			result["emsg"] = "not.CA"
			return 1
		}
	}

	# minimal address has 2 fields: street, town
	if(nf < 2){
		result["emsg"] = "short.addr"
		return 1
	}

	# check that we have a known town
	if(!(ary[nf] in towns)){
		result["emsg"] = "bad.town"
		return 1
	}
	town = do_subs ? towns[ary[nf]] : ary[nf]

	# find the street.
	# street is 1) 1st elt of ary[1:nf-1] that begins w/number & ends w/st_type or 2) last elt of ary[1:nf-1] that begins w/number
	f_st = 0
	for(i = 1; i < nf; i++){
		if(ary[i] ~ /^[1-9]/){
			nf2 = split(ary[i], ary2, /  */)
			if(ary2[nf2] in st_types){
				f_st = i
				break
			}
			f_st = i
		}
	}
	if(f_st == 0){
		result["emsg"] = "no.street"
		return 1
	}

	# The name is what precedes f_st.  if f_st == 1, set name to "Residence"
	if(f_st == 1)
		name = "Residence"
	else{
		name = ary[1]
		for(i = 2; i < f_st; i++)
			name = name ", " ary[i]
	}

	# Street qualifiers are those fields, if any, that follow f_st and precede nf
	if(nf - f_st > 1){
		quals = ary[f_st+1]
		for(i = f_st+2; i < nf; i++)
			quals = quals ", " ary[i]
	}else
		quals = ""

	# clean up street
	nf2 = split(ary[f_st], ary2, /  */)
	if(nf2 < 2){
		result["emsg"] = "short.street"
		return 1
	}

	# Street Hacks: Probably should just compare ignore case but not ready for that step yet
	# change el Camino X to El Camino X
	# change el Monte X  to El Monte X
	if(do_subs){
		if(nf2 > 3){
			if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Camino")
				ary2[nf2-2] = "El"
			else if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Monte")
				ary2[nf2-2] =  "El"
		}
	}

	# Street should be num [ dir ] str [ st ]
	street = ary2[1]
	if(do_subs){
		# This is amusing.  What is South Court? Is is S. Court or South Ct.  No idea so, leave such streets in long form
		if(nf2 == 3 && (ary2[2] in dirs) && (ary2[3] in st_types)){
			for(i = 2; i <= nf2; i++)
				street = street " " ary2[i]
		}else{
			if(ary2[2] in dirs){
				street = street " " dirs[ary2[2]]
				i1 = 3
			}else
				i1 = 2
			for(i = i1; i < nf2; i++){
				if(i == i1)
					street = street ((ary2[i] in st_ords) ? (" " st_ords[ary2[i]]) : (" "ary2[i]))
				else
					street = street " " ary2[i]
			}
			street = street ((ary2[nf2] in st_types) ? (" " st_types[ary2[nf2]]) : (" " ary2[nf2]))
		}
	}else{
		for(i = 2; i <= nf2; i++)
			street = street " " ary2[i]
	}

	result["status"] = "G"
	result["name"  ] = name
	result["street"] = street
	result["quals" ] = quals
	result["town"  ] = town
	result["state" ] = "CA"

	return 0
}
function AU_get_addr_data(addr_info, key, data,   k, keys, nk, i) {

	n_data = 0
	for(k in addr_info){
		nk = split(k, keys, SUBSEP)
		if(keys[1] == key){
			data[keys[2]] = addr_info[k]
			n_data++
		}
	}
	return n_data
}
function AU_match(cand, ref,   is_mat, i, n_cand_st, cand_st, cand_odd, n_ref_st, ref_st, ref_odd) {

	is_mat = 0

	# deal with street ranges
	n_cand_st = split(cand["street"], cand_st, /-/)
	n_ref_st = split(ref["street"], ref_st, /-/)

	# US rule: require all numbers to be either odd or even
	cand_odd = cand_st[1] % 2
	for(i = 2; i <= n_cand_st; i++){
		if(cand_st[i] % 2 != cand_odd){
			printf("ERROR: odd/even candidate address range: %s\n", cand["street"]) > "/dev/stderr"
			return 0
		}
	}
	n_ref_st = split(ref["street"], ref_st, /-/)
	ref_odd = ref_st[1] % 2
	for(i = 2; i <= n_ref_st; i++){
		if(ref_st[i] % 2 != ref_odd){
			printf("ERROR: odd/even reference address range: %s\n", ref["street"]) > "/dev/stderr"
			return 0
		}
	}
	if(cand_odd != ref_odd)
		return 0

	# odd/even good, check the actual numbers/ranges
	if(n_cand_st == 1){
		if(n_ref_st == 1){
			is_mat = cand_st[1] == ref_st[1]
		}else{
			is_mat = cand_st[1] >= ref_st[1] && cand_st[1] <= ref_st[2]
		}
	}else if(n_ref_st == 1){
		is_mat = ref_st[1] >= cand_st[1] && ref_st[1] <= cand_st[2]
	}else{
		is_mat = !(cand_st[2] < ref_st[1] || cand_st[1] > ref_st[2])
	}

	# streets are good, deal with towns
	if(is_mat){
		is_mat = cand["town"] == ref["town"]
	}

	return is_mat
}
function AU_match_2(cand, ref,   n_cand_fields, cand_fields, n_ref_fields, ref_fields, i, n_cand_rtab, cand_rtab, n_ref_rtab, ref_rtab) {

	# street numbers and/or ranges, etc is 1st field
	n_cand_fields = split(cand["street"], cand_fields, /  */)
	n_ref_fields = split(ref["street"], ref_fields, /  */)

	# check that non-number parts of streets agree
	if(n_cand_fields != n_ref_fields)
		return 0
	for(i = 2; i <= n_cand_fields; i++){
		if(cand_fields[i] != ref_fields[i])
			return 0
	}

	n_cand_rtab = AU_get_rtab(cand_fields[1], cand_rtab)
	if(n_cand_rtab == 0)
		return 0

	n_ref_rtab = AU_get_rtab(ref_fields[1], ref_rtab)
	if(n_ref_rtab == 0)
		return 0

	if(!AU_rtabs_intersect(n_cand_rtab, cand_rtab, n_ref_rtab, ref_rtab))
		return 0

	return cand["town"] = ref["town"]
}
function AU_get_rtab(street, rtab,   n_rtab, i, nw, work) {

	n_rtab = split(street, rtab, ";")
	for(i = 1; i <= n_rtab; i++){
		nw = split(rtab[i], work, "-")
		if(nw > 1){
			rtab[i, 1] = work[1] + 0	# force number
			rtab[i, 2] = work[2] + 0	# force number
			rtab[i, "rng"] = 1
			if(rtab[i, 1] % 2 != rtab[i, 2] % 2){
				printf("ERROR: bad range: odd/even %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
				return 0
			}
		}else{
			rtab[i] = work[1] + 0	# force number
			rtab[i, "rng"] = 0
		}
	}
	return n_rtab
}
function AU_rtabs_intersect(n_rtab1, rtab1, n_rtab2, rtab2,   i, j) {

	# the most common case: number v number
	if(n_rtab1 == 1 && !rtab1[1, "rng"] && n_rtab2 == 1 && !rtab2[1, "rng"])
		return rtab1[1] == rtab2[1]

	for(i = 1; i <= n_rtab1; i++){
		for(j = 1; j <= n_rtab2; j++){
			if(!rtab1[i, "rng"]){
				if(!rtab2[j, "rng"]){	# number v number
					if(rtab1[i] == rtab2[j])
						return 1
				}else{			# number v range
					if(rtab1[i] >= rtab2[j, 1] && rtab1[i] <= rtab2[j, 2])
						return 1
				}
			}else if(!rtab2[j, "rng"]){	# range v number
				if(rtab2[j] >= rtab1[i, 1] && rtab2[j] <= rtab1[i, 2])
					return 1
			}else{				# range v range
				if(!(rtab1[i, 2] < rtab2[j, 1] || rtab1[i, 1] > rtab2[j, 2]))
					return 1
			}
		}
	}
	return 0
}
function AU_rtab_dump(file, n_rtab, rtab,   i) {

	printf("n_rtab = %d {\n", n_rtab) > file
	for(i = 1; i <= n_rtab; i++){
		if(rtab[i, "rng"])
			printf("\trtab[%d, 1-2] = %d-%d\n", i, rtab[i, 1], rtab[i, 2]) > file
		else
			printf("\trtab[%d     ] = %d\n", i, rtab[i]) > file
	}
	printf("}\n") > file
}


Default missing 'PR |' episode descriptions to Title

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode type, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-.[[:digit:]]{3,4}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2) + 0
    shortURL = episodeURL
    sub (/.*watch/,"watch",shortURL)
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeURL ~ /-c-x[[:digit:]]{3,4}$|montme-c-01001|richard-sammel-inetrview/)
        episodeType = "X"
    # Take care of extra fields introduced in February 2020
    titleString = substr ($0, match ($0, /strong title/))
    split (titleString,fld,"\"")
    episodeTitle = fld[2]
    if (episodeTitle ~ /^PR \|/) {
        # Episode is a Trailer (i.e. First look), set episodeType to "T"
        episodeType = "T"
        print episodeTitle "\t" shortURL >> ERROR_FILE
        printf ("-1") >> NUM_EPISODES_FILE
    }
    #
    # If start of episodeTitle == seriesTitle ": ", remove the redundant part.
    if ((match (episodeTitle, seriesTitle ": ")) == 1) {
        episodeTitle = substr(episodeTitle, RLENGTH + 1)
    }
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number, and correct if necessary
# Detective Montalbano requires special processing for page 2 episodeNumber
/<h4 class="/ {
    episodeNumber = $0
    sub (/.*Episode /,"",episodeNumber)
    sub (/<\/span>.*/,"",episodeNumber)
    episodeNumber = episodeNumber + 0
    if (seriesTitle == "Detective Montalbano" && page2 == "yes") {
        oldEpisodeNumber = episodeNumber
        episodeNumber += 24
        printf ("==> Changed E%02d to %s%02d: %s\n", oldEpisodeNumber, episodeType, \
                episodeNumber, shortURL) >> ERROR_FILE
    }
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/
<span class=/,/<\/div>/ {
    if ($0 ~ /
 Changed %s00 to %s%02d: %s\n", episodeType, episodeType,
                        episodeNumberFromURL, shortURL) >> ERROR_FILE
                episodeNumber = episodeNumberFromURL
            } else {
                printf ("==> Episode number is 00: %s\n", shortURL) >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
                SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeType, episodeNumber, \
                episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "") {
            if (episodeType == "T") {
                episodeDescription = episodeTitle
            } else {
            print "==> No description: " shortURL >> ERROR_FILE
            }
        }
        sub (/^PR \| /,"",episodeDescription)
        print episodeDescription >> EPISODE_INFO_FILE
    }
}

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode type, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-.[[:digit:]]{3,4}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2) + 0
    shortURL = episodeURL
    sub (/.*watch/,"watch",shortURL)
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeURL ~ /-c-x[[:digit:]]{3,4}$|montme-c-01001|richard-sammel-inetrview/)
        episodeType = "X"
    # Take care of extra fields introduced in February 2020
    titleString = substr ($0, match ($0, /strong title/))
    split (titleString,fld,"\"")
    episodeTitle = fld[2]
    if (episodeTitle ~ /^PR \|/) {
        # Episode is a Trailer (i.e. First look), set episodeType to "T"
        episodeType = "T"
        print episodeTitle "\t" shortURL >> ERROR_FILE
        printf ("-1") >> NUM_EPISODES_FILE
    }
    #
    # If start of episodeTitle == seriesTitle ": ", remove the redundant part.
    if ((match (episodeTitle, seriesTitle ": ")) == 1) {
        episodeTitle = substr(episodeTitle, RLENGTH + 1)
    }
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number, and correct if necessary
# Detective Montalbano requires special processing for page 2 episodeNumber
/<h4 class="/ {
    episodeNumber = $0
    sub (/.*Episode /,"",episodeNumber)
    sub (/<\/span>.*/,"",episodeNumber)
    episodeNumber = episodeNumber + 0
    if (seriesTitle == "Detective Montalbano" && page2 == "yes") {
        oldEpisodeNumber = episodeNumber
        episodeNumber += 24
        printf ("==> Changed E%02d to %s%02d: %s\n", oldEpisodeNumber, episodeType, \
                episodeNumber, shortURL) >> ERROR_FILE
    }
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/
<span class=/,/<\/div>/ {
    if ($0 ~ /
 Changed %s00 to %s%02d: %s\n", episodeType, episodeType,
                        episodeNumberFromURL, shortURL) >> ERROR_FILE
                episodeNumber = episodeNumberFromURL
            } else {
                printf ("==> Episode number is 00: %s\n", shortURL) >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
                SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeType, episodeNumber, \
                episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " shortURL >> ERROR_FILE
        sub (/^PR \| /,"",episodeDescription)
        print episodeDescription >> EPISODE_INFO_FILE
    }
}


pass on less that three, order by howmany

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology namespace they are from. (curi prefix )
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
    print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
    return substr(str,2,length(str)-2)
}

# break URI into tokens to use a a path in
# multi-dim array pointing to a curie-prefix
# base URI include "!#&-=?_"
# which are all also present in localID as well
function tokenize (str){
    # b/c found in local IDs leave .:
    gsub(/[!&:#=?/_-]+/, SUBSEP, str)
    # discard the first token it is only low info content  http|https|ftp
    # discard any empty trailing tokens
    tail = match(str, ".*" SUBSEP)  # side effects RSTART & RLENGTH
    # print "tail = " tail " tail length = " RLENGTH
    finish = 0
    start = index(str, SUBSEP) + 1
    if (tail > 0)
        finish = RLENGTH - start
    else
        finish = length(str) - start + 1
    return substr(str, start, finish)
}

# remove final token from token path delimited with SUBSEP
# this ends up matching chars included in localID
# ._-;():%#,?=@*\&+   (yea ... nope)

function detail(key,   start) {
    start = match(key, ".+" SUBSEP)
    if (start == 1)
        return substr(key, 1, RLENGTH -1)
    else
        return key
}

# for output, keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
    gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
    gsub(/^_*|_*$/, "",str)
    gsub(/__*/,"_",str)
    return str
}

# Replace IRI with CuriePrefix when possible
# Otherwise whinge and return a gv printable version of the original
function contract(uri,  u){
    u = tokenize(uri)

    # sometimes uri is already correct as is
    # if (index(u, SUBSEP) > 0)
    #    u = detail(u)
    # if (u == "") {
    #    printf("ERROR contracting %s\n", uri) > "/dev/stderr"
    #    printf("ERROR tokenizeing %s\n", tokenize(uri)) > "/dev/stderr"
    #    printf("ERROR detailing   %s\n", detail(tokenize(uri))) > "/dev/stderr"
    #    # exit(1)  # while testing
    # }

    # shorten till longest uri in curi map is found (or not)
    while(!(u in prefix) && index(u, SUBSEP)>0)
        u = detail(u)

    if(u in prefix)
        return prefix[u]
    else{
        # printf("WARN  %s\n", uri) > "/dev/stderr"  # too many
        # printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
        warn[uri]++
        return simplify(uri)
    }
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
    split(uri, b, "/")
    p = b[length(b)]
    anchor = match(p, "#")
    if(anchor > 0)
        p = substr(p, anchor+1)
    return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123
# is this still needed circa 2020?
function deoboify(curie,  a){
    split(curie, a , ":")
    if(match(a[2], a[1]"_") > 0){
        sub("_", ":", a[2]);
        curie = a[2]
    }
    return curie
}

BEGIN{
    # exceptions
    prefix["BNODE"]="BNODE"  # is a fixed point
    prefix[tokenize("https://monarchinitiative.org/.well-known/genid")]="BNODE"
    prefix[tokenize("https://archive.monarchinitiative.org/")]="MonarchArchive"
    ############################################################
    # not expected to be added to curie map
    # in HPOA
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part=")]="NCBIBook"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/books/NBK")]="NCBINBK"

    ############################################################
    # Often re-visit whether these exceptions are still necessary
    # they may have been moved into the curie prefix mapping file
    # or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

    # in mgi/impc
    prefix[tokenize("https://www.mousephenotype.org/")]="IMPC"
    # in IMPC (not httpS  --- hmm all three IRI exist)
    # note in curie_map IMPC is:
    # http://www.mousephenotype.org/data/genes/

    # in panther
    # prefix[tokenize("http://identifiers.org/wormbase/"])="WormBase"
    # note in curie_map WormBase is
    # 'https://www.wormbase.org/get?name='

    # various, transient?
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/gene/")]="NCBIGene"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/genome/")]="NCBIGenome"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/assembly?term=")]="NCBIAssembly"
    # kegg
    prefix[tokenize("http://www.genome.jp/kegg/pathway/map/")]="KEGG-img"
    # in EOM
    prefix[tokenize("https://elementsofmorphology.nih.gov/images/terms/")]="EOM_IMG"
    # prefix[tokenize("http://elementsofmorphology.nih.gov/index.cgi?tid=")]="EOM"  # w/o httpS

    # playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
    # and their IRI are unroutable, i.e in same machine (or local net)
    # this is motivated in part by not wanting to publish
    # urls to our own servers which go nowhere.
    # prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
    # prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
    # prefix["http://0.0.0.0/"]="LCL"                              # ID halfway house

    # LCL's obvious down side is zero options for global collisions
    # best you can do is go the uuid which is ugly, or
    # belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# convert the base url to a path in a multi-dimensional array
# (FNR == NR) && /^'[fht]\+tp[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
    split($0, arr, "'")
    if(arr[2]=="")
        arr[2]="BASE"
    prefix[tokenize(arr[4])]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    o = contract(trim($3))
    edgelist[s,ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
    s = contract(trim($1))
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist[s, ns ":" p, "LITERAL"]++
    done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
    # once our bnode syntax is uniform we may want more info out of it
    # s = contract(($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    ### Object (like subject)
    o = contract(trim($3))
    edgelist["BNODE",ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    #o = contract($3))
    edgelist[s,ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
    # s = contract($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    # o = contract($3)
    edgelist["BNODE",ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {  # "balance quote for wonky hiliter
    # s = contract($1)
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist["BNODE", ns ":" p, "LITERAL"]++
    done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
    printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
    # output the graphviz directed graph
    print "digraph {"
    print "rankdir=LR;"
    print "charset=\"utf-8\";"
    # sort the edges to facilitate future inter dot comparisons
    n=asorti(edgelist, sortlist)
    for (i=1; i<=n; i++) {
        edge=sortlist[i]
        split(edge ,spo, SUBSEP);
        print simplify(spo[1]) " -> " simplify(spo[3]) \
        " [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
    }
    print "LITERAL [shape=record];"
    print "labelloc=\"t\";"
    title = final(FILENAME)
    datestamp = strftime("%Y%m%d", systime())
    print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
    print "}"

    ######################################################
    # dump a summary of significant numbers of unresolved-curi uri  to stderr
    # FNR gives at least half the number ot URI attempted to resolve for %s
    # FILENAME gives filename we know where they were generated
    # have warn[uri] = count
    toofew = 3

    n = asorti(warn, wsort, "@val_num_desc")
    for(i=1;i<=n;i++){
        if(warn[wsort[i]] >= toofew)
            printf("%s\t%s\t%i\n", FILENAME, wsort[i], warn[wsort[i]]) > "/dev/stderr"
        else
            i = n + 1
    }
}

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology namespace they are from. (curi prefix )
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
    print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
    return substr(str,2,length(str)-2)
}

# break URI into tokens to use a a path in
# multi-dim array pointing to a curie-prefix
# base URI include "!#&-=?_"
# which are all also present in localID as well
function tokenize (str){
    # b/c found in local IDs leave .:
    gsub(/[!&:#=?/_-]+/, SUBSEP, str)
    # discard the first token it is only low info content  http|https|ftp
    # discard any empty trailing tokens
    tail = match(str, ".*" SUBSEP)  # side effects RSTART & RLENGTH
    # print "tail = " tail " tail length = " RLENGTH
    finish = 0
    start = index(str, SUBSEP) + 1
    if (tail > 0)
        finish = RLENGTH - start
    else
        finish = length(str) - start + 1
    return substr(str, start, finish)
}

# remove final token from token path delimited with SUBSEP
# this ends up matching chars included in localID
# ._-;():%#,?=@*\&+   (yea ... nope)

function detail(key,   start) {
    start = match(key, ".+" SUBSEP)
    if (start == 1)
        return substr(key, 1, RLENGTH -1)
    else
        return key
}

# for output, keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
    gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
    gsub(/^_*|_*$/, "",str)
    gsub(/__*/,"_",str)
    return str
}

# Replace IRI with CuriePrefix when possible
# Otherwise whinge and return a gv printable version of the original
function contract(uri,  u){
    u = tokenize(uri)

    # sometimes uri is already correct as is
    # if (index(u, SUBSEP) > 0)
    #    u = detail(u)
    # if (u == "") {
    #    printf("ERROR contracting %s\n", uri) > "/dev/stderr"
    #    printf("ERROR tokenizeing %s\n", tokenize(uri)) > "/dev/stderr"
    #    printf("ERROR detailing   %s\n", detail(tokenize(uri))) > "/dev/stderr"
    #    # exit(1)  # while testing
    # }

    # shorten till longest uri in curi map is found (or not)
    while(!(u in prefix) && index(u, SUBSEP)>0)
        u = detail(u)

    if(u in prefix)
        return prefix[u]
    else{
        # printf("WARN  %s\n", uri) > "/dev/stderr"  # too many
        # printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
        warn[uri]++
        return simplify(uri)
    }
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
    split(uri, b, "/")
    p = b[length(b)]
    anchor = match(p, "#")
    if(anchor > 0)
        p = substr(p, anchor+1)
    return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123
# is this still needed circa 2020?
function deoboify(curie,  a){
    split(curie, a , ":")
    if(match(a[2], a[1]"_") > 0){
        sub("_", ":", a[2]);
        curie = a[2]
    }
    return curie
}

BEGIN{
    # exceptions
    prefix["BNODE"]="BNODE"  # is a fixed point
    prefix[tokenize("https://monarchinitiative.org/.well-known/genid")]="BNODE"
    prefix[tokenize("https://archive.monarchinitiative.org/")]="MonarchArchive"
    ############################################################
    # not expected to be added to curie map
    # in HPOA
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part=")]="NCBIBook"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/books/NBK")]="NCBINBK"

    ############################################################
    # Often re-visit whether these exceptions are still necessary
    # they may have been moved into the curie prefix mapping file
    # or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

    # in mgi/impc
    prefix[tokenize("https://www.mousephenotype.org/")]="IMPC"
    # in IMPC (not httpS  --- hmm all three IRI exist)
    # note in curie_map IMPC is:
    # http://www.mousephenotype.org/data/genes/

    # in panther
    # prefix[tokenize("http://identifiers.org/wormbase/"])="WormBase"
    # note in curie_map WormBase is
    # 'https://www.wormbase.org/get?name='

    # various, transient?
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/gene/")]="NCBIGene"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/genome/")]="NCBIGenome"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/assembly?term=")]="NCBIAssembly"
    # kegg
    prefix[tokenize("http://www.genome.jp/kegg/pathway/map/")]="KEGG-img"
    # in EOM
    prefix[tokenize("https://elementsofmorphology.nih.gov/images/terms/")]="EOM_IMG"
    # prefix[tokenize("http://elementsofmorphology.nih.gov/index.cgi?tid=")]="EOM"  # w/o httpS

    # playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
    # and their IRI are unroutable, i.e in same machine (or local net)
    # this is motivated in part by not wanting to publish
    # urls to our own servers which go nowhere.
    # prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
    # prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
    # prefix["http://0.0.0.0/"]="LCL"                              # ID halfway house

    # LCL's obvious down side is zero options for global collisions
    # best you can do is go the uuid which is ugly, or
    # belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# convert the base url to a path in a multi-dimensional array
# (FNR == NR) && /^'[fht]\+tp[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
    split($0, arr, "'")
    if(arr[2]=="")
        arr[2]="BASE"
    prefix[tokenize(arr[4])]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    o = contract(trim($3))
    edgelist[s,ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
    s = contract(trim($1))
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist[s, ns ":" p, "LITERAL"]++
    done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
    # once our bnode syntax is uniform we may want more info out of it
    # s = contract(($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    ### Object (like subject)
    o = contract(trim($3))
    edgelist["BNODE",ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    #o = contract($3))
    edgelist[s,ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
    # s = contract($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    # o = contract($3)
    edgelist["BNODE",ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {  # "balance quote for wonky hiliter
    # s = contract($1)
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist["BNODE", ns ":" p, "LITERAL"]++
    done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
    printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
    # output the graphviz directed graph
    print "digraph {"
    print "rankdir=LR;"
    print "charset=\"utf-8\";"
    # sort the edges to facilitate future inter dot comparisons
    n=asorti(edgelist, sortlist)
    for (i=1; i<=n; i++) {
        edge=sortlist[i]
        split(edge ,spo, SUBSEP);
        print simplify(spo[1]) " -> " simplify(spo[3]) \
        " [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
    }
    print "LITERAL [shape=record];"
    print "labelloc=\"t\";"
    title = final(FILENAME)
    datestamp = strftime("%Y%m%d", systime())
    print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
    print "}"

    ######################################################
    # dump a summary of significant numbers of unresolved-curi uri  to stderr
    # FNR gives at least half the number ot URI attempted to resolve for %s
    # FILENAME gives filename we know where they were generated
    # have warn[uri] = count
    toofew = 10

    n = asorti(warn, wsort, "@ind_str_desc")
    for(i=1;i<=n;i++){
        if(warn[wsort[i]] > toofew)
            printf("%s\t%s\t%i\n", FILENAME, wsort[i], warn[wsort[i]]) > "/dev/stderr"
        else
            i = n + 1
    }
}


Make sure to not add empty strings into file list.

	modified:   code/awk/generate_source_list_from_make_output.awk

#!/usr/bin/awk -f
BEGIN{
    # This program generates source list to be used in `CMakeLists.txt`
    # or elsewhere using `foo.txt` produced by `make >foo.txt 2>&1` as input.

    array_index = 0;
    split("", dir_array);
    current_dir = "";
    new_filename = "";
    current_filename = "";
    entering_directory_regex = "Entering directory '";
    cpp_regex = "\\<[a-zA-Z0-9_-]+\\.cpp\\>";
    c_regex = "\\<[a-zA-Z0-9_-]+\\.c\\>";
} # BEGIN

function get_filename()
{
    # OK, there's a `.cpp` file on this line.
    cpp_filename = substr($0, RSTART, RLENGTH);
    new_filename = current_dir "/" cpp_filename;

    if (new_filename != current_filename && new_filename != "")
    {
        current_filename = new_filename;

        # find out if there's a matching source file.
        if (system("test -f " current_filename) == 0)
        {
            # print the filename.
            print current_filename;
        }

        # find out if there's a matching `*.h` header file too.
        header_filename = gensub(/(^.*\.)c(|pp)$/, "\\1h", "g", current_filename);

        if (system("test -f " header_filename) == 0)
        {
            # header exists, so print its filename too!
            print header_filename;
        }
    }
}
{
    entering_directory_match_index = match($0, entering_directory_regex);
    cpp_match_index = match($0, cpp_regex);
    c_match_index = match($0, c_regex);

    if (entering_directory_match_index > 0)
    {
        # OK, we are entering a new directory on this line.
        junk_part_length = entering_directory_match_index + length(entering_directory_regex);

        # read current dir.
        current_dir = substr($0, junk_part_length, length($0) - junk_part_length);

        # store current dir into array.
        dir_array[array_index++] = current_dir "/";
    }
    else if (cpp_match_index > 0)
    {
        match($0, cpp_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
    else if (c_match_index > 0)
    {
        match($0, c_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
}
END{
    n_dirs = array_index;
    asort(dir_array);

    print "";
    print "include_directories(";

    for (i = 0; i < n_dirs; i++)
    {
        print dir_array[i];
    }
} # END

#!/usr/bin/awk -f
BEGIN{
    # This program generates source list to be used in `CMakeLists.txt`
    # or elsewhere using `foo.txt` produced by `make >foo.txt 2>&1` as input.

    array_index = 0;
    split("", dir_array);
    current_dir = "";
    new_filename = "";
    current_filename = "";
    entering_directory_regex = "Entering directory '";
    cpp_regex = "\\<[a-zA-Z0-9_-]+\\.cpp\\>";
    c_regex = "\\<[a-zA-Z0-9_-]+\\.c\\>";
} # BEGIN

function get_filename()
{
    # OK, there's a `.cpp` file on this line.
    cpp_filename = substr($0, RSTART, RLENGTH);
    new_filename = current_dir "/" cpp_filename;

    if (new_filename != current_filename)
    {
        current_filename = new_filename;

        # find out if there's a matching source file.
        if (system("test -f " current_filename) == 0)
        {
            # print the filename.
            print current_filename;
        }

        # find out if there's a matching `*.h` header file too.
        header_filename = gensub(/(^.*\.)c(|pp)$/, "\\1h", "g", current_filename);

        if (system("test -f " header_filename) == 0)
        {
            # header exists, so print its filename too!
            print header_filename;
        }
    }
}
{
    entering_directory_match_index = match($0, entering_directory_regex);
    cpp_match_index = match($0, cpp_regex);
    c_match_index = match($0, c_regex);

    if (entering_directory_match_index > 0)
    {
        # OK, we are entering a new directory on this line.
        junk_part_length = entering_directory_match_index + length(entering_directory_regex);

        # read current dir.
        current_dir = substr($0, junk_part_length, length($0) - junk_part_length);

        # store current dir into array.
        dir_array[array_index++] = current_dir "/";
    }
    else if (cpp_match_index > 0)
    {
        match($0, cpp_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
    else if (c_match_index > 0)
    {
        match($0, c_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
}
END{
    n_dirs = array_index;
    asort(dir_array);

    print "";
    print "include_directories(";

    for (i = 0; i < n_dirs; i++)
    {
        print dir_array[i];
    }
} # END