add_utils.awk	-- allow street num w/trailing letter

function AU_init(addr_info, us_states, us_states_long, towns_a2q, towns_r2q, st_types_2qry, dirs_2qry, ords_2qry,    err, ad_counts, ad) {

	err = 0

	# check that we got all the data we need
	ad_counts["n_us_states"] = AU_get_addr_data(addr_info, "us_states", us_states)

	ad_counts["n_towns_a2q"] = AU_get_addr_data(addr_info, "towns_a2q", towns_a2q)
	ad_counts["n_st_types_2qry"] = AU_get_addr_data(addr_info, "st_types_2qry", st_types_2qry)
	ad_counts["n_dirs_2qry"] = AU_get_addr_data(addr_info, "dirs_2qry", dirs_2qry)
	ad_counts["n_ords_2qry"] = AU_get_addr_data(addr_info, "ords_2qry", ords_2qry)

	ad_counts["n_towns_r2q"] = AU_get_addr_data(addr_info, "towns_r2q", towns_r2q)

	for(ad in ad_counts){
		if(ad_counts[ad] == 0){
			printf("ERROR: %s no \"%s\" data\n", ai_file, substr(ad, 3)) > "/dev/stderr"
			err = 1
		}
	}
	if(err)
		return err

	# create a map of full state names
	for(s in us_states)
		us_states_long[us_states[s]] = s

	return err
}

function AU_parse(options, addr, addr_ary, states, states_long, towns, st_types, dirs, st_ords,   nf, ary, i, f_st, f_twn, nf2, ary2, b1, e1, name, street, quals, town, state, work) {

	addr_ary["status"] = "B"
	addr_ary["emsg"  ] = ""
	addr_ary["name"  ] = ""
	addr_ary["street"] = ""
	addr_ary["quals" ] = ""
	addr_ary["town"  ] = ""
	addr_ary["state" ] = ""

	nf = split(addr, ary, ",")
	for(i = 1; i <= nf; i++){
		sub(/^ */, "", ary[i])
		sub(/ *$/, "", ary[i])
	}

	# Some geocoders are US only so the last field is the state or state zip
	if(options["rply"]){
		if(!options["us_only"]){
			if(ary[nf] == "United States of America")
				nf--
			else{
				addr_ary["emsg"] = "not.US"
				return 1
			}
		}
	}

	if(!options["rply"]){
		# As a convenience to the user, given that nearly all the addresses
		# are in small set of nearby towns, the minimal query address can be
		#
		#	street, TOWN
		#
		# where TOWN is an abbreviation for town, state. Example: PA -> Palo Alto, CA
		if(nf < 2){
			addr_ary["emsg"] = "short.addr.1"
			return 1
		}
		# expand town if needed
		if(ary[nf] in towns){
			nf2 = split(towns[ary[nf]], ary2, ",")
			if(nf2 != 2){
				addr_ary["emsg"] = "bad.town"
				return  1
			}
			for(i = 1; i <= nf2; i++){
				sub(/^  */, "", ary2[i])
				sub(/  *$/, "", ary2[i])
			}
			ary[nf] = ary2[1]
			ary[nf+1] = ary2[2]
			nf++
		}
	}

	# At this point, all addresses must have at least three fields: street, town, state
	if(nf < 3){
		addr_ary["emsg"] = "short.addr.2"
		return 1
	}

	# check for a valid state, ignoring zip code
	if(substr(ary[nf], 1, 2) in states)
		state = ary[nf]
	else if(ary[nf] in states_long)
		state = ary[nf]
	else{
		addr_ary["emsg"] = sprintf("bad.US.state.%s", ary[nf])
		return 1
	}

	town = ary[nf-1]

	# find the street:
	# street is the LAST element in ary[1:nf-1] that begins with an integer and has more
	# than one word so, 200 by itself is not a street, but 200 Broadway is
	f_st = 0
	for(i = 1; i < nf - 1; i++){
		nf2 = split(ary[i], ary2, /  */)
		if(ary2[1] ~ /^[1-9][0-9]*$/ && nf2 > 1){
			f_st = i
		}else if(ary2[1] ~ /^[1-9][0-9]*[A-Z]$/ && nf2 > 1){
			f_st = i
		}
	}
	if(f_st == 0){
		addr_ary["emsg"] = "no.street.num"
		return 1
	}

	# The name is what precedes f_st.  if f_st == 1, set name to user supplied default value
	if(f_st == 1)
		name = options["no_name"]
	else{
		name = ary[1]
		for(i = 2; i < f_st; i++)
			name = name ", " ary[i]
	}

	# Street qualifiers are those fields, if any, that follow f_st and precede nf - 1
	if(nf - f_st > 2){
		quals = ary[f_st+1]
		for(i = f_st+2; i < nf - 1; i++)
			quals = quals ", " ary[i]
	}else
		quals = ""

	# clean up street
	nf2 = split(ary[f_st], ary2, /  */)
	if(nf2 < 2){
		addr_ary["emsg"] = "short.street"
		return 1
	}

	# If a word in street begins w/lower case, upper case it.  Convert, say el Camino to El Camino, etc
	for(i = 2; i <= nf2; i++){
		if(substr(ary2[i], 1, 1) ~ /^[a-z]/)
			ary2[i] = toupper(substr(ary2[i], 1, 1)) substr(ary2[i], 2)
	}

	# Street should be num [ dir ] str [ st ] [ dir ]
	street = ary2[1]
	if(options["do_subs"]){
		# TODO: generalize, as I've now seen an address w/3 words in dir, st_types
		# This is amusing.  What is South Court? Is it S. Court or South Ct.  No idea so leave such streets in long form
		if(nf2 == 3 && (ary2[2] in dirs) && (ary2[3] in st_types)){
			for(i = 2; i <= nf2; i++)
				street = street " " ary2[i]
		}else{
			# handle any leading direction
			if(ary2[2] in dirs){
				street = street " " dirs[ary2[2]]
				b1 = 3
			}else
				b1 = 2

			# detect trailing direction
			e1 = ary2[nf2] in dirs ? nf2-1 : nf2
			for(i = b1; i < e1; i++){
				if(i == b1)
					street = street ((ary2[i] in st_ords) ? (" " st_ords[ary2[i]]) : (" " ary2[i]))
				else
					street = street " " ary2[i]
			}
			street = street ((ary2[e1] in st_types) ? (" " st_types[ary2[e1]]) : (" " ary2[e1]))
			# handle any trailing direction
			if(e1 < nf2)
				street = street " " dirs[ary2[nf2]]
		}
	}else{
		for(i = 2; i <= nf2; i++)
			street = street " " ary2[i]
	}

	if(options["rply"]){
		if(town in towns)
			town = towns[town]
	}

	addr_ary["status"] = "G"
	addr_ary["name"  ] = name
	addr_ary["street"] = street
	addr_ary["quals" ] = quals
	addr_ary["town"  ] = town
	addr_ary["state" ] = state

	return 0
}

function AU_get_addr_data(addr_info, key, data,   k, keys, nk, i) {

	n_data = 0
	for(k in addr_info){
		nk = split(k, keys, SUBSEP)
		if(keys[1] == key){
			data[keys[2]] = addr_info[k]
			n_data++
		}
	}
	return n_data
}

function AU_match(options, ref, cand,   nc_fields, c_fields, nr_fields, r_fields, i, nc_rtab, c_rtab, nr_rtab, r_rtab, nr_nwords, r_nwords, nc_nwords, c_nwords) {

	if(options["verbose"]){
		printf("ref.name    = %s\n", ref["name"]) > "/dev/stderr"
		printf("ref.street  = %s\n", ref["street"]) > "/dev/stderr"
		printf("ref.town    = %s\n", ref["town"]) > "/dev/stderr"
		printf("ref.state   = %s\n", ref["state"]) > "/dev/stderr"
		printf("cand.name   = %s\n", cand["name"]) > "/dev/stderr"
		printf("cand.street = %s\n", cand["street"]) > "/dev/stderr"
		printf("cand.town   = %s\n", cand["town"]) > "/dev/stderr"
		printf("cand.state  = %s\n", cand["state"]) > "/dev/stderr"
	}

	cand["emsg"] = ""
	if(cand["state"] != ref["state"]){
		if(options["ign_zip"]){
			if(substr(cand["state"], 1, 2) != substr(ref["state"], 1, 2)){
				cand["emsg"] = "states.ign_zip.diff"
				return 0
			}
		}else{
			cand["emsg"] = "states.diff"
			return 0
		}
	}

	if(cand["town"] != ref["town"]){
		cand["emsg"] = "towns.diff"
		return 0
	}

	# street numbers and/or ranges, etc is 1st field
	nc_fields = split(cand["street"], c_fields, /  */)
	nr_fields = split(ref["street"], r_fields, /  */)

	# check that non-number parts of streets agree
	if(nc_fields != nr_fields){
		cand["emsg"] = "num.st.fields.diff"
		return 0
	}
	for(i = 2; i <= nc_fields; i++){
		if(c_fields[i] != r_fields[i]){
			cand["emsg"] = sprintf("st.field.%d.diff|%s|%s", i, c_fields[i], r_fields[i])
			return 0
		}
	}

	nc_rtab = AU_get_rtab(c_fields[1], c_rtab)
	if(nc_rtab == 0){
		cand["emsg"] = "nc_rtab.is.zero"
		return 0
	}

	nr_rtab = AU_get_rtab(r_fields[1], r_rtab)
	if(nr_rtab == 0){
		cand["emsg"] = "nr_rtab.is.zero"
		return 0
	}

	if(!AU_rtabs_intersect(nc_rtab, c_rtab, nr_rtab, r_rtab)){
		cand["emsg"] = "rtab.no.ovlp"
		return 0
	}

	# deal with names
	if(cand["name"] == ref["name"]){
		return 3
	}else if(cand["name"] == ""){	# this is an unannotated address
		return 1
	}else if(ref["name"] == options["no_name"]){	# many destinations are just street, town, state
		return 1
	}else{	# check if 1st words match; but no matter what this is a match
		nc_nwords = split(cand["name"], c_nwords, /  */)
		nr_nwords = split(ref["name"], r_nwords, /  */)
		return c_nwords[1] == r_nwords[1] ? 2 : 1
	}
}

function AU_get_rtab(street, rtab,   n_rtab, i, nw, work, l_pfx, e_work2) {

	n_rtab = split(street, rtab, ";")
	for(i = 1; i <= n_rtab; i++){
		nw = split(rtab[i], work, "-")
		if(nw > 1){
			rtab[i, 1] = work[1] + 0	# force number
			rtab[i, 2] = work[2] + 0	# force number

			if(rtab[i,1] > rtab[i,2]){
				# Is this a "delta" range where the 2d value shows only the different address suffix?
				# If so try to fix it
				# For example 945-51 would be expanded to 945-951 which is good, but
				# 945-37 would be expanded to 945-937 which is still wrong.
				l_pfx = length(work[1]) - length(work[2])
				e_work2 = ((l_pfx > 0) ? ((substr(work[1], 1, l_pfx) work[2])) : work[2]) + 0
				# did we fix it?
				if(rtab[i, 1] > e_work2){
					printf("ERROR: bad range 1st precedes 2nd: %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
					return 0
				}else
					rtab[i, 2] = e_work2
			}

			rtab[i, "rng"] = 1
			if(rtab[i, 1] % 2 != rtab[i, 2] % 2){
				printf("ERROR: bad range: odd/even %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
				return 0
			}
		}else{
			rtab[i] = work[1] + 0	# force number
			rtab[i, "rng"] = 0
		}
	}
	return n_rtab
}

function AU_rtabs_intersect(n_rtab1, rtab1, n_rtab2, rtab2,   i, j) {

	# the most common case: number v number
	if(n_rtab1 == 1 && !rtab1[1, "rng"] && n_rtab2 == 1 && !rtab2[1, "rng"])
		return rtab1[1] == rtab2[1]

	for(i = 1; i <= n_rtab1; i++){
		for(j = 1; j <= n_rtab2; j++){
			if(!rtab1[i, "rng"]){
				if(!rtab2[j, "rng"]){	# number v number
					if(rtab1[i] == rtab2[j])
						return 1
				}else{			# number v range
					if(rtab1[i] >= rtab2[j, 1] && rtab1[i] <= rtab2[j, 2])
						return 1
				}
			}else if(!rtab2[j, "rng"]){	# range v number
				if(rtab2[j] >= rtab1[i, 1] && rtab2[j] <= rtab1[i, 2])
					return 1
			}else{				# range v range
				if(!(rtab1[i, 2] < rtab2[j, 1] || rtab1[i, 1] > rtab2[j, 2]))
					return 1
			}
		}
	}
	return 0
}

function AU_rtab_dump(file, n_rtab, rtab,   i) {

	printf("n_rtab = %d {\n", n_rtab) > file
	for(i = 1; i <= n_rtab; i++){
		if(rtab[i, "rng"])
			printf("\trtab[%d, 1-2] = %d-%d\n", i, rtab[i, 1], rtab[i, 2]) > file
		else
			printf("\trtab[%d     ] = %d\n", i, rtab[i]) > file
	}
	printf("}\n") > file
}

function AU_init(addr_info, us_states, us_states_long, towns_a2q, towns_r2q, st_types_2qry, dirs_2qry, ords_2qry,    err, ad_counts, ad) {

	err = 0

	# check that we got all the data we need
	ad_counts["n_us_states"] = AU_get_addr_data(addr_info, "us_states", us_states)

	ad_counts["n_towns_a2q"] = AU_get_addr_data(addr_info, "towns_a2q", towns_a2q)
	ad_counts["n_st_types_2qry"] = AU_get_addr_data(addr_info, "st_types_2qry", st_types_2qry)
	ad_counts["n_dirs_2qry"] = AU_get_addr_data(addr_info, "dirs_2qry", dirs_2qry)
	ad_counts["n_ords_2qry"] = AU_get_addr_data(addr_info, "ords_2qry", ords_2qry)

	ad_counts["n_towns_r2q"] = AU_get_addr_data(addr_info, "towns_r2q", towns_r2q)

	for(ad in ad_counts){
		if(ad_counts[ad] == 0){
			printf("ERROR: %s no \"%s\" data\n", ai_file, substr(ad, 3)) > "/dev/stderr"
			err = 1
		}
	}
	if(err)
		return err

	# create a map of full state names
	for(s in us_states)
		us_states_long[us_states[s]] = s

	return err
}

function AU_parse(options, addr, addr_ary, states, states_long, towns, st_types, dirs, st_ords,   nf, ary, i, f_st, f_twn, nf2, ary2, b1, e1, name, street, quals, town, state, work) {

	addr_ary["status"] = "B"
	addr_ary["emsg"  ] = ""
	addr_ary["name"  ] = ""
	addr_ary["street"] = ""
	addr_ary["quals" ] = ""
	addr_ary["town"  ] = ""
	addr_ary["state" ] = ""

	nf = split(addr, ary, ",")
	for(i = 1; i <= nf; i++){
		sub(/^ */, "", ary[i])
		sub(/ *$/, "", ary[i])
	}

	# Some geocoders are US only so the last field is the state or state zip
	if(options["rply"]){
		if(!options["us_only"]){
			if(ary[nf] == "United States of America")
				nf--
			else{
				addr_ary["emsg"] = "not.US"
				return 1
			}
		}
	}

	if(!options["rply"]){
		# As a convenience to the user, given that nearly all the addresses
		# are in small set of nearby towns, the minimal query address can be
		#
		#	street, TOWN
		#
		# where TOWN is an abbreviation for town, state. Example: PA -> Palo Alto, CA
		if(nf < 2){
			addr_ary["emsg"] = "short.addr.1"
			return 1
		}
		# expand town if needed
		if(ary[nf] in towns){
			nf2 = split(towns[ary[nf]], ary2, ",")
			if(nf2 != 2){
				addr_ary["emsg"] = "bad.town"
				return  1
			}
			for(i = 1; i <= nf2; i++){
				sub(/^  */, "", ary2[i])
				sub(/  *$/, "", ary2[i])
			}
			ary[nf] = ary2[1]
			ary[nf+1] = ary2[2]
			nf++
		}
	}

	# At this point, all addresses must have at least three fields: street, town, state
	if(nf < 3){
		addr_ary["emsg"] = "short.addr.2"
		return 1
	}

	# check for a valid state, ignoring zip code
	if(substr(ary[nf], 1, 2) in states)
		state = ary[nf]
	else if(ary[nf] in states_long)
		state = ary[nf]
	else{
		addr_ary["emsg"] = sprintf("bad.US.state.%s", ary[nf])
		return 1
	}

	town = ary[nf-1]

	f_st = 0

	# find the street:
	# street is the last element in ary[1:nf-1] that begins with an integer and has more
	# than one word so, 200 by itself is not a street, but 200 Broadway is
	f_st = 0
	for(i = 1; i < nf - 1; i++){
		nf2 = split(ary[i], ary2, /  */)
		if(ary2[1] ~ /^[1-9][0-9]*$/ && nf2 > 1){
			f_st = i
		}
	}
	if(f_st == 0){
		addr_ary["emsg"] = "no.street.num"
		return 1
	}

	# The name is what precedes f_st.  if f_st == 1, set name to user supplied default value
	if(f_st == 1)
		name = options["no_name"]
	else{
		name = ary[1]
		for(i = 2; i < f_st; i++)
			name = name ", " ary[i]
	}

	# Street qualifiers are those fields, if any, that follow f_st and precede nf - 1
	if(nf - f_st > 2){
		quals = ary[f_st+1]
		for(i = f_st+2; i < nf - 1; i++)
			quals = quals ", " ary[i]
	}else
		quals = ""

	# clean up street
	nf2 = split(ary[f_st], ary2, /  */)
	if(nf2 < 2){
		addr_ary["emsg"] = "short.street"
		return 1
	}

	# If a word in street begins w/lower case, upper case it.  Convert, say el Camino to El Camino, etc
	for(i = 2; i <= nf2; i++){
		if(substr(ary2[i], 1, 1) ~ /^[a-z]/)
			ary2[i] = toupper(substr(ary2[i], 1, 1)) substr(ary2[i], 2)
	}

	# Street should be num [ dir ] str [ st ] [ dir ]
	street = ary2[1]
	if(options["do_subs"]){
		# TODO: generalize, as I've now seen an address w/3 words in dir, st_types
		# This is amusing.  What is South Court? Is it S. Court or South Ct.  No idea so leave such streets in long form
		if(nf2 == 3 && (ary2[2] in dirs) && (ary2[3] in st_types)){
			for(i = 2; i <= nf2; i++)
				street = street " " ary2[i]
		}else{
			# handle any leading direction
			if(ary2[2] in dirs){
				street = street " " dirs[ary2[2]]
				b1 = 3
			}else
				b1 = 2

			# detect trailing direction
			e1 = ary2[nf2] in dirs ? nf2-1 : nf2
			for(i = b1; i < e1; i++){
				if(i == b1)
					street = street ((ary2[i] in st_ords) ? (" " st_ords[ary2[i]]) : (" " ary2[i]))
				else
					street = street " " ary2[i]
			}
			street = street ((ary2[e1] in st_types) ? (" " st_types[ary2[e1]]) : (" " ary2[e1]))
			# handle any trailing direction
			if(e1 < nf2)
				street = street " " dirs[ary2[nf2]]
		}
	}else{
		for(i = 2; i <= nf2; i++)
			street = street " " ary2[i]
	}

	if(options["rply"]){
		if(town in towns)
			town = towns[town]
	}

	addr_ary["status"] = "G"
	addr_ary["name"  ] = name
	addr_ary["street"] = street
	addr_ary["quals" ] = quals
	addr_ary["town"  ] = town
	addr_ary["state" ] = state

	return 0
}

function AU_get_addr_data(addr_info, key, data,   k, keys, nk, i) {

	n_data = 0
	for(k in addr_info){
		nk = split(k, keys, SUBSEP)
		if(keys[1] == key){
			data[keys[2]] = addr_info[k]
			n_data++
		}
	}
	return n_data
}

function AU_match(options, ref, cand,   nc_fields, c_fields, nr_fields, r_fields, i, nc_rtab, c_rtab, nr_rtab, r_rtab, nr_nwords, r_nwords, nc_nwords, c_nwords) {

	if(options["verbose"]){
		printf("ref.name    = %s\n", ref["name"]) > "/dev/stderr"
		printf("ref.street  = %s\n", ref["street"]) > "/dev/stderr"
		printf("ref.town    = %s\n", ref["town"]) > "/dev/stderr"
		printf("ref.state   = %s\n", ref["state"]) > "/dev/stderr"
		printf("cand.name   = %s\n", cand["name"]) > "/dev/stderr"
		printf("cand.street = %s\n", cand["street"]) > "/dev/stderr"
		printf("cand.town   = %s\n", cand["town"]) > "/dev/stderr"
		printf("cand.state  = %s\n", cand["state"]) > "/dev/stderr"
	}

	cand["emsg"] = ""
	if(cand["state"] != ref["state"]){
		if(options["ign_zip"]){
			if(substr(cand["state"], 1, 2) != substr(ref["state"], 1, 2)){
				cand["emsg"] = "states.ign_zip.diff"
				return 0
			}
		}else{
			cand["emsg"] = "states.diff"
			return 0
		}
	}

	if(cand["town"] != ref["town"]){
		cand["emsg"] = "towns.diff"
		return 0
	}

	# street numbers and/or ranges, etc is 1st field
	nc_fields = split(cand["street"], c_fields, /  */)
	nr_fields = split(ref["street"], r_fields, /  */)

	# check that non-number parts of streets agree
	if(nc_fields != nr_fields){
		cand["emsg"] = "num.st.fields.diff"
		return 0
	}
	for(i = 2; i <= nc_fields; i++){
		if(c_fields[i] != r_fields[i]){
			cand["emsg"] = sprintf("st.field.%d.diff|%s|%s", i, c_fields[i], r_fields[i])
			return 0
		}
	}

	nc_rtab = AU_get_rtab(c_fields[1], c_rtab)
	if(nc_rtab == 0){
		cand["emsg"] = "nc_rtab.is.zero"
		return 0
	}

	nr_rtab = AU_get_rtab(r_fields[1], r_rtab)
	if(nr_rtab == 0){
		cand["emsg"] = "nr_rtab.is.zero"
		return 0
	}

	if(!AU_rtabs_intersect(nc_rtab, c_rtab, nr_rtab, r_rtab)){
		cand["emsg"] = "rtab.no.ovlp"
		return 0
	}

	# deal with names
	if(cand["name"] == ref["name"]){
		return 3
	}else if(cand["name"] == ""){	# this is an unannotated address
		return 1
	}else if(ref["name"] == options["no_name"]){	# many destinations are just street, town, state
		return 1
	}else{	# check if 1st words match; but no matter what this is a match
		nc_nwords = split(cand["name"], c_nwords, /  */)
		nr_nwords = split(ref["name"], r_nwords, /  */)
		return c_nwords[1] == r_nwords[1] ? 2 : 1
	}
}

function AU_get_rtab(street, rtab,   n_rtab, i, nw, work, l_pfx, e_work2) {

	n_rtab = split(street, rtab, ";")
	for(i = 1; i <= n_rtab; i++){
		nw = split(rtab[i], work, "-")
		if(nw > 1){
			rtab[i, 1] = work[1] + 0	# force number
			rtab[i, 2] = work[2] + 0	# force number

			if(rtab[i,1] > rtab[i,2]){
				# Is this a "delta" range where the 2d value shows only the different address suffix?
				# If so try to fix it
				# For example 945-51 would be expanded to 945-951 which is good, but
				# 945-37 would be expanded to 945-937 which is still wrong.
				l_pfx = length(work[1]) - length(work[2])
				e_work2 = ((l_pfx > 0) ? ((substr(work[1], 1, l_pfx) work[2])) : work[2]) + 0
				# did we fix it?
				if(rtab[i, 1] > e_work2){
					printf("ERROR: bad range 1st precedes 2nd: %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
					return 0
				}else
					rtab[i, 2] = e_work2
			}

			rtab[i, "rng"] = 1
			if(rtab[i, 1] % 2 != rtab[i, 2] % 2){
				printf("ERROR: bad range: odd/even %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
				return 0
			}
		}else{
			rtab[i] = work[1] + 0	# force number
			rtab[i, "rng"] = 0
		}
	}
	return n_rtab
}

function AU_rtabs_intersect(n_rtab1, rtab1, n_rtab2, rtab2,   i, j) {

	# the most common case: number v number
	if(n_rtab1 == 1 && !rtab1[1, "rng"] && n_rtab2 == 1 && !rtab2[1, "rng"])
		return rtab1[1] == rtab2[1]

	for(i = 1; i <= n_rtab1; i++){
		for(j = 1; j <= n_rtab2; j++){
			if(!rtab1[i, "rng"]){
				if(!rtab2[j, "rng"]){	# number v number
					if(rtab1[i] == rtab2[j])
						return 1
				}else{			# number v range
					if(rtab1[i] >= rtab2[j, 1] && rtab1[i] <= rtab2[j, 2])
						return 1
				}
			}else if(!rtab2[j, "rng"]){	# range v number
				if(rtab2[j] >= rtab1[i, 1] && rtab2[j] <= rtab1[i, 2])
					return 1
			}else{				# range v range
				if(!(rtab1[i, 2] < rtab2[j, 1] || rtab1[i, 1] > rtab2[j, 2]))
					return 1
			}
		}
	}
	return 0
}

function AU_rtab_dump(file, n_rtab, rtab,   i) {

	printf("n_rtab = %d {\n", n_rtab) > file
	for(i = 1; i <= n_rtab; i++){
		if(rtab[i, "rng"])
			printf("\trtab[%d, 1-2] = %d-%d\n", i, rtab[i, 1], rtab[i, 2]) > file
		else
			printf("\trtab[%d     ] = %d\n", i, rtab[i]) > file
	}
	printf("}\n") > file
}


Add VOP_FOO_APV() which takes a pointer to the vop_vector.

This allows stacked or partitioned filesystems to say "Continue
the normal resolution from here", for instace from FFS to UFS.

Use VNASSERT() instead of KASSERT().


Help: enable 4-column reference tables

Help: enable 4-column reference tables

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead) {
    initAudioPlayer()
    initPager()
    Platform = Platform ? Platform : detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", Platform)                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s\n", "bi-di emulator", BiDiTerm ? BiDiTerm :
                "[N/A]")                                                \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "web browser", Option["browser"] != NONE ?
                Option["browser"] :"[NONE]")                            \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", join(Option["sls"], "+")) \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "translation engine", Option["engine"])    \
        sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "ip version", Option["ip-version"] ? Option["ip-version"] :
                "[DEFAULT]")                                            \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCES") "]"                           \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-S") ", " ansi("bold", "-list-engines")) RS \
        ins(2, "List available translation engines and exit.") RS       \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Translator options:" RS                                     \
        ins(1, ansi("bold", "-e ") ansi("underline", "ENGINE")          \
            ", " ansi("bold", "-engine ") ansi("underline", "ENGINE")) RS \
        ins(2, "Specify the translation engine to use.") RS             \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-identify")) RS                            \
        ins(2, "Language identification.") RS                           \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        ins(1, ansi("bold", "-no-autocorrect")) RS                      \
        ins(2, "Do not autocorrect. (if defaulted by the translation engine)") RS \
        ins(1, ansi("bold", "-no-bidi")) RS                             \
        ins(2, "Do not convert bidirectional texts.") RS                \
        ins(1, ansi("bold", "-bidi")) RS                                \
        ins(2, "Always convert bidirectional texts.") RS                \
        ins(1, ansi("bold", "-no-warn")) RS                             \
        ins(2, "Do not write warning messages to stderr.") RS           \
        ins(1, ansi("bold", "-dump")) RS                                \
        ins(2, "Print raw API response instead.") RS                    \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-speak")) RS                               \
        ins(2, "Listen to the original text.") RS                       \
        ins(1, ansi("bold", "-n ") ansi("underline", "VOICE")           \
            ", " ansi("bold", "-narrator ") ansi("underline", "VOICE")) RS \
        ins(2, "Specify the narrator, and listen to the translation.") RS \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        ins(1, ansi("bold", "-no-translate")) RS                        \
        ins(2, "Do not translate anything when using -speak.") RS       \
        ins(1, ansi("bold", "-download-audio")) RS                      \
        ins(2, "Download the audio to the current directory.") RS       \
        ins(1, ansi("bold", "-download-audio-as ") ansi("underline", "FILENAME")) RS \
        ins(2, "Download the audio to the specified file.") RS          \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view") ", " ansi("bold", "-no-pager")) RS \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        ins(1, ansi("bold", "-no-browser")) RS                          \
        ins(2, "Do not open the web browser.") RS                       \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        ins(1, ansi("bold", "-4") ", " ansi("bold", "-ipv4")            \
            ", " ansi("bold", "-inet4-only")) RS                        \
        ins(2, "Connect only to IPv4 addresses.") RS                    \
        ins(1, ansi("bold", "-6") ", " ansi("bold", "-ipv6")            \
            ", " ansi("bold", "-inet6-only")) RS                        \
        ins(2, "Connect only to IPv6 addresses.") RS                    \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-sl ") ansi("underline", "CODES")        \
            ", " ansi("bold", "-source ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-from ") ansi("underline", "CODES")) RS  \
        ins(2, "Specify the source language(s), joined by '+'.") RS     \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODES")        \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-to ") ansi("underline", "CODES")) RS    \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Text preprocessing options:" RS                             \
        ins(1, ansi("bold", "-j") ", " ansi("bold", "-join-sentence")) RS \
        ins(2, "Treat all arguments as one single sentence.") RS        \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists(ENVIRON["TRANS_DIR"] "/man/" Command ".1"))
        system("man " parameterize(ENVIRON["TRANS_DIR"] "/man/" Command ".1") SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, colNum, i, j, name, num, offset, r, rows, saveSortedIn,
                      t1, t2) {
    # number of language codes with stable support
    num = 0
    for (code in Locale) {
        # only show languages that are supported
        if (Locale[code]["supported-by"])
           num++
    }
    colNum = (Option["width"] >= 104) ? 4 : 3
    if (colNum == 4) {
        rows = int(num / 4) + (num % 4 ? 1 : 0)
        cols[0][0] = cols[1][0] = cols[2][0] = cols[3][0] = NULLSTR
    } else {
        rows = int(num / 3) + (num % 3 ? 1 : 0)
        cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    }
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        # only show languages that are supported
        if (Locale[code]["supported-by"]) {
            col = int(i / rows)
            append(cols[col], code)
            i++
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        if (colNum == 4) {  # 4-column
            offset = int((Option["width"] - 104) / 4)
            r = "┌" replicate("─", 25 + offset) "┬" replicate("─", 25 + offset) \
                "┬" replicate("─", 25 + offset) "┬" replicate("─", 25 + offset) "┐" RS
            for (i = 0; i < rows; i++) {
                r = r "│"
                for (j = 0; j < 4; j++) {
                    if (cols[j][i]) {
                        t1 = getDisplay(cols[j][i])
                        if (length(t1) > 17 + offset)
                            t1 = substr(t1, 1, 14 + offset) "..."
                        switch (cols[j][i]) { # fix rendered text width
                            case "sa":
                                t1 = sprintf(" %-"21+offset"s", t1)
                                break
                            case "he": case "dv":
                                t1 = sprintf(" %-"20+offset"s", t1)
                                break
                            case "bo": case "or": case "ur":
                                t1 = sprintf(" %-"19+offset"s", t1)
                                break
                            case "as": case "gom": case "mai":
                            case "gu": case "hi": case "bho":
                            case "ta": case "te": case "my":
                            case "ne": case "pa": case "km":
                            case "kn": case "yi": case "si":
                                t1 = sprintf(" %-"18+offset"s", t1)
                                break
                            case "lzh": case "yue":
                                t1 = sprintf(" %-"15+offset"s", t1)
                                break
                            case "ja": case "ko":
                                t1 = sprintf(" %-"14+offset"s", t1)
                                break
                            case "zh-CN": case "zh-TW":
                                t1 = sprintf(" %-"13+offset"s", t1)
                                break
                            default:
                                if (length(t1) <= 17+offset)
                                    t1 = sprintf(" %-"17+offset"s", t1)
                        }
                        switch (length(cols[j][i])) {
                            case 1: case 2: case 3: case 4:
                                t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                                break
                            case 5:
                                t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                                break
                            case 6:
                                t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                                break
                            case 7:
                                t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                                break
                            default:
                                t2 = ansi("bold", cols[j][i])
                        }
                        r = r t1 t2
                    } else
                        r = r sprintf("%"25+offset"s│", NULLSTR)
                }
                r = r RS
            }
            r = r "└" replicate("─", 25 + offset) "┴" replicate("─", 25 + offset) \
                "┴" replicate("─", 25 + offset) "┴" replicate("─", 25 + offset) "┘"
        } else {  # fixed-width 3-column
            r = "┌" replicate("─", 25) "┬" replicate("─", 25) "┬" replicate("─", 25) "┐" RS
            for (i = 0; i < rows; i++) {
                r = r "│"
                for (j = 0; j < 3; j++) {
                    if (cols[j][i]) {
                        t1 = getDisplay(cols[j][i])
                        if (length(t1) > 17)
                            t1 = substr(t1, 1, 14) "..."
                        switch (cols[j][i]) { # fix rendered text width
                            case "sa":
                                t1 = sprintf(" %-21s", t1)
                                break
                            case "he": case "dv":
                                t1 = sprintf(" %-20s", t1)
                                break
                            case "bo": case "or": case "ur":
                                t1 = sprintf(" %-19s", t1)
                                break
                            case "as": case "gom": case "mai":
                            case "gu": case "hi": case "bho":
                            case "ta": case "te": case "my":
                            case "ne": case "pa": case "km":
                            case "kn": case "yi": case "si":
                                t1 = sprintf(" %-18s", t1)
                                break
                            case "lzh": case "yue":
                                t1 = sprintf(" %-15s", t1)
                                break
                            case "ja": case "ko":
                                t1 = sprintf(" %-14s", t1)
                                break
                            case "zh-CN": case "zh-TW":
                                t1 = sprintf(" %-13s", t1)
                                break
                            default:
                                if (length(t1) <= 17)
                                    t1 = sprintf(" %-17s", t1)
                        }
                        switch (length(cols[j][i])) {
                            case 1: case 2: case 3: case 4:
                                t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                                break
                            case 5:
                                t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                                break
                            case 6:
                                t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                                break
                            case 7:
                                t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                                break
                            default:
                                t2 = ansi("bold", cols[j][i])
                        }
                        r = r t1 t2
                    } else
                        r = r sprintf("%25s│", NULLSTR)
                }
                r = r RS
            }
            r = r "└" replicate("─", 25) "┴" replicate("─", 25) "┴" replicate("─", 25) "┘"
        }
    } else {
        if (colNum == 4) {  # 4-column
            offset = int((Option["width"] - 104) / 4)
            r = "┌" replicate("─", 25 + offset) "┬" replicate("─", 25 + offset) \
                "┬" replicate("─", 25 + offset) "┬" replicate("─", 25 + offset) "┐" RS
            for (i = 0; i < rows; i++) {
                r = r "│"
                for (j = 0; j < 4; j++) {
                    if (cols[j][i]) {
                        t1 = getName(cols[j][i])
                        if (length(t1) > 17 + offset)
                            t1 = substr(t1, 1, 14 + offset) "..."
                        t1 = sprintf(" %-"17+offset"s", t1)
                        switch (length(cols[j][i])) {
                            case 1: case 2: case 3: case 4:
                                t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                                break
                            case 5:
                                t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                                break
                            case 6:
                                t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                                break
                            case 7:
                                t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                                break
                            default:
                                t2 = ansi("bold", cols[j][i])
                        }
                        r = r t1 t2
                    } else
                        r = r sprintf("%"25+offset"s│", NULLSTR)
                }
                r = r RS
            }
            r = r "└" replicate("─", 25 + offset) "┴" replicate("─", 25 + offset) \
                "┴" replicate("─", 25 + offset) "┴" replicate("─", 25 + offset) "┘"
        } else {  # fixed-width 3-column
            r = "┌" replicate("─", 25) "┬" replicate("─", 25) "┬" replicate("─", 25) "┐" RS
            for (i = 0; i < rows; i++) {
                r = r "│"
                for (j = 0; j < 3; j++) {
                    if (cols[j][i]) {
                        t1 = getName(cols[j][i])
                        if (length(t1) > 17)
                            t1 = substr(t1, 1, 14) "..."
                        t1 = sprintf(" %-17s", t1)
                        switch (length(cols[j][i])) {
                            case 1: case 2: case 3: case 4:
                                t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                                break
                            case 5:
                                t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                                break
                            case 6:
                                t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                                break
                            case 7:
                                t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                                break
                            default:
                                t2 = ansi("bold", cols[j][i])
                        }
                        r = r t1 t2
                    } else
                        r = r sprintf("%25s│", NULLSTR)
                }
                r = r RS
            }
            r = r "└" replicate("─", 25) "┴" replicate("─", 25) "┴" replicate("─", 25) "┘"
        }
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead) {
    initAudioPlayer()
    initPager()
    Platform = Platform ? Platform : detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", Platform)                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s\n", "bi-di emulator", BiDiTerm ? BiDiTerm :
                "[N/A]")                                                \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "web browser", Option["browser"] != NONE ?
                Option["browser"] :"[NONE]")                            \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", join(Option["sls"], "+")) \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "translation engine", Option["engine"])    \
        sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "ip version", Option["ip-version"] ? Option["ip-version"] :
                "[DEFAULT]")                                            \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCES") "]"                           \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-S") ", " ansi("bold", "-list-engines")) RS \
        ins(2, "List available translation engines and exit.") RS       \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Translator options:" RS                                     \
        ins(1, ansi("bold", "-e ") ansi("underline", "ENGINE")          \
            ", " ansi("bold", "-engine ") ansi("underline", "ENGINE")) RS \
        ins(2, "Specify the translation engine to use.") RS             \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-identify")) RS                            \
        ins(2, "Language identification.") RS                           \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        ins(1, ansi("bold", "-no-autocorrect")) RS                      \
        ins(2, "Do not autocorrect. (if defaulted by the translation engine)") RS \
        ins(1, ansi("bold", "-no-bidi")) RS                             \
        ins(2, "Do not convert bidirectional texts.") RS                \
        ins(1, ansi("bold", "-bidi")) RS                                \
        ins(2, "Always convert bidirectional texts.") RS                \
        ins(1, ansi("bold", "-no-warn")) RS                             \
        ins(2, "Do not write warning messages to stderr.") RS           \
        ins(1, ansi("bold", "-dump")) RS                                \
        ins(2, "Print raw API response instead.") RS                    \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-speak")) RS                               \
        ins(2, "Listen to the original text.") RS                       \
        ins(1, ansi("bold", "-n ") ansi("underline", "VOICE")           \
            ", " ansi("bold", "-narrator ") ansi("underline", "VOICE")) RS \
        ins(2, "Specify the narrator, and listen to the translation.") RS \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        ins(1, ansi("bold", "-no-translate")) RS                        \
        ins(2, "Do not translate anything when using -speak.") RS       \
        ins(1, ansi("bold", "-download-audio")) RS                      \
        ins(2, "Download the audio to the current directory.") RS       \
        ins(1, ansi("bold", "-download-audio-as ") ansi("underline", "FILENAME")) RS \
        ins(2, "Download the audio to the specified file.") RS          \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view") ", " ansi("bold", "-no-pager")) RS \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        ins(1, ansi("bold", "-no-browser")) RS                          \
        ins(2, "Do not open the web browser.") RS                       \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        ins(1, ansi("bold", "-4") ", " ansi("bold", "-ipv4")            \
            ", " ansi("bold", "-inet4-only")) RS                        \
        ins(2, "Connect only to IPv4 addresses.") RS                    \
        ins(1, ansi("bold", "-6") ", " ansi("bold", "-ipv6")            \
            ", " ansi("bold", "-inet6-only")) RS                        \
        ins(2, "Connect only to IPv6 addresses.") RS                    \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-sl ") ansi("underline", "CODES")        \
            ", " ansi("bold", "-source ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-from ") ansi("underline", "CODES")) RS  \
        ins(2, "Specify the source language(s), joined by '+'.") RS     \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODES")        \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-to ") ansi("underline", "CODES")) RS    \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Text preprocessing options:" RS                             \
        ins(1, ansi("bold", "-j") ", " ansi("bold", "-join-sentence")) RS \
        ins(2, "Treat all arguments as one single sentence.") RS        \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists(ENVIRON["TRANS_DIR"] "/man/" Command ".1"))
        system("man " parameterize(ENVIRON["TRANS_DIR"] "/man/" Command ".1") SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, i, j, name, num, r, rows, saveSortedIn,
                      t1, t2) {
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + (num % 3 ? 1 : 0)
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        # show languages only with stable support
        if (Locale[code]["support"] != "unstable") {
            col = int(i / rows)
            append(cols[col], code)
            i++
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        r = "┌" replicate("─", 25) "┬" replicate("─", 25) "┬" replicate("─", 25) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getDisplay(cols[j][i])
                    if (length(t1) > 17)
                        t1 = substr(t1, 1, 14) "..."
                    switch (cols[j][i]) { # fix rendered text width
                    case "sa":
                        t1 = sprintf(" %-21s", t1)
                        break
                    case "he": case "dv":
                        t1 = sprintf(" %-20s", t1)
                        break
                    case "or": case "ur":
                        t1 = sprintf(" %-19s", t1)
                        break
                    case "hi": case "gu": case "km": case "kn":
                    case "my": case "ne": case "pa": case "si":
                    case "ta": case "te": case "yi": case "as":
                    case "bho": case "mai": case "gom":
                        t1 = sprintf(" %-18s", t1)
                        break
                    case "yue":
                        t1 = sprintf(" %-15s", t1)
                        break
                    case "ja": case "ko":
                        t1 = sprintf(" %-14s", t1)
                        break
                    case "zh-CN": case "zh-TW":
                        t1 = sprintf(" %-13s", t1)
                        break
                    default:
                        if (length(t1) <= 17)
                            t1 = sprintf(" %-17s", t1)
                    }
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%25s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 25) "┴" replicate("─", 25) "┴" replicate("─", 25) "┘"
    } else {
        r = "┌" replicate("─", 25) "┬" replicate("─", 25) "┬" replicate("─", 25) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getName(cols[j][i])
                    if (length(t1) > 17)
                        t1 = substr(t1, 1, 14) "..."
                    t1 = sprintf(" %-17s", t1)
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s │", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s│", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s│", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%25s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 25) "┴" replicate("─", 25) "┴" replicate("─", 25) "┘"
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}


Whitespace

Whitespace

#! /usr/bin/awk -f

{
  region = $1

  if (NF == 1) {
    results[region] = ""
  } else {
    for (i = 2; i <= NF; i++) {
      if ($i ~ /^arn:/) {
        split($i, arn_parts, /:/)
        aws_service             = arn_parts[3]
        aws_resource_identifier = arn_parts[6]

        switch(aws_service) {
          case "cloudformation":
            split(aws_resource_identifier, identifier_parts, "/")
            stack_constant = identifier_parts[1]
            stack_name     = identifier_parts[2]
            stack_additional_identifier = identifier_parts[3]

            results[region, aws_service] = results[region, aws_service] " " stack_name
        }
      } else {
        split($i, parts, /-/)
        aws_resource_type = parts[1]
        if (aws_resource_type ~ /^(aki|ami|ari|eipalloc|elb|subnet|i|vpc)$/) {
          aws_resource_identifier = $i
          if (length(results[region, aws_resource_type]) == 0)
            results[region, aws_resource_type] = results[region, aws_resource_type] aws_resource_identifier
          else
            results[region, aws_resource_type] = results[region, aws_resource_type] " " aws_resource_identifier
        }
      }
    }
  }
}

END {
  for (item in results) {
    split(item, keys, SUBSEP)
    print keys[1], keys[2], results[item]
  }
}


TestCommons: add toString()

TestCommons: add toString()

BEGIN {
    START_TEST("Commons.awk")

    # Arrays
    T("anything()", 3)
    {
        delete something
        assertFalse(anything(nothing))
        something[0] = 0; assertFalse(anything(something)) # edge case
        something[0] = 1; assertTrue(anything(something))
    }
    T("exists()", 4)
    {
        delete something
        assertFalse(exists(something))
        something[0] = 0; assertFalse(exists(something)) # edge case
        something[0] = 1; assertTrue(exists(something))
        assertTrue(exists(something[0]))
    }
    T("belongsTo()", 3)
    {
        delete array; array[0] = "foo"; array[1] = "bar"
        assertTrue(belongsTo("foo", array))
        assertTrue(belongsTo("bar", array))
        assertFalse(belongsTo("world", array))
    }
    T("identical()", 4)
    {
        delete x; x[0] = 42
        delete y; y[0][0] = 42
        assertFalse(identical(x, y))
        assertTrue(identical(x, y[0]))
        assertFalse(identical(x[0], y[0]))
        assertTrue(identical(x[0], y[0][0]))
    }
    T("append()", 1)
    {
        delete array; array[0] = "foo"
        delete expected; expected[0] = "foo"; expected[1] = "bar"
        append(array, "bar")
        assertTrue(identical(array, expected))
    }

    # Strings
    T("isnum()", 4)
    {
        assertTrue(isnum(0))
        assertTrue(isnum(42.0))
        assertFalse(isnum(""))
        assertFalse(isnum("hello world"))
    }
    T("startsWithAny()", 3)
    {
        delete substrings; substrings[0] = "A"; substrings[1] = "a"
        assertTrue(startsWithAny("absolute", substrings))
        assertTrue(startsWithAny("ABSOLUTE", substrings))
        assertFalse(startsWithAny("ZOO", substrings))
    }
    T("matchesAny()", 4)
    {
        delete patterns; patterns[0] = "[[:space:]]"; patterns[1] = "[0Oo]"
        assertTrue(matchesAny("  ", patterns))
        assertTrue(matchesAny("obsolete", patterns))
        assertTrue(matchesAny("0.0", patterns))
        assertFalse(matchesAny("1.0", patterns))
    }
    T("replicate()", 4)
    {
        assertEqual(replicate("", 0), "")
        assertEqual(replicate("", 2), "")
        assertEqual(replicate("foo bar", 1), "foo bar")
        assertEqual(replicate("foo bar", 3), "foo barfoo barfoo bar")
    }
    T("join()", 4)
    {
        assertEqual(join("", "-"), "")
        delete array; array[0]
        assertEqual(join(array, "-"), "")
        delete array; array[0] = "foo"; array[1] = "bar"
        assertEqual(join(array, " "), "foo bar")
        assertEqual(join(array, ","), "foo,bar")
    }
    T("explode()", 1)
    {
        delete array
        delete expected; expected[1] = "f"; expected[2] = "o"; expected[3] = "o"
        explode("foo", array)
        assertTrue(identical(array, expected))
    }
    T("escapeChar()", 8)
    {
        assertEqual(escapeChar("b"), "\b")
        assertEqual(escapeChar("f"), "\f")
        assertEqual(escapeChar("n"), "\n")
        assertEqual(escapeChar("r"), "\r")
        assertEqual(escapeChar("t"), "\t")
        assertEqual(escapeChar("v"), "\v")
        assertEqual(escapeChar("_"), "_")
        assertNotEqual(escapeChar("_"), "\\_")
    }
    T("literal()", 4)
    {
        assertEqual(literal(""), "")
        assertEqual(literal("foo"), "foo")
        assertEqual(literal("\"foo\""), "foo")
        assertEqual(literal("\"\\\"foo\\\"\""), "\"foo\"")
    }
    T("escape()", 4)
    {
        assertEqual(escape(""), "")
        assertEqual(escape("foo"), "foo")
        assertEqual(escape("\""), "\\\\\"")
        assertEqual(escape("\"foo\""), "\\\\\"foo\\\\\"")
    }
    T("parameterize()", 10)
    {
        assertEqual(parameterize(""), "''")
        assertEqual(parameterize("foo"), "'foo'")
        assertEqual(parameterize("foo bar"), "'foo bar'")
        assertEqual(parameterize("foo 'bar"), "'foo '\\''bar'")
        assertEqual(parameterize("foo \"bar\""), "'foo \"bar\"'")
        assertEqual(parameterize("", "\""), "\"\"")
        assertEqual(parameterize("foo", "\""), "\"foo\"")
        assertEqual(parameterize("foo bar", "\""), "\"foo bar\"")
        assertEqual(parameterize("foo 'bar", "\""), "\"foo 'bar\"")
        assertEqual(parameterize("foo \"bar\"", "\""), "\"foo \\\\\"bar\\\\\"\"")
    }
    T("toString()", 4)
    {
        assertEqual(toString(""), "")
        assertEqual(toString(42), "42")
        assertEqual(toString("foo"), "foo")
        assertEqual(toString("\"foo bar\""), "\"foo bar\"")
    }
    T("squeeze()", 4)
    {
        assertEqual(squeeze(""), "")
        assertEqual(squeeze(" "), "")
        assertEqual(squeeze("	"), "")
        assertEqual(squeeze("  foo = bar #comments"), "foo = bar")
    }
    T("yn()", 12)
    {
        assertFalse(yn(0))
        assertFalse(yn("0"))
        assertFalse(yn("false"))
        assertFalse(yn("no"))
        assertFalse(yn("off"))
        assertFalse(yn("OFF"))
        assertTrue(yn(1))
        assertTrue(yn("1"))
        assertTrue(yn("true"))
        assertTrue(yn("yes"))
        assertTrue(yn("on"))
        assertTrue(yn("ON"))
    }

    # URLs
    T("quote()", 4)
    {
        assertEqual(quote(""), "")
        assertEqual(quote("foo"), "foo")
        assertEqual(quote("foo bar"), "foo%20bar")
        assertEqual(quote("\"hello, world!\""), "%22hello%2C%20world%21%22")
    }

    # System
    T("fileExists()", 4)
    {
        assertFalse(fileExists("README"))
        assertFalse(fileExists("README .md"))
        assertTrue(fileExists("README.md"))
        assertFalse(fileExists("."))
    }
    T("dirExists()", 4)
    {
        assertFalse(dirExists("README"))
        assertFalse(dirExists("README.md"))
        assertTrue(dirExists("."))
        assertFalse(dirExists(" ."))
    }
    T("detectProgram()", 2)
    {
        assertTrue(detectProgram("ls"))
        assertTrue(detectProgram("gawk", "--version"))
    }

    END_TEST()
}

BEGIN {
    START_TEST("Commons.awk")

    # Arrays
    T("anything()", 3)
    {
        delete something
        assertFalse(anything(nothing))
        something[0] = 0; assertFalse(anything(something)) # edge case
        something[0] = 1; assertTrue(anything(something))
    }
    T("exists()", 4)
    {
        delete something
        assertFalse(exists(something))
        something[0] = 0; assertFalse(exists(something)) # edge case
        something[0] = 1; assertTrue(exists(something))
        assertTrue(exists(something[0]))
    }
    T("belongsTo()", 3)
    {
        delete array; array[0] = "foo"; array[1] = "bar"
        assertTrue(belongsTo("foo", array))
        assertTrue(belongsTo("bar", array))
        assertFalse(belongsTo("world", array))
    }
    T("identical()", 4)
    {
        delete x; x[0] = 42
        delete y; y[0][0] = 42
        assertFalse(identical(x, y))
        assertTrue(identical(x, y[0]))
        assertFalse(identical(x[0], y[0]))
        assertTrue(identical(x[0], y[0][0]))
    }
    T("append()", 1)
    {
        delete array; array[0] = "foo"
        delete expected; expected[0] = "foo"; expected[1] = "bar"
        append(array, "bar")
        assertTrue(identical(array, expected))
    }

    # Strings
    T("isnum()", 4)
    {
        assertTrue(isnum(0))
        assertTrue(isnum(42.0))
        assertFalse(isnum(""))
        assertFalse(isnum("hello world"))
    }
    T("startsWithAny()", 3)
    {
        delete substrings; substrings[0] = "A"; substrings[1] = "a"
        assertTrue(startsWithAny("absolute", substrings))
        assertTrue(startsWithAny("ABSOLUTE", substrings))
        assertFalse(startsWithAny("ZOO", substrings))
    }
    T("matchesAny()", 4)
    {
        delete patterns; patterns[0] = "[[:space:]]"; patterns[1] = "[0Oo]"
        assertTrue(matchesAny("  ", patterns))
        assertTrue(matchesAny("obsolete", patterns))
        assertTrue(matchesAny("0.0", patterns))
        assertFalse(matchesAny("1.0", patterns))
    }
    T("replicate()", 4)
    {
        assertEqual(replicate("", 0), "")
        assertEqual(replicate("", 2), "")
        assertEqual(replicate("foo bar", 1), "foo bar")
        assertEqual(replicate("foo bar", 3), "foo barfoo barfoo bar")
    }
    T("join()", 4)
    {
        assertEqual(join("", "-"), "")
        delete array; array[0]
        assertEqual(join(array, "-"), "")
        delete array; array[0] = "foo"; array[1] = "bar"
        assertEqual(join(array, " "), "foo bar")
        assertEqual(join(array, ","), "foo,bar")
    }
    T("explode()", 1)
    {
        delete array
        delete expected; expected[1] = "f"; expected[2] = "o"; expected[3] = "o"
        explode("foo", array)
        assertTrue(identical(array, expected))
    }
    T("escapeChar()", 8)
    {
        assertEqual(escapeChar("b"), "\b")
        assertEqual(escapeChar("f"), "\f")
        assertEqual(escapeChar("n"), "\n")
        assertEqual(escapeChar("r"), "\r")
        assertEqual(escapeChar("t"), "\t")
        assertEqual(escapeChar("v"), "\v")
        assertEqual(escapeChar("_"), "_")
        assertNotEqual(escapeChar("_"), "\\_")
    }
    T("literal()", 4)
    {
        assertEqual(literal(""), "")
        assertEqual(literal("foo"), "foo")
        assertEqual(literal("\"foo\""), "foo")
        assertEqual(literal("\"\\\"foo\\\"\""), "\"foo\"")
    }
    T("escape()", 4)
    {
        assertEqual(escape(""), "")
        assertEqual(escape("foo"), "foo")
        assertEqual(escape("\""), "\\\\\"")
        assertEqual(escape("\"foo\""), "\\\\\"foo\\\\\"")
    }
    T("parameterize()", 10)
    {
        assertEqual(parameterize(""), "''")
        assertEqual(parameterize("foo"), "'foo'")
        assertEqual(parameterize("foo bar"), "'foo bar'")
        assertEqual(parameterize("foo 'bar"), "'foo '\\''bar'")
        assertEqual(parameterize("foo \"bar\""), "'foo \"bar\"'")
        assertEqual(parameterize("", "\""), "\"\"")
        assertEqual(parameterize("foo", "\""), "\"foo\"")
        assertEqual(parameterize("foo bar", "\""), "\"foo bar\"")
        assertEqual(parameterize("foo 'bar", "\""), "\"foo 'bar\"")
        assertEqual(parameterize("foo \"bar\"", "\""), "\"foo \\\\\"bar\\\\\"\"")
    }
    T("squeeze()", 4)
    {
        assertEqual(squeeze(""), "")
        assertEqual(squeeze(" "), "")
        assertEqual(squeeze("	"), "")
        assertEqual(squeeze("  foo = bar #comments"), "foo = bar")
    }
    T("yn()", 12)
    {
        assertFalse(yn(0))
        assertFalse(yn("0"))
        assertFalse(yn("false"))
        assertFalse(yn("no"))
        assertFalse(yn("off"))
        assertFalse(yn("OFF"))
        assertTrue(yn(1))
        assertTrue(yn("1"))
        assertTrue(yn("true"))
        assertTrue(yn("yes"))
        assertTrue(yn("on"))
        assertTrue(yn("ON"))
    }

    # URLs
    T("quote()", 4)
    {
        assertEqual(quote(""), "")
        assertEqual(quote("foo"), "foo")
        assertEqual(quote("foo bar"), "foo%20bar")
        assertEqual(quote("\"hello, world!\""), "%22hello%2C%20world%21%22")
    }

    # System
    T("fileExists()", 4)
    {
        assertFalse(fileExists("README"))
        assertFalse(fileExists("README .md"))
        assertTrue(fileExists("README.md"))
        assertFalse(fileExists("."))
    }
    T("dirExists()", 4)
    {
        assertFalse(dirExists("README"))
        assertFalse(dirExists("README.md"))
        assertTrue(dirExists("."))
        assertFalse(dirExists(" ."))
    }
    T("detectProgram()", 2)
    {
        assertTrue(detectProgram("ls"))
        assertTrue(detectProgram("gawk", "--version"))
    }

    END_TEST()
}


TestParser: remove invalid JSON (hopefully get things right this time)

@include "include/Parser"

BEGIN {
    START_TEST("Parser.awk")

    T("tokenize()", 8)
    {
        delete tokens
        tokenize(tokens, "0")
        delete expected
        expected[0] = "0"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "3.14")
        delete expected
        expected[0] = "3.14"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "Atom")
        delete expected
        expected[0] = "Atom"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"foo bar\"")
        delete expected
        expected[0] = "\"foo bar\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"\\\"\"")
        delete expected
        expected[0] = "\"\\\"\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "(QUOTE A)")
        delete expected
        expected[0] = "("
        expected[1] = "QUOTE"
        expected[2] = "A"
        expected[3] = ")"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "1 + 2 * 3")
        delete expected
        expected[0] = "1"
        expected[1] = "+"
        expected[2] = "2"
        expected[3] = "*"
        expected[4] = "3"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "[42, \"hello, world\", [Foo, bar]]")
        delete expected
        expected[0] = "["
        expected[1] = "42"
        expected[2] = ","
        expected[3] = "\"hello, world\""
        expected[4] = ","
        expected[5] = "["
        expected[6] = "Foo"
        expected[7] = ","
        expected[8] = "bar"
        expected[9] = "]"
        expected[10] = "]"
        assertEqual(tokens, expected)
    }

    T("parseJson()", 10)
    {
        delete tokens; delete ast; delete expected
        tokenize(tokens, "0")
        parseJson(ast, tokens)
        expected[0] = "0"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "null")
        parseJson(ast, tokens)
        expected[0] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "[42]")
        parseJson(ast, tokens)
        expected[0 SUBSEP 0] = "42"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "[42, \"answer\", null]")
        parseJson(ast, tokens)
        expected[0 SUBSEP 0] = "42"
        expected[0 SUBSEP 1] = "\"answer\""
        expected[0 SUBSEP 2] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": [42], \"Answer\": null}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "answer" SUBSEP 0] = 42
        expected[0 SUBSEP "Answer"] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {\"everything\": 42}}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "answer" SUBSEP "everything"] = "42"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {\"everything\": [42]}}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "answer" SUBSEP "everything" SUBSEP 0] = "42"
        assertEqual(ast, expected)

        # empty object - what is the "correct" parsing result?

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{}")
        parseJson(ast, tokens)
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {}}")
        parseJson(ast, tokens)
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {}, \"Answer\": null}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "Answer"] = "null"
        assertEqual(ast, expected)
    }

    T("parseList()", 1)
    {
        delete tokens; delete ast; delete expected
        tokenize(tokens, "0")
        parseList(ast, tokens)
        expected[NULLSTR][0] = "0"
        assertEqual(ast, expected)
    }

    END_TEST()
}

@include "include/Parser"

BEGIN {
    START_TEST("Parser.awk")

    T("tokenize()", 8)
    {
        delete tokens
        tokenize(tokens, "0")
        delete expected
        expected[0] = "0"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "3.14")
        delete expected
        expected[0] = "3.14"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "Atom")
        delete expected
        expected[0] = "Atom"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"foo bar\"")
        delete expected
        expected[0] = "\"foo bar\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"\\\"\"")
        delete expected
        expected[0] = "\"\\\"\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "(QUOTE A)")
        delete expected
        expected[0] = "("
        expected[1] = "QUOTE"
        expected[2] = "A"
        expected[3] = ")"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "1 + 2 * 3")
        delete expected
        expected[0] = "1"
        expected[1] = "+"
        expected[2] = "2"
        expected[3] = "*"
        expected[4] = "3"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "[42, \"hello, world\", [Foo, bar]]")
        delete expected
        expected[0] = "["
        expected[1] = "42"
        expected[2] = ","
        expected[3] = "\"hello, world\""
        expected[4] = ","
        expected[5] = "["
        expected[6] = "Foo"
        expected[7] = ","
        expected[8] = "bar"
        expected[9] = "]"
        expected[10] = "]"
        assertEqual(tokens, expected)
    }

    T("parseJson()", 10)
    {
        delete tokens; delete ast; delete expected
        tokenize(tokens, "0")
        parseJson(ast, tokens)
        expected[0] = "0"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "null")
        parseJson(ast, tokens)
        expected[0] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "[42]")
        parseJson(ast, tokens)
        expected[0 SUBSEP 0] = "42"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "[42, \"answer\", null]")
        parseJson(ast, tokens)
        expected[0 SUBSEP 0] = "42"
        expected[0 SUBSEP 1] = "\"answer\""
        expected[0 SUBSEP 2] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": [42], \"Answer\": null}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "answer" SUBSEP 0] = 42
        expected[0 SUBSEP "Answer"] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {42}, \"Answer\": null}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "answer" SUBSEP] = 42
        expected[0 SUBSEP "Answer"] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {[42]}, \"Answer\": null}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "answer" SUBSEP SUBSEP 0] = 42
        expected[0 SUBSEP "Answer"] = "null"
        assertEqual(ast, expected)

        # empty object - what is the "correct" parsing result?

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{}")
        parseJson(ast, tokens)
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {}}")
        parseJson(ast, tokens)
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": {}, \"Answer\": null}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "Answer"] = "null"
        assertEqual(ast, expected)
    }

    T("parseList()", 1)
    {
        delete tokens; delete ast; delete expected
        tokenize(tokens, "0")
        parseList(ast, tokens)
        expected[NULLSTR][0] = "0"
        assertEqual(ast, expected)
    }

    END_TEST()
}


add|sub rsp alter the indent

add|sub rsp alter the indent

#!/bin/sh

exec awk '
/pop|leave/ {
	if(--indent < 0)
		print "AWK: indent less than zero (" indent ")" #| "cat >&2"
}

{
	i = indent
	while(i --> 0)
		printf "\t"

	print $0
}

/push/ {
	indent++
}

/(add|sub) rsp/ {
	n = $3 / 8
	if($1 == "sub")
		n = -n
	indent -= n
}
'

#!/bin/sh

exec awk '
/pop/ {
	if(--indent < 0)
		print "AWK: indent less than zero (" indent ")" #| "cat >&2"
}

{
	i = indent
	while(i --> 0)
		printf "\t"

	print $0
}

/push/ {
	indent++
}
'


ahrf.awk: Silence gawk(1) forever

ahrf.awk: Silence gawk(1) forever

#!/home/ypnose/Dev/AWK/nawk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0, ":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====/ {
	if (NF > 2 && $NF == "====") {
		gsub(/^[\t ]*====[\t ]*\n|\n[\t ]*====[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\*/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l != "") {
			syb = substr($l,2,1)
			if (syb == " ") {
				str = substr($l,3)
			} else {
				str = substr($l,2)
			}
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u, "\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			printf("\t
%s %s
\n", num, url, url)
		}
	}
	printf("
\n")
	next
}

#!/home/ypnose/Dev/AWK/nawk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0, ":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====/ {
	if (NF > 2 && $NF == "====") {
		gsub(/^[\t ]*====[\t ]*\n|\n[\t ]*====[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\*/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l != "") {
			syb = substr($l,2,1)
			if (syb == " ") {
				str = substr($l,3)
			} else {
				str = substr($l,2)
			}
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u, "\[[0-9]+\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			printf("\t
%s %s
\n", num, url, url)
		}
	}
	printf("
\n")
	next
}