x86: Fix the error of using "const" in gen-insn-attr-x86.awk

The original version code causes following sparse warnings:
arch/x86/lib/inat-tables.c:1080:25: warning: duplicate const
arch/x86/lib/inat-tables.c:1095:25: warning: duplicate const
arch/x86/lib/inat-tables.c:1118:25: warning: duplicate const

for the variables inat_escape_tables, inat_group_tables, and inat_avx_tables
in the code generated by gen-insn-attr-x86.awk.

The author Masami Hiramutsu says here is to make both the value pointed by the
pointers and the pointers itself read-only, so we move the "const" to be after
the "*".

Signed-off-by: Cong Ding <8001b54ad539330acc19ca3e17ac1f04f07b36a6@gmail.com>
Link: 3ec41833c29d285066dfad622a5b82e61ee0def7@gmail.com
Acked-by: Masami Hiramatsu <60fcdccfd958d359debfe6e6e7417cd7f1aaa9e3@hitachi.com>
Signed-off-by: H. Peter Anvin <8a453bad9912ffe59bc0f0b8abe03df9be19379e@linux.intel.com>

#!/bin/awk -f
# gen-insn-attr-x86.awk: Instruction attribute table generator
# Written by Masami Hiramatsu <mhiramat@redhat.com>
#
# Usage: awk -f gen-insn-attr-x86.awk x86-opcode-map.txt > inat-tables.c

# Awk implementation sanity check
function check_awk_implement() {
	if (sprintf("%x", 0) != "0")
		return "Your awk has a printf-format problem."
	return ""
}

# Clear working vars
function clear_vars() {
	delete table
	delete lptable2
	delete lptable1
	delete lptable3
	eid = -1 # escape id
	gid = -1 # group id
	aid = -1 # AVX id
	tname = ""
}

BEGIN {
	# Implementation error checking
	awkchecked = check_awk_implement()
	if (awkchecked != "") {
		print "Error: " awkchecked > "/dev/stderr"
		print "Please try to use gawk." > "/dev/stderr"
		exit 1
	}

	# Setup generating tables
	print "/* x86 opcode map generated from x86-opcode-map.txt */"
	print "/* Do not change this code. */\n"
	ggid = 1
	geid = 1
	gaid = 0
	delete etable
	delete gtable
	delete atable

	opnd_expr = "^[A-Za-z/]"
	ext_expr = "^\\("
	sep_expr = "^\\|$"
	group_expr = "^Grp[0-9A-Za-z]+"

	imm_expr = "^[IJAOL][a-z]"
	imm_flag["Ib"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Jb"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Iw"] = "INAT_MAKE_IMM(INAT_IMM_WORD)"
	imm_flag["Id"] = "INAT_MAKE_IMM(INAT_IMM_DWORD)"
	imm_flag["Iq"] = "INAT_MAKE_IMM(INAT_IMM_QWORD)"
	imm_flag["Ap"] = "INAT_MAKE_IMM(INAT_IMM_PTR)"
	imm_flag["Iz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Jz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Iv"] = "INAT_MAKE_IMM(INAT_IMM_VWORD)"
	imm_flag["Ob"] = "INAT_MOFFSET"
	imm_flag["Ov"] = "INAT_MOFFSET"
	imm_flag["Lx"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"

	modrm_expr = "^([CDEGMNPQRSUVW/][a-z]+|NTA|T[012])"
	force64_expr = "\\([df]64\\)"
	rex_expr = "^REX(\\.[XRWB]+)*"
	fpu_expr = "^ESC" # TODO

	lprefix1_expr = "\\((66|!F3)\\)"
	lprefix2_expr = "\\(F3\\)"
	lprefix3_expr = "\\((F2|!F3)\\)"
	lprefix_expr = "\\((66|F2|F3)\\)"
	max_lprefix = 4

	# All opcodes starting with lower-case 'v' or with (v1) superscript
	# accepts VEX prefix
	vexok_opcode_expr = "^v.*"
	vexok_expr = "\\(v1\\)"
	# All opcodes with (v) superscript supports *only* VEX prefix
	vexonly_expr = "\\(v\\)"

	prefix_expr = "\\(Prefix\\)"
	prefix_num["Operand-Size"] = "INAT_PFX_OPNDSZ"
	prefix_num["REPNE"] = "INAT_PFX_REPNE"
	prefix_num["REP/REPE"] = "INAT_PFX_REPE"
	prefix_num["LOCK"] = "INAT_PFX_LOCK"
	prefix_num["SEG=CS"] = "INAT_PFX_CS"
	prefix_num["SEG=DS"] = "INAT_PFX_DS"
	prefix_num["SEG=ES"] = "INAT_PFX_ES"
	prefix_num["SEG=FS"] = "INAT_PFX_FS"
	prefix_num["SEG=GS"] = "INAT_PFX_GS"
	prefix_num["SEG=SS"] = "INAT_PFX_SS"
	prefix_num["Address-Size"] = "INAT_PFX_ADDRSZ"
	prefix_num["VEX+1byte"] = "INAT_PFX_VEX2"
	prefix_num["VEX+2byte"] = "INAT_PFX_VEX3"

	clear_vars()
}

function semantic_error(msg) {
	print "Semantic error at " NR ": " msg > "/dev/stderr"
	exit 1
}

function debug(msg) {
	print "DEBUG: " msg
}

function array_size(arr,   i,c) {
	c = 0
	for (i in arr)
		c++
	return c
}

/^Table:/ {
	print "/* " $0 " */"
	if (tname != "")
		semantic_error("Hit Table: before EndTable:.");
}

/^Referrer:/ {
	if (NF != 1) {
		# escape opcode table
		ref = ""
		for (i = 2; i <= NF; i++)
			ref = ref $i
		eid = escape[ref]
		tname = sprintf("inat_escape_table_%d", eid)
	}
}

/^AVXcode:/ {
	if (NF != 1) {
		# AVX/escape opcode table
		aid = $2
		if (gaid <= aid)
			gaid = aid + 1
		if (tname == "")	# AVX only opcode table
			tname = sprintf("inat_avx_table_%d", $2)
	}
	if (aid == -1 && eid == -1)	# primary opcode table
		tname = "inat_primary_table"
}

/^GrpTable:/ {
	print "/* " $0 " */"
	if (!($2 in group))
		semantic_error("No group: " $2 )
	gid = group[$2]
	tname = "inat_group_table_" gid
}

function print_table(tbl,name,fmt,n)
{
	print "const insn_attr_t " name " = {"
	for (i = 0; i < n; i++) {
		id = sprintf(fmt, i)
		if (tbl[id])
			print "	[" id "] = " tbl[id] ","
	}
	print "};"
}

/^EndTable/ {
	if (gid != -1) {
		# print group tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1, tname "_1[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2, tname "_2[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3, tname "_3[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,3] = tname "_3"
		}
	} else {
		# print primary/escaped tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,0] = tname
			if (aid >= 0)
				atable[aid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1,tname "_1[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,1] = tname "_1"
			if (aid >= 0)
				atable[aid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2,tname "_2[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,2] = tname "_2"
			if (aid >= 0)
				atable[aid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3,tname "_3[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,3] = tname "_3"
			if (aid >= 0)
				atable[aid,3] = tname "_3"
		}
	}
	print ""
	clear_vars()
}

function add_flags(old,new) {
	if (old && new)
		return old " | " new
	else if (old)
		return old
	else
		return new
}

# convert operands to flags.
function convert_operands(count,opnd,       i,j,imm,mod)
{
	imm = null
	mod = null
	for (j = 1; j <= count; j++) {
		i = opnd[j]
		if (match(i, imm_expr) == 1) {
			if (!imm_flag[i])
				semantic_error("Unknown imm opnd: " i)
			if (imm) {
				if (i != "Ib")
					semantic_error("Second IMM error")
				imm = add_flags(imm, "INAT_SCNDIMM")
			} else
				imm = imm_flag[i]
		} else if (match(i, modrm_expr))
			mod = "INAT_MODRM"
	}
	return add_flags(imm, mod)
}

/^[0-9a-f]+\:/ {
	if (NR == 1)
		next
	# get index
	idx = "0x" substr($1, 1, index($1,":") - 1)
	if (idx in table)
		semantic_error("Redefine " idx " in " tname)

	# check if escaped opcode
	if ("escape" == $2) {
		if ($3 != "#")
			semantic_error("No escaped name")
		ref = ""
		for (i = 4; i <= NF; i++)
			ref = ref $i
		if (ref in escape)
			semantic_error("Redefine escape (" ref ")")
		escape[ref] = geid
		geid++
		table[idx] = "INAT_MAKE_ESCAPE(" escape[ref] ")"
		next
	}

	variant = null
	# converts
	i = 2
	while (i <= NF) {
		opcode = $(i++)
		delete opnds
		ext = null
		flags = null
		opnd = null
		# parse one opcode
		if (match($i, opnd_expr)) {
			opnd = $i
			count = split($(i++), opnds, ",")
			flags = convert_operands(count, opnds)
		}
		if (match($i, ext_expr))
			ext = $(i++)
		if (match($i, sep_expr))
			i++
		else if (i < NF)
			semantic_error($i " is not a separator")

		# check if group opcode
		if (match(opcode, group_expr)) {
			if (!(opcode in group)) {
				group[opcode] = ggid
				ggid++
			}
			flags = add_flags(flags, "INAT_MAKE_GROUP(" group[opcode] ")")
		}
		# check force(or default) 64bit
		if (match(ext, force64_expr))
			flags = add_flags(flags, "INAT_FORCE64")

		# check REX prefix
		if (match(opcode, rex_expr))
			flags = add_flags(flags, "INAT_MAKE_PREFIX(INAT_PFX_REX)")

		# check coprocessor escape : TODO
		if (match(opcode, fpu_expr))
			flags = add_flags(flags, "INAT_MODRM")

		# check VEX codes
		if (match(ext, vexonly_expr))
			flags = add_flags(flags, "INAT_VEXOK | INAT_VEXONLY")
		else if (match(ext, vexok_expr) || match(opcode, vexok_opcode_expr))
			flags = add_flags(flags, "INAT_VEXOK")

		# check prefixes
		if (match(ext, prefix_expr)) {
			if (!prefix_num[opcode])
				semantic_error("Unknown prefix: " opcode)
			flags = add_flags(flags, "INAT_MAKE_PREFIX(" prefix_num[opcode] ")")
		}
		if (length(flags) == 0)
			continue
		# check if last prefix
		if (match(ext, lprefix1_expr)) {
			lptable1[idx] = add_flags(lptable1[idx],flags)
			variant = "INAT_VARIANT"
		}
		if (match(ext, lprefix2_expr)) {
			lptable2[idx] = add_flags(lptable2[idx],flags)
			variant = "INAT_VARIANT"
		}
		if (match(ext, lprefix3_expr)) {
			lptable3[idx] = add_flags(lptable3[idx],flags)
			variant = "INAT_VARIANT"
		}
		if (!match(ext, lprefix_expr)){
			table[idx] = add_flags(table[idx],flags)
		}
	}
	if (variant)
		table[idx] = add_flags(table[idx],variant)
}

END {
	if (awkchecked != "")
		exit 1
	# print escape opcode map's array
	print "/* Escape opcode map array */"
	print "const insn_attr_t * const inat_escape_tables[INAT_ESC_MAX + 1]" \
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < geid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (etable[i,j])
				print "	["i"]["j"] = "etable[i,j]","
	print "};\n"
	# print group opcode map's array
	print "/* Group opcode map array */"
	print "const insn_attr_t * const inat_group_tables[INAT_GRP_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < ggid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (gtable[i,j])
				print "	["i"]["j"] = "gtable[i,j]","
	print "};\n"
	# print AVX opcode map's array
	print "/* AVX opcode map array */"
	print "const insn_attr_t * const inat_avx_tables[X86_VEX_M_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < gaid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (atable[i,j])
				print "	["i"]["j"] = "atable[i,j]","
	print "};"
}

#!/bin/awk -f
# gen-insn-attr-x86.awk: Instruction attribute table generator
# Written by Masami Hiramatsu <mhiramat@redhat.com>
#
# Usage: awk -f gen-insn-attr-x86.awk x86-opcode-map.txt > inat-tables.c

# Awk implementation sanity check
function check_awk_implement() {
	if (sprintf("%x", 0) != "0")
		return "Your awk has a printf-format problem."
	return ""
}

# Clear working vars
function clear_vars() {
	delete table
	delete lptable2
	delete lptable1
	delete lptable3
	eid = -1 # escape id
	gid = -1 # group id
	aid = -1 # AVX id
	tname = ""
}

BEGIN {
	# Implementation error checking
	awkchecked = check_awk_implement()
	if (awkchecked != "") {
		print "Error: " awkchecked > "/dev/stderr"
		print "Please try to use gawk." > "/dev/stderr"
		exit 1
	}

	# Setup generating tables
	print "/* x86 opcode map generated from x86-opcode-map.txt */"
	print "/* Do not change this code. */\n"
	ggid = 1
	geid = 1
	gaid = 0
	delete etable
	delete gtable
	delete atable

	opnd_expr = "^[A-Za-z/]"
	ext_expr = "^\\("
	sep_expr = "^\\|$"
	group_expr = "^Grp[0-9A-Za-z]+"

	imm_expr = "^[IJAOL][a-z]"
	imm_flag["Ib"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Jb"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Iw"] = "INAT_MAKE_IMM(INAT_IMM_WORD)"
	imm_flag["Id"] = "INAT_MAKE_IMM(INAT_IMM_DWORD)"
	imm_flag["Iq"] = "INAT_MAKE_IMM(INAT_IMM_QWORD)"
	imm_flag["Ap"] = "INAT_MAKE_IMM(INAT_IMM_PTR)"
	imm_flag["Iz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Jz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Iv"] = "INAT_MAKE_IMM(INAT_IMM_VWORD)"
	imm_flag["Ob"] = "INAT_MOFFSET"
	imm_flag["Ov"] = "INAT_MOFFSET"
	imm_flag["Lx"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"

	modrm_expr = "^([CDEGMNPQRSUVW/][a-z]+|NTA|T[012])"
	force64_expr = "\\([df]64\\)"
	rex_expr = "^REX(\\.[XRWB]+)*"
	fpu_expr = "^ESC" # TODO

	lprefix1_expr = "\\((66|!F3)\\)"
	lprefix2_expr = "\\(F3\\)"
	lprefix3_expr = "\\((F2|!F3)\\)"
	lprefix_expr = "\\((66|F2|F3)\\)"
	max_lprefix = 4

	# All opcodes starting with lower-case 'v' or with (v1) superscript
	# accepts VEX prefix
	vexok_opcode_expr = "^v.*"
	vexok_expr = "\\(v1\\)"
	# All opcodes with (v) superscript supports *only* VEX prefix
	vexonly_expr = "\\(v\\)"

	prefix_expr = "\\(Prefix\\)"
	prefix_num["Operand-Size"] = "INAT_PFX_OPNDSZ"
	prefix_num["REPNE"] = "INAT_PFX_REPNE"
	prefix_num["REP/REPE"] = "INAT_PFX_REPE"
	prefix_num["LOCK"] = "INAT_PFX_LOCK"
	prefix_num["SEG=CS"] = "INAT_PFX_CS"
	prefix_num["SEG=DS"] = "INAT_PFX_DS"
	prefix_num["SEG=ES"] = "INAT_PFX_ES"
	prefix_num["SEG=FS"] = "INAT_PFX_FS"
	prefix_num["SEG=GS"] = "INAT_PFX_GS"
	prefix_num["SEG=SS"] = "INAT_PFX_SS"
	prefix_num["Address-Size"] = "INAT_PFX_ADDRSZ"
	prefix_num["VEX+1byte"] = "INAT_PFX_VEX2"
	prefix_num["VEX+2byte"] = "INAT_PFX_VEX3"

	clear_vars()
}

function semantic_error(msg) {
	print "Semantic error at " NR ": " msg > "/dev/stderr"
	exit 1
}

function debug(msg) {
	print "DEBUG: " msg
}

function array_size(arr,   i,c) {
	c = 0
	for (i in arr)
		c++
	return c
}

/^Table:/ {
	print "/* " $0 " */"
	if (tname != "")
		semantic_error("Hit Table: before EndTable:.");
}

/^Referrer:/ {
	if (NF != 1) {
		# escape opcode table
		ref = ""
		for (i = 2; i <= NF; i++)
			ref = ref $i
		eid = escape[ref]
		tname = sprintf("inat_escape_table_%d", eid)
	}
}

/^AVXcode:/ {
	if (NF != 1) {
		# AVX/escape opcode table
		aid = $2
		if (gaid <= aid)
			gaid = aid + 1
		if (tname == "")	# AVX only opcode table
			tname = sprintf("inat_avx_table_%d", $2)
	}
	if (aid == -1 && eid == -1)	# primary opcode table
		tname = "inat_primary_table"
}

/^GrpTable:/ {
	print "/* " $0 " */"
	if (!($2 in group))
		semantic_error("No group: " $2 )
	gid = group[$2]
	tname = "inat_group_table_" gid
}

function print_table(tbl,name,fmt,n)
{
	print "const insn_attr_t " name " = {"
	for (i = 0; i < n; i++) {
		id = sprintf(fmt, i)
		if (tbl[id])
			print "	[" id "] = " tbl[id] ","
	}
	print "};"
}

/^EndTable/ {
	if (gid != -1) {
		# print group tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1, tname "_1[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2, tname "_2[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3, tname "_3[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,3] = tname "_3"
		}
	} else {
		# print primary/escaped tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,0] = tname
			if (aid >= 0)
				atable[aid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1,tname "_1[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,1] = tname "_1"
			if (aid >= 0)
				atable[aid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2,tname "_2[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,2] = tname "_2"
			if (aid >= 0)
				atable[aid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3,tname "_3[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,3] = tname "_3"
			if (aid >= 0)
				atable[aid,3] = tname "_3"
		}
	}
	print ""
	clear_vars()
}

function add_flags(old,new) {
	if (old && new)
		return old " | " new
	else if (old)
		return old
	else
		return new
}

# convert operands to flags.
function convert_operands(count,opnd,       i,j,imm,mod)
{
	imm = null
	mod = null
	for (j = 1; j <= count; j++) {
		i = opnd[j]
		if (match(i, imm_expr) == 1) {
			if (!imm_flag[i])
				semantic_error("Unknown imm opnd: " i)
			if (imm) {
				if (i != "Ib")
					semantic_error("Second IMM error")
				imm = add_flags(imm, "INAT_SCNDIMM")
			} else
				imm = imm_flag[i]
		} else if (match(i, modrm_expr))
			mod = "INAT_MODRM"
	}
	return add_flags(imm, mod)
}

/^[0-9a-f]+\:/ {
	if (NR == 1)
		next
	# get index
	idx = "0x" substr($1, 1, index($1,":") - 1)
	if (idx in table)
		semantic_error("Redefine " idx " in " tname)

	# check if escaped opcode
	if ("escape" == $2) {
		if ($3 != "#")
			semantic_error("No escaped name")
		ref = ""
		for (i = 4; i <= NF; i++)
			ref = ref $i
		if (ref in escape)
			semantic_error("Redefine escape (" ref ")")
		escape[ref] = geid
		geid++
		table[idx] = "INAT_MAKE_ESCAPE(" escape[ref] ")"
		next
	}

	variant = null
	# converts
	i = 2
	while (i <= NF) {
		opcode = $(i++)
		delete opnds
		ext = null
		flags = null
		opnd = null
		# parse one opcode
		if (match($i, opnd_expr)) {
			opnd = $i
			count = split($(i++), opnds, ",")
			flags = convert_operands(count, opnds)
		}
		if (match($i, ext_expr))
			ext = $(i++)
		if (match($i, sep_expr))
			i++
		else if (i < NF)
			semantic_error($i " is not a separator")

		# check if group opcode
		if (match(opcode, group_expr)) {
			if (!(opcode in group)) {
				group[opcode] = ggid
				ggid++
			}
			flags = add_flags(flags, "INAT_MAKE_GROUP(" group[opcode] ")")
		}
		# check force(or default) 64bit
		if (match(ext, force64_expr))
			flags = add_flags(flags, "INAT_FORCE64")

		# check REX prefix
		if (match(opcode, rex_expr))
			flags = add_flags(flags, "INAT_MAKE_PREFIX(INAT_PFX_REX)")

		# check coprocessor escape : TODO
		if (match(opcode, fpu_expr))
			flags = add_flags(flags, "INAT_MODRM")

		# check VEX codes
		if (match(ext, vexonly_expr))
			flags = add_flags(flags, "INAT_VEXOK | INAT_VEXONLY")
		else if (match(ext, vexok_expr) || match(opcode, vexok_opcode_expr))
			flags = add_flags(flags, "INAT_VEXOK")

		# check prefixes
		if (match(ext, prefix_expr)) {
			if (!prefix_num[opcode])
				semantic_error("Unknown prefix: " opcode)
			flags = add_flags(flags, "INAT_MAKE_PREFIX(" prefix_num[opcode] ")")
		}
		if (length(flags) == 0)
			continue
		# check if last prefix
		if (match(ext, lprefix1_expr)) {
			lptable1[idx] = add_flags(lptable1[idx],flags)
			variant = "INAT_VARIANT"
		}
		if (match(ext, lprefix2_expr)) {
			lptable2[idx] = add_flags(lptable2[idx],flags)
			variant = "INAT_VARIANT"
		}
		if (match(ext, lprefix3_expr)) {
			lptable3[idx] = add_flags(lptable3[idx],flags)
			variant = "INAT_VARIANT"
		}
		if (!match(ext, lprefix_expr)){
			table[idx] = add_flags(table[idx],flags)
		}
	}
	if (variant)
		table[idx] = add_flags(table[idx],variant)
}

END {
	if (awkchecked != "")
		exit 1
	# print escape opcode map's array
	print "/* Escape opcode map array */"
	print "const insn_attr_t const *inat_escape_tables[INAT_ESC_MAX + 1]" \
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < geid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (etable[i,j])
				print "	["i"]["j"] = "etable[i,j]","
	print "};\n"
	# print group opcode map's array
	print "/* Group opcode map array */"
	print "const insn_attr_t const *inat_group_tables[INAT_GRP_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < ggid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (gtable[i,j])
				print "	["i"]["j"] = "gtable[i,j]","
	print "};\n"
	# print AVX opcode map's array
	print "/* AVX opcode map array */"
	print "const insn_attr_t const *inat_avx_tables[X86_VEX_M_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < gaid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (atable[i,j])
				print "	["i"]["j"] = "atable[i,j]","
	print "};"
}


asdas

asdas

BEGIN { FS=","; Id=0}

function array0(a,i) {
    a[i][""]
    delete a[i][""]
}
function id(i) {
     if (i == "") i = Id = Id + 1;
     return i
}
function num0(_Num,  i) {
    i = id(i)
    txt[i] = ""
    n[i]   = mu[i] = m2[i] = 0
    lo[i]  =  10^32
    up[i]  = -10^32
    return i
}
function sym0(_Sym,  i) {
    i = id(i)
    txt[i]  = ""
    n[i]    = 0
    most[i] = 0
    mode[i] = ""
    array0(count,i)
    return i
}
function sym1(c,x,_Sym,   new) {
    n[c]++
    counts[c][x]++
    new = counts[c][x]
    if (new > most[c]) {
        most[c] = new
        mode[c] = x }
}
function num1(c,x,_Num,   delta) {
    if (n > up[c]) up[c] = n
    if (n < lo[c]) lo[c] = n
    delta = x - mu[c]
    mu[c] += delta / n[c]
    m2[c] += x - mu[c]
}
function sd(c,_Num) {
    return (m2[c] / (n[c] - 1))^0.5
}
function entireLine(line,   tmp) {
    gsub(/[ \t\r]/, "", line)
    gsub(/##.*/   , "", line)
    if ( line !~ /,$/ )
        return line
    if ((getline tmp) == 0)
        return line
    return line entireLine(tmp)
}
function using(cells0, use, cells) {
    for(c in use)
        cells[c] = cells0[ uses[c] ]
}
function usable(cells0, use,cells,  c0,c) {
    max = length(cells0)
    for(c0=1; c0 <= max; c0++)
        if (cells0[c0] !~ /^X/)
            use[++c] = c0;
    using(cells0,use,cells)
}
function readcsv(com,m,_Num,_Sym,_Out,txt, \
                 cell,srows,i,use,_Num) {
  com = com ? com : "cat -"
  while((com | getline)  > 0)  {
      line = entireLine($0)
      if (line) {
          split(line,cells0,",")
          rows++
          if (rows==1) {
              usable(cells0, use, cells)
              head(cells,m,use, _Num, _Sym, _Out)
          }
          else {
              using(cells0, use, cells)
              row( cells,m, rows-1, _Num, _Sym) }}}
}
function row(cells,m,r, _Num,_Sym,  c) {
    for(c in cells) {
        if (c in lo) {
            cells[c] += 0
            num1(c, cells[c], _Num)
        } else
            sym1(c, cells[c], _Sym);
        m[r][c] = line[c]}
}
function head(line,m, _Num,_Sym, _Out,   c0,c) {
    for(c in line) {
        if ( $c ~ /^</ ) less[c]
        if ( $c ~ /^>/ ) more[c]
        if ( $c ~ /^=/ ) klass[c]
        txt[c] = line[c]
        if ( $c ~ /^_/ )
            num0(_Num, c)
        else
            sym0(_Sym,c) }
}

BEGIN { FS=","; Id=0}

function array0(a,i) {
    a[i][""]
    delete a[i][""]
}
function id(i) {
     if (i == "") i = Id = Id + 1;
     return i
}
function num0(_Num,  i) {
    i = id(i)
    txt[i] = ""
    n[i]   = mu[i] = m2[i] = 0
    lo[i]  =  10^32
    up[i]  = -10^32
    return i
}
function sym0(_Sym,  i) {
    i = id(i)
    txt[i]  = ""
    n[i]    = 0
    most[i] = 0
    mode[i] = ""
    array0(count,i)
    return i
}
function sym(c,x,_Sym,   new) {
    n[c]++
    counts[c][x]++
    new = counts[c][x]
    if (new > most[c]) {
        most[c] = new
        mode[c] = x }
}
function num(c,x,_Num,   delta) {
    if (n > up[c]) up[c] = n
    if (n < lo[c]) lo[c] = n
    delta = x - mu[c]
    mu[c] += delta / n[c]
    m2[c] += x - mu[c]
}
function sd(c,_Num) {
    return (m2[c] / (n[c] - 1))^0.5
}
function readcsv(com,m,_Num,_Sym,_Out,txt,    rows,i,_Num) {
  com = com ? com : "cat -"
  while((com | getline)  > 0)  {
    gsub(/[ \t\r]/,"")
    gsub(/##.*/,"")
    if ($0) {
        for(i=1;i<=NF;i++)
            line[i] = $i;
        if (rows)
            row(line,rows,m, _Num, _Sym)
        else
            header(line,r,m, _Num, _Sym, _Out);
        rows++ }}
}
function row(line,r,m, _Num,_Sym,  c) {
    for(c in line) {
        if (c in lo) {
            line[c] += 0
            num(c, line[c], _Num)
        } else
            sym(c, line[c], _Sym)
        m[r][c] = line[c]}
}
function header(line,txt,_Num,_Sym, _Out,   c) {
    for(c in line)
        if ( $c ~ /^</ ) less[c]
        if ( $c ~ /^>/ ) more[c]
        if ( $c ~ /^=/ ) klass[c]
        if ( $c ~ /^X/ ) {
            txt[c] = line[c]
            if ( $c ~ /^_/ )
               num0(_Num, c)
            else
                sym0(_Sym,c) }
}


Translators/BingTranslator: fix original phonetics (purge old AST)

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 25 Jul 2019
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# [OBSOLETE?] Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    #bingSetIG() # set IG (no longer needed)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code  # FIXME
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            delete ast  # purge old AST
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 \
                                            SUBSEP "transliteration" SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 25 Jul 2019
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# [OBSOLETE?] Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    #bingSetIG() # set IG (no longer needed)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code  # FIXME
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                            SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}


Commons: add "\t" to initUrlEncoding (fix #108)

####################################################################
# Commons.awk                                                      #
####################################################################

# Initialize constants.
function initConst() {
    NULLSTR = ""
    TRUE = 1

    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
    PIPE = " | "
}

## Arrays:

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Return 1 if the value is non-empty or an array that contains anything;
# Otherwise, return 0.
function exists(value) {
    if (isarray(value))
        return anything(value)
    else
        return value ? 1 : 0
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return NULLSTR
}

# Return non-zero if two values are identical;
# Otherwise, return 0.
function identical(x, y,
                   ####
                   i) {
    if (!isarray(x) && !isarray(y))
        return x == y
    else if (isarray(x) && isarray(y)) {
        if (length(x) != length(y)) return 0
        for (i in x)
            if (!identical(x[i], y[i])) return 0
        return 1
    } else
        return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

## Strings:

# Return non-zero if the string represents a numeral;
# Otherwise, return 0.
function isnum(string) {
    return string == string + 0
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return NULLSTR
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return NULLSTR
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = NULLSTR
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = NULLSTR
    j = 0
    if (isarray(array)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != NULLSTR)
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array

    return temp
}

# Split a string into characters.
function explode(string, array) {
    split(string, array, NULLSTR)
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    case "u0026":
        return "&" # Unicode Character 'AMPERSAND'
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, cc, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    explode(string, s)
    string = NULLSTR
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            if (cc) {
                cc = cc c
                if (length(cc) == 5) {
                    string = string escapeChar(cc)
                    escaping = 0 # escape ends
                    cc = NULLSTR
                }
            } else if (c == "u") {
                cc = c
            } else {
                string = string escapeChar(c)
                escaping = 0 # escape ends
            }
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/\\/, "\\\\", string) # substitute backslashes first
    gsub(/"/, "\\\"", string)

    return string
}

# Reverse of escape(string).
function unescape(string) {
    gsub(/\\\"/, "\"", string)
    gsub(/\\\\/, "\\", string) # substitute backslashes last

    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Reverse of parameterize(string, quotationMark).
function unparameterize(string,    temp) {
    match(string, /^'(.*)'$/, temp)
    if (temp[0]) { # use temp[0] (there IS a match for quoted empty string)
        string = temp[1]
        gsub(/'\\''/, "'", string)
        return string
    }
    match(string, /^"(.*)"$/, temp)
    if (temp[0]) {
        string = temp[1]
        return unescape(string)
    }
    return string
}

# Convert any value to human-readable string.
function toString(value, inline, heredoc, valOnly, numSub, level, sortedIn,
                  ####
                  i, items, j, k, p, saveSortedIn, temp, v) {
    if (!level) level = 0
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    if (isarray(value)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        p = 0
        for (i in value) {
            split(i, j, SUBSEP); k = join(j, ",")
            if (!numSub || !isnum(k)) k = parameterize(k, "\"")
            v = toString(value[i], inline, heredoc, valOnly, numSub, level + 1, sortedIn)
            if (!isarray(value[i])) v = parameterize(v, "\"")
            if (valOnly)
                items[p++] = inline ? v : (replicate("\t", level) v)
            else
                items[p++] = inline ? (k ": " v) :
                    (replicate("\t", level) k "\t" v)
        }
        PROCINFO["sorted_in"] = saveSortedIn
        temp = inline ? join(items, ", ") :
            ("\n" join(items, "\n") "\n" replicate("\t", level))
        temp = valOnly ? ("[" temp "]") : ("{" temp "}")
        return temp
    } else {
        if (heredoc)
            return "'''\n" value "\n'''"
        else
            return value
    }
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, NULLSTR, line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, NULLSTR, line)
    # Remove in-line comment
    gsub(/#[^"/]*$/, NULLSTR, line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, NULLSTR, line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 0 if the string starts with '0', 'f', 'n' or 'off';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^([0fn]|off)/) ? 0 : 1
}

## Display & Debugging:

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("yellow", text)) > STDERR
}

# What a terrible failure.
function wtf(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Debug any value.
function da(value, name, inline, heredoc, valOnly, numSub, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!name)
        name = "_"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    d(name " = " toString(value, inline, heredoc, valOnly, numSub, 0, sortedIn))
    #if (isarray(value)) {
    #    saveSortedIn = PROCINFO["sorted_in"]
    #    PROCINFO["sorted_in"] = sortedIn
    #    for (i in value) {
    #        split(i, j, SUBSEP)
    #        da(value[i], sprintf(name "[%s]", join(j, ",")), sortedIn)
    #    }
    #    PROCINFO["sorted_in"] = saveSortedIn
    #} else
    #    d(name " = " value)
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

## URLs:

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\t"] = "%09"
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = NULLSTR
    explode(string, s)
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

## System:

# Read from a file and return its content.
function readFrom(file,    line, text) {
    if (!file) file = "/dev/stdin"
    text = NULLSTR
    while (getline line < file)
        text = (text ? text "\n" : NULLSTR) line
    return text
}

# Write text to file.
function writeTo(text, file) {
    if (!file) file = "/dev/stdout"
    print text > file
}

# Return the output of a command.
function getOutput(command,    content, line) {
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Return non-zero if file exists; otherwise return 0.
function fileExists(file) {
    return !system("test -f " parameterize(file))
}

# Return non-zero if file exists and is a directory; otherwise return 0.
function dirExists(file) {
    return !system("test -d " parameterize(file))
}

# Detect whether a program exists in path.
# Return the name (or output) if the program call writes anything to stdout;
# Otherwise, return a null string.
function detectProgram(prog, arg, returnOutput,    temp) {
    if (returnOutput) {
        prog " " arg SUPERR | getline temp
        return temp
    } else
        return (prog " " arg SUPERR | getline) ? prog : NULLSTR
}

# Return the HEAD revision if the current directory is a git repo;
# Otherwise return a null string.
function getGitHead(    line, group) {
    if (fileExists(".git/HEAD")) {
        getline line < ".git/HEAD"
        match(line, /^ref: (.*)$/, group)
        if (fileExists(".git/" group[1])) {
            getline line < (".git/" group[1])
            return substr(line, 1, 7)
        } else
            return NULLSTR
    } else
        return NULLSTR
}

BEGIN {
    initConst()
    initAnsiCode()
    initUrlEncoding()
    initUriSchemes()
}

####################################################################
# Commons.awk                                                      #
####################################################################

# Initialize constants.
function initConst() {
    NULLSTR = ""
    TRUE = 1

    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
    PIPE = " | "
}

## Arrays:

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Return 1 if the value is non-empty or an array that contains anything;
# Otherwise, return 0.
function exists(value) {
    if (isarray(value))
        return anything(value)
    else
        return value ? 1 : 0
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return NULLSTR
}

# Return non-zero if two values are identical;
# Otherwise, return 0.
function identical(x, y,
                   ####
                   i) {
    if (!isarray(x) && !isarray(y))
        return x == y
    else if (isarray(x) && isarray(y)) {
        if (length(x) != length(y)) return 0
        for (i in x)
            if (!identical(x[i], y[i])) return 0
        return 1
    } else
        return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

## Strings:

# Return non-zero if the string represents a numeral;
# Otherwise, return 0.
function isnum(string) {
    return string == string + 0
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return NULLSTR
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return NULLSTR
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = NULLSTR
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = NULLSTR
    j = 0
    if (isarray(array)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != NULLSTR)
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array

    return temp
}

# Split a string into characters.
function explode(string, array) {
    split(string, array, NULLSTR)
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    case "u0026":
        return "&" # Unicode Character 'AMPERSAND'
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, cc, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    explode(string, s)
    string = NULLSTR
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            if (cc) {
                cc = cc c
                if (length(cc) == 5) {
                    string = string escapeChar(cc)
                    escaping = 0 # escape ends
                    cc = NULLSTR
                }
            } else if (c == "u") {
                cc = c
            } else {
                string = string escapeChar(c)
                escaping = 0 # escape ends
            }
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/\\/, "\\\\", string) # substitute backslashes first
    gsub(/"/, "\\\"", string)

    return string
}

# Reverse of escape(string).
function unescape(string) {
    gsub(/\\\"/, "\"", string)
    gsub(/\\\\/, "\\", string) # substitute backslashes last

    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Reverse of parameterize(string, quotationMark).
function unparameterize(string,    temp) {
    match(string, /^'(.*)'$/, temp)
    if (temp[0]) { # use temp[0] (there IS a match for quoted empty string)
        string = temp[1]
        gsub(/'\\''/, "'", string)
        return string
    }
    match(string, /^"(.*)"$/, temp)
    if (temp[0]) {
        string = temp[1]
        return unescape(string)
    }
    return string
}

# Convert any value to human-readable string.
function toString(value, inline, heredoc, valOnly, numSub, level, sortedIn,
                  ####
                  i, items, j, k, p, saveSortedIn, temp, v) {
    if (!level) level = 0
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    if (isarray(value)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = sortedIn
        p = 0
        for (i in value) {
            split(i, j, SUBSEP); k = join(j, ",")
            if (!numSub || !isnum(k)) k = parameterize(k, "\"")
            v = toString(value[i], inline, heredoc, valOnly, numSub, level + 1, sortedIn)
            if (!isarray(value[i])) v = parameterize(v, "\"")
            if (valOnly)
                items[p++] = inline ? v : (replicate("\t", level) v)
            else
                items[p++] = inline ? (k ": " v) :
                    (replicate("\t", level) k "\t" v)
        }
        PROCINFO["sorted_in"] = saveSortedIn
        temp = inline ? join(items, ", ") :
            ("\n" join(items, "\n") "\n" replicate("\t", level))
        temp = valOnly ? ("[" temp "]") : ("{" temp "}")
        return temp
    } else {
        if (heredoc)
            return "'''\n" value "\n'''"
        else
            return value
    }
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, NULLSTR, line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, NULLSTR, line)
    # Remove in-line comment
    gsub(/#[^"/]*$/, NULLSTR, line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, NULLSTR, line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 0 if the string starts with '0', 'f', 'n' or 'off';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^([0fn]|off)/) ? 0 : 1
}

## Display & Debugging:

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("yellow", text)) > STDERR
}

# What a terrible failure.
function wtf(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Debug any value.
function da(value, name, inline, heredoc, valOnly, numSub, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!name)
        name = "_"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    d(name " = " toString(value, inline, heredoc, valOnly, numSub, 0, sortedIn))
    #if (isarray(value)) {
    #    saveSortedIn = PROCINFO["sorted_in"]
    #    PROCINFO["sorted_in"] = sortedIn
    #    for (i in value) {
    #        split(i, j, SUBSEP)
    #        da(value[i], sprintf(name "[%s]", join(j, ",")), sortedIn)
    #    }
    #    PROCINFO["sorted_in"] = saveSortedIn
    #} else
    #    d(name " = " value)
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

## URLs:

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = NULLSTR
    explode(string, s)
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

## System:

# Read from a file and return its content.
function readFrom(file,    line, text) {
    if (!file) file = "/dev/stdin"
    text = NULLSTR
    while (getline line < file)
        text = (text ? text "\n" : NULLSTR) line
    return text
}

# Write text to file.
function writeTo(text, file) {
    if (!file) file = "/dev/stdout"
    print text > file
}

# Return the output of a command.
function getOutput(command,    content, line) {
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Return non-zero if file exists; otherwise return 0.
function fileExists(file) {
    return !system("test -f " parameterize(file))
}

# Return non-zero if file exists and is a directory; otherwise return 0.
function dirExists(file) {
    return !system("test -d " parameterize(file))
}

# Detect whether a program exists in path.
# Return the name (or output) if the program call writes anything to stdout;
# Otherwise, return a null string.
function detectProgram(prog, arg, returnOutput,    temp) {
    if (returnOutput) {
        prog " " arg SUPERR | getline temp
        return temp
    } else
        return (prog " " arg SUPERR | getline) ? prog : NULLSTR
}

# Return the HEAD revision if the current directory is a git repo;
# Otherwise return a null string.
function getGitHead(    line, group) {
    if (fileExists(".git/HEAD")) {
        getline line < ".git/HEAD"
        match(line, /^ref: (.*)$/, group)
        if (fileExists(".git/" group[1])) {
            getline line < (".git/" group[1])
            return substr(line, 1, 7)
        } else
            return NULLSTR
    } else
        return NULLSTR
}

BEGIN {
    initConst()
    initAnsiCode()
    initUrlEncoding()
    initUriSchemes()
}


parsegaspriceshtml.awk: Updated for Marathon. Disabled broken code for UDF.

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

// {
   sub("^.*
]*>","")
   sub("
.*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   sub("^[ ]*","",address[i,ai[i]])
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && / <a href="/ {
   gsub("","Marathon")
   gsub("","BP")
   gsub("","Thorntons")
   gsub("","Thorntons")
   gsub("","Certified")
   gsub("","Marathon")
   # gsub("","UDF")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.* ",",")
   sub("<.*$","")
   split($0,foo,",")
   reporter[i]=foo[1]
   time[i]=foo[2]
   timeAgo[i]=foo[3]
   # print
}

inGasPriceRecord && /<\/tr>/ {
   i ++
   # print "EOR"
   # print $0
   inGasPriceRecord=False
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

// {
   sub("^.*
]*>","")
   sub("
.*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   sub("^[ ]*","",address[i,ai[i]])
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && / <a href="/ {
   gsub("","Marathon")
   gsub("","BP")
   gsub("","Thorntons")
   gsub("","Thorntons")
   gsub("","Certified")
   gsub("","Marathon")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.* ",",")
   sub("<.*$","")
   split($0,foo,",")
   reporter[i]=foo[1]
   time[i]=foo[2]
   timeAgo[i]=foo[3]
   # print
}

inGasPriceRecord && /<\/tr>/ {
   i ++
   # print "EOR"
   # print $0
   inGasPriceRecord=False
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}