Add support for case/esac blocks in test cases

#!/usr/bin/awk -f
#
# Append "|| bsda:test:err …" to every command in a shell script.
#

BEGIN {
	# Define states
	ST_REG = 0      # Regular state
	ST_SQUOT = 1    # Inside single quotes
	ST_DQUOT = 2    # Inside double quotes
	ST_DSQUOT = 3   # Inside dollar single quotes
	ST_CASE_BLK = 4 # Inside case/esac
	ST_CASE_REG = 5 # Inside case match

	TOP = 0
	STATE[TOP] = ST_REG
	REG_STATES[ST_REG]
	REG_STATES[ST_CASE_REG]
}

#
# Manage single and double quotes.
#
# This maintains a state stack, representing the nesting of states.
#
# The current line is parsed character by character, the characters
# are interpreted according to the top of the state stack. Depending
# on the characters encountered a new state is pushed on top of the
# stack or the top is pointed back down to the previous state.
#
# If by the end of the line the stack top is not at the bottom the
# line is printed as is and the next line is pulled in to finish
# parsing.
#
{
	do {
		for (i = 1; i <= length; ++i) {
			ch = substr($0, i, 1)
			front = substr($0, i)
			# Single quotes are simple
			if (STATE[TOP] == ST_SQUOT) {
				if (ch == "'") {
					--TOP
				}
				continue
			}
			# All the following states support \ escapes
			if (ch == "\\") {
				++i
				continue
			}
			# Double quotes
			if (STATE[TOP] == ST_DQUOT) {
				if (ch == "\"") {
					--TOP
				} else if (ch == "$") {
					# Peek ahead
					ch = substr($0, i, 2)
					if (ch == "$(") {
						++i
						STATE[++TOP] = ST_REG
					}
					if (ch == "$'") {
						++i
						STATE[++TOP] = ST_DSQUOT
					}
				}
				continue
			}
			# Dollar single quotes
			if (STATE[TOP] == ST_DSQUOT) {
				if (ch == "'") {
					--TOP
				}
				continue
			}
			# Inside case match
			if (STATE[TOP] == ST_CASE_REG) {
				if (front ~ /^;;.*/) {
					++i
					--TOP
					continue
				}
			}
			# Inside case block
			if (STATE[TOP] == ST_CASE_BLK) {
				if (ch == ")") {
					STATE[++TOP] = ST_CASE_REG
					# Output the parsed section so
					# a check is only appended if the
					# closing parentheses are followed
					# by code
					printf("%s", substr($0, 1, i))
					$0 = substr($0, i + 1)
					continue
				}
				if (front ~ /^esac([ \t#].*)?/) {
					--TOP
					i += 3
					continue
				}
			}
			# Regular state, or command substitution
			if (ch == "'") {
				STATE[++TOP] = ST_SQUOT
			} else if (ch == "\"") {
				STATE[++TOP] = ST_DQUOT
			} else if (ch == "$") {
				# Peek ahead
				ch = substr($0, i, 2)
				if (ch == "$'") {
					++i
					STATE[++TOP] = ST_DSQUOT
				}
			} else if (ch == "(") {
				STATE[++TOP] = ST_REG
			} else if (ch == ")") {
				--TOP
			} else if (front ~ /^case[ \t].+[ \t]in([ \t#].*)?/) {
				STATE[++TOP] = ST_CASE_BLK
			}
		}
		# About to get the next line, print this one
		if (!(STATE[TOP] in REG_STATES)) {
			print
		}
	} while (!(STATE[TOP] in REG_STATES) && getline > 0)
}

# Do not append error checks to lines not ending in a command.
/((^|then|else|do|[({&;])|[ \t]*\\)[ \t]*$/ {
	print
	next
}

# Append the error check.
{
	print $0 " || bsda:test:err \"" FILENAME "\" " NR " $?"
}

#!/usr/bin/awk -f
#
# Append "|| bsda:test:err …" to every command in a shell script.
#

BEGIN {
	# Define states
	ST_REG = 0    # Regular state
	ST_SQUOT = 1  # Inside single quotes
	ST_DQUOT = 2  # Inside double quotes
	ST_DSQUOT = 3 # Inside dollar single quotes

	TOP = 0
	STATE[TOP] = ST_REG
}

#
# Manage single and double quotes.
#
# This maintains a state stack, representing the nesting of states.
#
# The current line is parsed character by character, the characters
# are interpreted according to the top of the state stack. Depending
# on the characters encountered a new state is pushed on top of the
# stack or the top is pointed back down to the previous state.
#
# If by the end of the line the stack top is not at the bottom the
# line is printed as is and the next line is pulled in to finish
# parsing.
#
{
	do {
		for (i = 1; i <= length; ++i) {
			ch = substr($0, i, 1)
			# Single quotes are simple
			if (STATE[TOP] == ST_SQUOT) {
				if (ch == "'") {
					--TOP
				}
				continue
			}
			# All the following states support \ escapes
			if (ch == "\\") {
				++i
				continue
			}
			# Double quotes
			if (STATE[TOP] == ST_DQUOT) {
				if (ch == "\"") {
					--TOP
				} else if (ch == "$") {
					# Peek ahead
					ch = substr($0, i, 2)
					if (ch == "$(") {
						++i
						STATE[++TOP] = ST_REG
					}
					if (ch == "$'") {
						++i
						STATE[++TOP] = ST_DSQUOT
					}
				}
				continue
			}
			# Dollar single quotes
			if (STATE[TOP] == ST_DSQUOT) {
				if (ch == "'") {
					--TOP
				}
				continue
			}
			# Regular state, or command substitution
			if (ch == "'") {
				STATE[++TOP] = ST_SQUOT
			} else if (ch == "\"") {
				STATE[++TOP] = ST_DQUOT
			} else if (ch == "$") {
				# Peek ahead
				ch = substr($0, i, 2)
				if (ch == "$'") {
					++i
					STATE[++TOP] = ST_DSQUOT
				}
			} else if (ch == "(") {
				STATE[++TOP] = ST_REG
			} else if (ch == ")") {
				--TOP
			}
		}
		# About to get the next line, print this one
		if (STATE[TOP] != ST_REG) {
			print
		}
	} while (STATE[TOP] != ST_REG && getline > 0)
}

# Do not append error checks to lines not ending in a command.
/((^|then|else|do|in|[({&;])|[ \t]*\\)$/ {
	print
	next
}

# Append the error check.
{
	print $0 " || bsda:test:err \"" FILENAME "\" " NR " $?"
}


TestUtils: add test cases for dump()

TestUtils: add test cases for dump()

@include "include/Utils"

BEGIN {
    START_TEST("Utils.awk")

    T("GawkVersion", 1)
    {
        initGawk()
        assertTrue(GawkVersion ~ "^4.")
    }

    T("Rlwrap", 1)
    {
        initRlwrap()
        assertEqual(Rlwrap, "rlwrap")
    }

    T("Emacs", 1)
    {
        initEmacs()
        assertEqual(Emacs, "emacs")
    }

    T("newerVersion()", 5)
    {
        assertTrue(newerVersion("0.9", "0.8"))
        assertTrue(newerVersion("0.9.0.1", "0.9.0"))
        assertTrue(newerVersion("1.0", "0.9.9999"))
        assertTrue(newerVersion("1.9.9999", "1.9.10"))
        assertTrue(newerVersion("2", "1.9.9999"))
    }

    T("dump()", 3)
    {
        delete group
        assertEqual(dump("a", group), 1)

        delete group
        assertEqual(dump("Århus", group), 6)

        delete group
        assertEqual(dump("안녕하세요 세계", group), 22)
    }

    END_TEST()
}

@include "include/Utils"

BEGIN {
    START_TEST("Utils.awk")

    T("GawkVersion", 1)
    {
        initGawk()
        assertTrue(GawkVersion ~ "^4.")
    }

    T("Rlwrap", 1)
    {
        initRlwrap()
        assertEqual(Rlwrap, "rlwrap")
    }

    T("Emacs", 1)
    {
        initEmacs()
        assertEqual(Emacs, "emacs")
    }

    T("newerVersion()", 5)
    {
        assertTrue(newerVersion("0.9", "0.8"))
        assertTrue(newerVersion("0.9.0.1", "0.9.0"))
        assertTrue(newerVersion("1.0", "0.9.9999"))
        assertTrue(newerVersion("1.9.9999", "1.9.10"))
        assertTrue(newerVersion("2", "1.9.9999"))
    }

    END_TEST()
}


Correct field numbers used in verification

# Verify that every show in BBoxPrograms is also in BBoxEpisodes

# INVOCATION:
#    awk -f fixExtraLinesFrom-webscraper.awk $PROGRAMS_FILE | sort -df --field-separator=$',' --key=3 |
#        awk -v EPISODES_FILE=$EPISODES_FILE -v SEASONS_FILE=$SEASONS_FILE \
#        -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#        -f verifyBBoxDownloadsFrom-webscraper.awk

# Field numbers
#    1 web-scraper-order  2 web-scraper-start-url  3 URL        4 Program_Title   5 Sn_Years
#    6 Seasons            7  Mv_Year               8  Duration  9  Rating        10  Description

# Example input from scraping with getBBoxProgramsFrom-webscraper.json
# "1523123734-54","https://www.britbox.com/us/programmes","/us/movie/Christmas_Lights_15512",\
#      "Christmas Lights"
# "1523201776-3607","https://www.britbox.com/us/programmes","/us/show/Are_You_Being_Served_6429",\
#      "Are You Being Served?"

BEGIN {
    FS="\""
}

/^"/ {
    programURL = $6
    nflds = split (programURL,fld,"/")
    showType = fld[3]
    showType == "show" ? spacer = "  " : spacer = " "
    target = sprintf ("'%s' ",fld[nflds])
    #
    cmd = "grep -c " target " " EPISODES_FILE
    if ((cmd | getline NumEpisodes) > 0) {
        NumEpisodes == 1 ? epiStr = " episode" : epiStr = " episodes"
        print "==> " showType spacer target "has " NumEpisodes epiStr >> EPISODE_INFO_FILE
    }
    close (cmd)
    if (NumEpisodes == 0) {
        badEpisodes += 1
        print "==> " showType spacer target "has " NumEpisodes epiStr >> ERROR_FILE
    }
    #
    cmd = "grep " target " " SEASONS_FILE
    while ((cmd | getline seasonLine) > 0) {
        split (seasonLine, fields, "\"")
        seasonField = fields[16]
        # print "seasonField = " seasonField
        episodeField = fields[20]
        # print "episodeField = " episodeField
        if (seasonField == "")
            continue
        print "          " target seasonField " has "episodeField >> EPISODE_INFO_FILE
        # print "          " target seasonField " has "episodeField
    }
    close (cmd)
}

END {
    print "==> " badEpisodes " missing episodes"
}

# Verify that every show in BBoxPrograms is also in BBoxEpisodes

# INVOCATION:
#    awk -f fixExtraLinesFrom-webscraper.awk $PROGRAMS_FILE | sort -df --field-separator=$',' --key=3 |
#        awk -v EPISODES_FILE=$EPISODES_FILE -v SEASONS_FILE=$SEASONS_FILE \
#        -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#        -f verifyBBoxDownloadsFrom-webscraper.awk

# Field numbers
#    1 web-scraper-order  2 web-scraper-start-url  3 URL        4 Program_Title   5 Sn_Years
#    6 Seasons            7  Mv_Year               8  Duration  9  Rating        10  Description

# Example input from scraping with getBBoxProgramsFrom-webscraper.json
# "1523123734-54","https://www.britbox.com/us/programmes","/us/movie/Christmas_Lights_15512",\
#      "Christmas Lights"
# "1523201776-3607","https://www.britbox.com/us/programmes","/us/show/Are_You_Being_Served_6429",\
#      "Are You Being Served?"

BEGIN {
    FS="\""
}

/^"/ {
    programURL = $6
    nflds = split (programURL,fld,"/")
    showType = fld[3]
    showType == "show" ? spacer = "  " : spacer = " "
    target = sprintf ("'%s' ",fld[nflds])
    #
    cmd = "grep -c " target " " EPISODES_FILE
    if ((cmd | getline NumEpisodes) > 0) {
        NumEpisodes == 1 ? epiStr = " episode" : epiStr = " episodes"
        print "==> " showType spacer target "has " NumEpisodes epiStr >> EPISODE_INFO_FILE
    }
    close (cmd)
    if (NumEpisodes == 0) {
        badEpisodes += 1
        print "==> " showType spacer target "has " NumEpisodes epiStr >> ERROR_FILE
    }
    #
    cmd = "grep " target " " SEASONS_FILE
    while ((cmd | getline seasonLine) > 0) {
        split (seasonLine, fields, "\"")
        seasonField = fields[20]
        episodeField = fields[24]
        if (seasonField == "")
            continue
        print "          " target seasonField " has "episodeField >> EPISODE_INFO_FILE
    }
    close (cmd)
}

END {
    print "==> " badEpisodes " missing episodes"
}


improve readability of script

improve readability of script

{
    title = $0
    if (match (title, /, The"$/)) {
        sub (/^"/,"\"The ", title)
        sub (/, The"$/,"\"", title)
    }
    printf ("echo '==> %3d: %s'\n",NR,title) >> TITLES_SCRIPT
    printf ("./getIMDbInfoFrom-titles.py %s\n",title) >> TITLES_SCRIPT
    printf ("echo \n\n") >> TITLES_SCRIPT

    printf ("echo '==> %3d: %s'\n",NR,title) >> ID_SCRIPT
    printf ("./getIMDb_IDsFrom-titles.py %s | head -7\n",title) >> ID_SCRIPT
    printf ("echo \n\n") >> ID_SCRIPT
}

{
    title = $0
    if (match (title, /, The"$/)) {
        sub (/^"/,"\"The ", title)
        sub (/, The"$/,"\"", title)
    }
    printf ("echo ==\\> %3d: %s\n",NR,title) >> TITLES_SCRIPT
    printf ("./getIMDbInfoFrom-titles.py %s\n",title) >> TITLES_SCRIPT
    printf ("echo \n\n") >> TITLES_SCRIPT

    printf ("echo ==\\> %3d: %s\n",NR,title) >> ID_SCRIPT
    printf ("./getIMDb_IDsFrom-titles.py %s | head -7\n",title) >> ID_SCRIPT
    printf ("echo \n\n") >> ID_SCRIPT
}


remove check_miss.awk

remove check_miss.awk

BEGIN{
	#file = $0
	current = ""
	while (getline)
	{
		if ($1 ~ /^0,/)
		{
			if ($2 != "(referee")
			{
				player[$3]
			}
			else
			{
				current = $1
				for (p in player)
				{
					player[p] = 1
				}
			}
		}
		else if (($1 !~ /^3000,/) && ($1 !~ /^6000,/))
		{
			new = $1
			#print current
			if (current == new)
			{
				if ($2 != "(referee")
				{
					 player[$3] = 1
				}
			}
			else
			{
				for (p in player)
				{
					if (player[p] == 0 && p !~ /Coach/)
					{
						print current " " p " missed!"
					}
					else
					{
						player[p] = 0
					}
				}
				current = new
				if ($2 != "(referee")
				{
					 player[$3] = 1
				}
			}
		}
	}
}
END{
	print "end"
}


show dictionary

show dictionary

####################################################################
# Translate.awk                                                    #
####################################################################

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mplayer >/dev/null 2>/dev/null") ?
        "mplayer" :
        (!system("mpv >/dev/null 2>/dev/null") ?
         "mpv" :
         (!system("mpg123 >/dev/null 2>/dev/null") ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say '' >/dev/null 2>/dev/null") ?
        "say" :
        (!system("espeak '' >/dev/null 2>/dev/null") ?
         "espeak" :
         "")
}

# Initialize `HttpService`.
function initHttpService() {
    HttpProtocol = "http://"
    HttpHost = "translate.google.com"
    HttpPort = 80
    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpService = "/inet/tcp/0/" HttpProxySpec[2] "/" HttpProxySpec[3]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/inet/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP request and get response from Google Translate.
function getResponse(text, sl, tl, hl,    content, url) {
    url = HttpPathPrefix "/translate_a/single?client=t"                 \
        "&ie=UTF-8&oe=UTF-8"                                            \
        "&dt=bd&dt=ex&dt=ld&dt=md&dt=qca&dt=rw&dt=rm&dt=ss&dt=t&dt=at"  \
        "&q=" preprocess(text) "&sl=" sl "&tl=" tl "&hl=" hl

    print "GET " url " HTTP/1.1\n"             \
          "Host: " HttpHost "\n"               \
          "Connection: close\n" |& HttpService
    while ((HttpService |& getline) > 0)
        content = $0
    close(HttpService)

    return assert(content, "[ERROR] Null response.")
}

# Play using Google Text-to-Speech engine.
function play(text, tl,    url) {
    url = HttpProtocol HttpHost "/translate_tts?ie=UTF-8"       \
        "&tl=" tl "&q=" preprocess(text)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    system(Option["player"] " '" url "' >/dev/null 2>/dev/null")
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist,
                        ####
                        altTranslations, article, ast, content, example,
                        explanation, group, i, il, ils, isPhonetic, j, k,
                        oExamples, oPhonetics, oRefs, oSeeAlso, oSynonyms,
                        oWords, oWordClasses, original, phonetics,
                        r, ref, rtl, saveSortedIn, segments,
                        temp, tokens, translation, translations,
                        word, words, wordClasses) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (getCode(tl) != "auto" && rtl = Locale[getCode(tl)]["rtl"]) {
        # Check if target language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(tl)]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    content = getResponse(text, sl, tl, hl)

    plTokenize(tokens, content)
    plParse(ast, tokens)

    if (!anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
    }

    # Debug mode
    if (Option["debug"]) {
        d(content)
        da(tokens, "tokens[%s]='%s'")
        da(ast, "ast[%s]='%s'")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in ast) {
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "0$")
            append(translations, postprocess(literal(ast[i])))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "1$")
            append(original, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "2$")
            append(phonetics, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "3$")
            append(oPhonetics, literal(ast[i]))

        # 1 - word classes and explanations
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            wordClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]][group[3]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]]["1"][group[3]] = literal(ast[i])

        # 5 - alternative translations
        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group)) {
            segments[group[1]] = literal(ast[i])
            altTranslations[group[1]][0] = ""
        }
        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            altTranslations[group[1]][group[2]] = postprocess(literal(ast[i]))

        # 8 - identified source languages
        if (i ~ "^0" SUBSEP "8" SUBSEP "0" SUBSEP "[[:digit:]]+$" ||
            i ~ "^0" SUBSEP "2$")
            append(ils, literal(ast[i]))

        # 11 - (original) word classes and synonyms
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oWordClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "1$", group))
            if (ast[i]) {
                oRefs[literal(ast[i])][1] = group[1]
                oRefs[literal(ast[i])][2] = group[2]
            }
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0" SUBSEP "([[:digit:]]+)$", group))
            oSynonyms[group[1]][group[2]][group[3]] = literal(ast[i])

        # 12 - (original) word classes and explanations
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oWordClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oWords[group[1]][group[2]][0] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "1$", group))
            oWords[group[1]][group[2]][1] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2$", group))
            oWords[group[1]][group[2]][2] = postprocess(literal(ast[i]))

        # 13 - (original) examples
        if (match(i, "^0" SUBSEP "13" SUBSEP "0" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oExamples[group[1]] = postprocess(literal(ast[i]))

        # 14 - (original) see also
        if (match(i, "^0" SUBSEP "14" SUBSEP "0" SUBSEP "([[:digit:]]+)$", group))
            oSeeAlso[group[1]] = literal(ast[i])
    }
    PROCINFO["sorted_in"] = saveSortedIn

    translation = join(translations)

    il = !anything(ils) || belongsTo(sl, ils) ? sl : ils[0]

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && anything(phonetics) ?
            join(phonetics) :  # phonetic transcription
            s(translation, tl) # target language

        if (toSpeech) {
            returnPlaylist[0]["text"] = translation
            returnPlaylist[0]["tl"] = tl
        }

    } else {
        # Verbose mode

        r = AnsiCode["bold"] show(join(original), il) AnsiCode["no bold"]
        if (anything(oPhonetics))
            r = r "\n" AnsiCode["bold"] "/" join(oPhonetics) "/" AnsiCode["no bold"]
        r = r "\n"
        # Dictionary mode
        if (isarray(oWordClasses) && anything(oWordClasses)) {
            for (i = 0; i < length(oWordClasses); i++) {
                r = r "\n" AnsiCode["negative"] AnsiCode["bold"]        \
                    show(oWordClasses[i], hl) AnsiCode["no bold"] AnsiCode["positive"]
                if (isarray(oWords[i])) {
                    # Detailed explanation
                    for (j = 0; j < length(oWords[i]); j++) {
                        explanation = oWords[i][j][0]
                        ref = oWords[i][j][1]
                        example = oWords[i][j][2]

                        r = r "\n    " AnsiCode["bold"] show(explanation, il) AnsiCode["no bold"]
                        if (example)
                            r = r "\n    - \"" show(example, il) "\""
                        if (ref && isarray(oRefs[ref])) {
                            r = r "\n    * synonyms: " AnsiCode["underline"] \
                                show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][0], il) AnsiCode["no underline"]
                            for (k = 1; k < length(oSynonyms[oRefs[ref][1]][oRefs[ref][2]]); k++)
                                r = r ", " AnsiCode["underline"]        \
                                    show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][k], il) AnsiCode["no underline"]
                        }
                        r = r "\n"
                    }
                } else {
                    # Synonyms only
                    for (j = 0; j < length(oSynonyms[i]); j++) {
                        r = r "\n  * " show(oSynonyms[i][j][0], il)
                        for (k = 1; k < length(oSynonyms[i][j]); k++)
                            r = r ", " show(oSynonyms[i][j][k], il)
                    }
                    r = r "\n"
                }
            }
        }
        # Examples
        if (isarray(oExamples) && anything(oExamples)) {
            r = r "\n" "Examples:"
            for (i = 0; i < length(oExamples); i++) {
                example = oExamples[i]
                sub(/\u003cb\u003e/, AnsiCode["negative"], example)
                sub(/\u003c\/b\u003e/, AnsiCode["positive"], example)
                r = r "\n  - " show(example, il)
                r = r "\n"
            }
        }
        # See also
        if (isarray(oSeeAlso) && anything(oSeeAlso)) {
            r = r "\n" "See also:"
            r = r "\n" AnsiCode["underline"] show(oSeeAlso[0], il) AnsiCode["no underline"]
            for (k = 1; k < length(oSeeAlso); k++)
                r = r ", " AnsiCode["underline"] show(oSeeAlso[k], il) AnsiCode["no underline"]
            r = r "\n"
        }

        r = r "\n" AnsiCode["bold"] s(translation, tl) AnsiCode["no bold"] # target language
        if (anything(phonetics))
            r = r "\n" AnsiCode["bold"] join(phonetics) AnsiCode["no bold"] # phonetic transcription

        if (isarray(altTranslations[0]) && anything(altTranslations[0])) {
            # List alternative translations

            if (Locale[getCode(hl)]["rtl"] || Locale[getCode(il)]["rtl"])
                r = r "\n\n" s(sprintf(Locale[getCode(hl)]["message"], join(original))) # caution: mixed languages, BiDi invoked must be implemented correctly (i.e. FriBidi is required)
            else
                r = r "\n\n" sprintf(Locale[getCode(hl)]["message"], join(original))
            if (Locale[getCode(il)]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" s("(" Locale[getCode(il)]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")") # caution: mixed languages
            else
                r = r "\n" "(" Locale[getCode(il)]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")"

            temp = segments[0] "(" join(altTranslations[0], "/") ")"
            for (i = 1; i < length(altTranslations); i++)
                temp = temp " " segments[i] "(" join(altTranslations[i], "/") ")"
            if (Locale[getCode(il)]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" AnsiCode["bold"] s(temp) AnsiCode["no bold"] # caution: mixed languages
            else
                r = r "\n" AnsiCode["bold"] temp AnsiCode["no bold"]
        }

        if (isarray(wordClasses) && anything(wordClasses)) {
            # List dictionary entries

            for (i = 0; i < length(words); i++) {
                r = r "\n\n" s("[" wordClasses[i] "]", hl) # home language
                for (j = 0; j < length(words[i]); j++) {
                    word = words[i][j][0]
                    explanation = join(words[i][j][1], ", ")
                    article = words[i][j][4]

                    if (rtl) {
                        r = r "\n" AnsiCode["bold"] sprintf("%" Option["width"] - 4 "s", s((article ?
                                                                                "(" article ")" :
                                                                                "") " " word, tl, Option["width"] - 4)) AnsiCode["no bold"] # target language
                        r = r "\n" s(explanation, il, Option["width"] - 8) # identified source language
                    } else {
                        r = r "\n" "    " AnsiCode["bold"] show((article ?
                                                                 "(" article ") " :
                                                                 "") word, tl) AnsiCode["no bold"] # target language
                        r = r "\n" "        " s(explanation, il, Option["width"] - 8) # identified source language
                    }
                }
            }
        }

        if (toSpeech) {
            if (index(Locale[getCode(hl)]["message"], "%s") > 2) {
                returnPlaylist[0]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[0]["tl"] = hl
                returnPlaylist[1]["text"] = join(original)
                returnPlaylist[1]["tl"] = il
            } else {
                returnPlaylist[0]["text"] = join(original)
                returnPlaylist[0]["tl"] = il
                returnPlaylist[1]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[1]["tl"] = hl
            }
            returnPlaylist[2]["text"] = translation
            returnPlaylist[2]["tl"] = tl
        }
    }

    return r
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl) {
    system(Option["browser"] " " parameterize("https://translate.google.com/translate?" \
                                              "hl=" hl "&sl=" sl "&tl=" tl "&u=" uri) "&")
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (getCode(Option["hl"]) != "auto" && Locale[getCode(Option["hl"])]["rtl"]) {
        # Check if home language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["hl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (getCode(Option["sl"]) != "auto" && Locale[getCode(Option["sl"])]["rtl"]) {
        # Check if source language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["sl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                print replicate("─", Option["width"])

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            print getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"], playlist) > Option["output"]

            if (Option["play"])
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    i = 0
    while (getline line < Option["input"]) {
        # Non-interactive verbose mode: separator between sources
        if (!Option["interactive"])
            if (Option["verbose"] && i++ > 0)
                print replicate("═", Option["width"])

        if (Option["interactive"]) {
            if (line ~ /:(q|quit)/)
                exit
            else if (line ~ /:(s|source)/)
                print Option["sl"]
            else if (line ~ /:(t|target)/) {
                printf "(" Option["tl"][1]
                for (i = 2; i <= length(Option["tl"]); i++)
                    printf ", " Option["tl"][i]
                print ")"
            }

            else {
                translate(line)

                # Interactive verbose mode: newline after each translation
                if (Option["verbose"]) printf "\n"
            }

            prompt()
        } else
            translate(line)
    }
}

####################################################################
# Translate.awk                                                    #
####################################################################

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mplayer >/dev/null 2>/dev/null") ?
        "mplayer" :
        (!system("mpv >/dev/null 2>/dev/null") ?
         "mpv" :
         (!system("mpg123 >/dev/null 2>/dev/null") ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say '' >/dev/null 2>/dev/null") ?
        "say" :
        (!system("espeak '' >/dev/null 2>/dev/null") ?
         "espeak" :
         "")
}

# Initialize `HttpService`.
function initHttpService() {
    HttpProtocol = "http://"
    HttpHost = "translate.google.com"
    HttpPort = 80
    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpService = "/inet/tcp/0/" HttpProxySpec[2] "/" HttpProxySpec[3]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/inet/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP request and get response from Google Translate.
function getResponse(text, sl, tl, hl,    content, url) {
    url = HttpPathPrefix "/translate_a/single?client=t"                 \
        "&ie=UTF-8&oe=UTF-8"                                            \
        "&dt=bd&dt=ex&dt=ld&dt=md&dt=qca&dt=rw&dt=rm&dt=ss&dt=t&dt=at"  \
        "&q=" preprocess(text) "&sl=" sl "&tl=" tl "&hl=" hl

    print "GET " url " HTTP/1.1\n"             \
          "Host: " HttpHost "\n"               \
          "Connection: close\n" |& HttpService
    while ((HttpService |& getline) > 0)
        content = $0
    close(HttpService)

    return assert(content, "[ERROR] Null response.")
}

# Play using Google Text-to-Speech engine.
function play(text, tl,    url) {
    url = HttpProtocol HttpHost "/translate_tts?ie=UTF-8"       \
        "&tl=" tl "&q=" preprocess(text)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    system(Option["player"] " '" url "' >/dev/null 2>/dev/null")
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist,
                        ####
                        altTranslations, article, ast, content,
                        explanation, group, i, il, ils,
                        isPhonetic, j, original, phonetics,
                        r, rtl, saveSortedIn, segments,
                        temp, tokens, translation, translations,
                        word, words, wordClasses) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (getCode(tl) != "auto" && rtl = Locale[getCode(tl)]["rtl"]) {
        # Check if target language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(tl)]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    content = getResponse(text, sl, tl, hl)

    plTokenize(tokens, content)
    plParse(ast, tokens)

    if (!anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
    }

    # Debug mode
    if (Option["debug"]) {
        d(content)
        da(tokens, "tokens[%s]='%s'")
        da(ast, "ast[%s]='%s'")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in ast) {
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "0$")
            append(translations, postprocess(literal(ast[i])))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "1$")
            append(original, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "2$")
            append(phonetics, literal(ast[i]))

        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            wordClasses[group[1]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]][group[3]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]]["1"][group[3]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group)) {
            segments[group[1]] = literal(ast[i])
            altTranslations[group[1]][0] = ""
        }

        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            altTranslations[group[1]][group[2]] = postprocess(literal(ast[i]))

        # Identified source languages
        if (i ~ "^0" SUBSEP "8" SUBSEP "0" SUBSEP "[[:digit:]]+$" ||
            i ~ "^0" SUBSEP "2$")
            append(ils, literal(ast[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn

    translation = join(translations)

    il = !anything(ils) || belongsTo(sl, ils) ? sl : ils[0]

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && anything(phonetics) ?
            join(phonetics) :  # phonetic transcription
            s(translation, tl) # target language

        if (toSpeech) {
            returnPlaylist[0]["text"] = translation
            returnPlaylist[0]["tl"] = tl
        }

    } else {
        # Verbose mode

        r = AnsiCode["bold"] s(translation, tl) AnsiCode["no bold"] # target language
        if (anything(phonetics))
            r = r "\n" AnsiCode["bold"] join(phonetics) AnsiCode["no bold"] # phonetic transcription

        if (isarray(altTranslations[0]) && anything(altTranslations[0])) {
            # List alternative translations

            if (Locale[getCode(hl)]["rtl"] || Locale[getCode(il)]["rtl"])
                r = r "\n\n" s(sprintf(Locale[getCode(hl)]["message"], join(original))) # caution: mixed languages, BiDi invoked must be implemented correctly (i.e. FriBidi is required)
            else
                r = r "\n\n" sprintf(Locale[getCode(hl)]["message"], join(original))
            if (Locale[getCode(il)]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" s("(" Locale[getCode(il)]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")") # caution: mixed languages
            else
                r = r "\n" "(" Locale[getCode(il)]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")"

            temp = segments[0] "(" join(altTranslations[0], "/") ")"
            for (i = 1; i < length(altTranslations); i++)
                temp = temp " " segments[i] "(" join(altTranslations[i], "/") ")"
            if (Locale[getCode(il)]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" AnsiCode["bold"] s(temp) AnsiCode["no bold"] # caution: mixed languages
            else
                r = r "\n" AnsiCode["bold"] temp AnsiCode["no bold"]
        }

        if (isarray(wordClasses) && anything(wordClasses)) {
            # List dictionary entries

            for (i = 0; i < length(words); i++) {
                r = r "\n\n" s("[" wordClasses[i] "]", hl) # home language
                for (j = 0; j < length(words[i]); j++) {
                    word = words[i][j][0]
                    explanation = join(words[i][j][1], ", ")
                    article = words[i][j][4]

                    if (rtl) {
                        r = r "\n" AnsiCode["bold"] sprintf("%" Option["width"] - 4 "s", s((article ?
                                                                                "(" article ")" :
                                                                                "") " " word, tl, Option["width"] - 4)) AnsiCode["no bold"] # target language
                        r = r "\n" s(explanation, il, Option["width"] - 8) # identified source language
                    } else {
                        r = r "\n" "    " AnsiCode["bold"] show((article ?
                                                                 "(" article ") " :
                                                                 "") word, tl) AnsiCode["no bold"] # target language
                        r = r "\n" "        " s(explanation, il, Option["width"] - 8) # identified source language
                    }
                }
            }
        }

        if (toSpeech) {
            if (index(Locale[getCode(hl)]["message"], "%s") > 2) {
                returnPlaylist[0]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[0]["tl"] = hl
                returnPlaylist[1]["text"] = join(original)
                returnPlaylist[1]["tl"] = il
            } else {
                returnPlaylist[0]["text"] = join(original)
                returnPlaylist[0]["tl"] = il
                returnPlaylist[1]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[1]["tl"] = hl
            }
            returnPlaylist[2]["text"] = translation
            returnPlaylist[2]["tl"] = tl
        }
    }

    return r
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl) {
    system(Option["browser"] " " parameterize("https://translate.google.com/translate?" \
                                              "hl=" hl "&sl=" sl "&tl=" tl "&u=" uri) "&")
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (getCode(Option["hl"]) != "auto" && Locale[getCode(Option["hl"])]["rtl"]) {
        # Check if home language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["hl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (getCode(Option["sl"]) != "auto" && Locale[getCode(Option["sl"])]["rtl"]) {
        # Check if source language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["sl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                print replicate("─", Option["width"])

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            print getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"], playlist) > Option["output"]

            if (Option["play"])
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    i = 0
    while (getline line < Option["input"]) {
        # Non-interactive verbose mode: separator between sources
        if (!Option["interactive"])
            if (Option["verbose"] && i++ > 0)
                print replicate("═", Option["width"])

        if (Option["interactive"]) {
            if (line ~ /:(q|quit)/)
                exit
            else if (line ~ /:(s|source)/)
                print Option["sl"]
            else if (line ~ /:(t|target)/) {
                printf "(" Option["tl"][1]
                for (i = 2; i <= length(Option["tl"]); i++)
                    printf ", " Option["tl"][i]
                print ")"
            }

            else {
                translate(line)

                # Interactive verbose mode: newline after each translation
                if (Option["verbose"]) printf "\n"
            }

            prompt()
        } else
            translate(line)
    }
}


Move down the functions.

Move down the functions.

#!/usr/bin/awk -f

BEGIN {
	"date \"+%Y-%m-%d %H:%M:%S\"" | getline now
	print "# gfw list ipset rules for dnsmasq"
	print "# updated on " now
	print "#"

	host = host != "" ? host : "127.0.0.1"
	port = port != "" ? port : 5353
}

{
	original_line = $0

	if (/^$/ || /^#/ || /^\!/ || /^\[/ || /^@@/) {
		# Empty line

		# #
		# comments in user_rule.txt

		# !
		# comments
		# see https://adblockplus.org/en/filters#comments

		# [
		# comments

		# @@
		# whitelist
		# see https://adblockplus.org/en/filters#whitelist
	} else if (/^\|\|/) {
		# ||
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|\|/, "") # remove leading ||
		extract($0, original_line)
	} else if (/^\|/) {
		# |
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|/, "") # removing leading |
		extract($0, original_line)
	} else if (/^\/.*\/$/) {
		# /.../
		# https://adblockplus.org/en/filters#regexps
		sub(/^\//, "") # remove leading /
		sub(/\/$/, "") # remove tailing /
		extract_regex($0, original_line)
	} else if (/^\./) {
		# start with dot
		extract($0, original_line)
	} else {
		extract($0, original_line)
	}
}

END {
	for (i = 1; i <= domain_count; i++) {
		printf "server=/.%s/%s#%d\n", domains[i], host, port | "sort | uniq"
	}
	close("sort | uniq")
}

# Extract domain name from non-regex line
function extract(line, original_line) {
	sub(/.*:\/\//, "", line) # remove everything till ://, such as http:// https://
	sub(/\/.*/, "", line)    # remove everything from /
	sub(/:.*/, "", line)     # remove everything from :
	sub(/\*/, "", line)      # remove *
	sub(/^\./, "", line)     # remove leading dot

	if (match(line, /([A-Za-z0-9\-]+\.)+[A-Za-z0-9\-]+/)) {
		domains[++domain_count] = line
	} else {
		print "[WARN] <NR>" NR "</NR><original>" original_line "</original><extracted>" line "</extracted>" | "cat >&2"
		close("cat >&2")
	}
}

# Extract domain name from regex line
function extract_regex(line, original_line) {

	# Expand the lines like (aa|bb|cc) into multiple records
	pos = match(line, /\([A-Za-z0-9\.\|]+\)/)
	if (pos != 0) {
		in_bracket = substr(line, RSTART + 1, RLENGTH - 2)
		n = split(in_bracket, arr, "|")
		for (i = 1; i <= n; i++) {
			expanded_line = line
			sub(/\([A-Za-z0-9\.\|]+\)/, arr[i], expanded_line)
			extract_regex(expanded_line, original_line)
		}
	} else {
		sub(/.*:\\\/\\\//, "", line)    # remove everything till :\/\/ in regex, :// in plain text, such as http:// https://
		gsub(/\([^\)]+\)\*?/, "", line) # remove (...)*? such as ([^\/]+\.)
		gsub(/\[.*\]\+/, "", line)      # remove [...]
		sub(/\\\/.*/, "", line)         # remove everything from \/ in regex, / in plaint text
		gsub(/\\\./, ".", line)         # replace \. to .
		extract(line, original_line)
	}
}

#!/usr/bin/awk -f

# Extract domain name from non-regex line
function extract(line, original_line) {
	sub(/.*:\/\//, "", line) # remove everything till ://, such as http:// https://
	sub(/\/.*/, "", line)    # remove everything from /
	sub(/:.*/, "", line)     # remove everything from :
	sub(/\*/, "", line)      # remove *
	sub(/^\./, "", line)     # remove leading dot

	if (match(line, /([A-Za-z0-9\-]+\.)+[A-Za-z0-9\-]+/)) {
		domains[++domain_count] = line
	} else {
		print "[WARN] <NR>" NR "</NR><original>" original_line "</original><extracted>" line "</extracted>" | "cat >&2"
		close("cat >&2")
	}
}

# Extract domain name from regex line
function extract_regex(line, original_line) {

	# Expand the lines like (aa|bb|cc) into multiple records
	pos = match(line, /\([A-Za-z0-9\.\|]+\)/)
	if (pos != 0) {
		in_bracket = substr(line, RSTART + 1, RLENGTH - 2)
		n = split(in_bracket, arr, "|")
		for (i = 1; i <= n; i++) {
			expanded_line = line
			sub(/\([A-Za-z0-9\.\|]+\)/, arr[i], expanded_line)
			extract_regex(expanded_line, original_line)
		}
	} else {
		sub(/.*:\\\/\\\//, "", line)    # remove everything till :\/\/ in regex, :// in plain text, such as http:// https://
		gsub(/\([^\)]+\)\*?/, "", line) # remove (...)*? such as ([^\/]+\.)
		gsub(/\[.*\]\+/, "", line)      # remove [...]
		sub(/\\\/.*/, "", line)         # remove everything from \/ in regex, / in plaint text
		gsub(/\\\./, ".", line)         # replace \. to .
		extract(line, original_line)
	}
}

BEGIN {
	"date \"+%Y-%m-%d %H:%M:%S\"" | getline now
	print "# gfw list ipset rules for dnsmasq"
	print "# updated on " now
	print "#"

	host = host != "" ? host : "127.0.0.1"
	port = port != "" ? port : 5353
}
{
	original_line = $0

	if (/^$/ || /^#/ || /^\!/ || /^\[/ || /^@@/) {
		# Empty line

		# #
		# comments in user_rule.txt

		# !
		# comments
		# see https://adblockplus.org/en/filters#comments

		# [
		# comments

		# @@
		# whitelist
		# see https://adblockplus.org/en/filters#whitelist
	} else if (/^\|\|/) {
		# ||
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|\|/, "") # remove leading ||
		extract($0, original_line)
	} else if (/^\|/) {
		# |
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|/, "") # removing leading |
		extract($0, original_line)
	} else if (/^\/.*\/$/) {
		# /.../
		# https://adblockplus.org/en/filters#regexps
		sub(/^\//, "") # remove leading /
		sub(/\/$/, "") # remove tailing /
		extract_regex($0, original_line)
	} else if (/^\./) {
		# start with dot
		extract($0, original_line)
	} else {
		extract($0, original_line)
	}
}
END {
	for (i = 1; i <= domain_count; i++) {
		printf "server=/.%s/%s#%d\n", domains[i], host, port | "sort | uniq"
	}
	close("sort | uniq")
}


[rnc2rng.awk] Support namespaced element and attribute names


x86: Don't use POSIX character classes in gen-insn-attr-x86.awk

Not all awk implementations (including the default awk in Ubuntu 9.10)
support POSIX character classes.  Since x86-opcode-map.txt is plain
ASCII, we can just use explicit ranges for lower case, alphabetic, and
alphanumeric characters instead.

Signed-off-by: Roland Dreier <91e9b5f7ca0bb6300133ed378670d64af90dde66@cisco.com>
Acked-by: Masami Hiramatsu <308c3db2ad395e67f502db11a9ceb1603d4442e0@redhat.com>
LKML-Reference: <114d9d49de02dde04eecef799da927cfb6fa13bf@roland-alpha.cisco.com>
Signed-off-by: H. Peter Anvin <8a453bad9912ffe59bc0f0b8abe03df9be19379e@zytor.com>

#!/bin/awk -f
# gen-insn-attr-x86.awk: Instruction attribute table generator
# Written by Masami Hiramatsu <mhiramat@redhat.com>
#
# Usage: awk -f gen-insn-attr-x86.awk x86-opcode-map.txt > inat-tables.c

# Awk implementation sanity check
function check_awk_implement() {
	if (sprintf("%x", 0) != "0")
		return "Your awk has a printf-format problem."
	return ""
}

# Clear working vars
function clear_vars() {
	delete table
	delete lptable2
	delete lptable1
	delete lptable3
	eid = -1 # escape id
	gid = -1 # group id
	aid = -1 # AVX id
	tname = ""
}

BEGIN {
	# Implementation error checking
	awkchecked = check_awk_implement()
	if (awkchecked != "") {
		print "Error: " awkchecked > "/dev/stderr"
		print "Please try to use gawk." > "/dev/stderr"
		exit 1
	}

	# Setup generating tables
	print "/* x86 opcode map generated from x86-opcode-map.txt */"
	print "/* Do not change this code. */\n"
	ggid = 1
	geid = 1
	gaid = 0
	delete etable
	delete gtable
	delete atable

	opnd_expr = "^[A-Za-z/]"
	ext_expr = "^\\("
	sep_expr = "^\\|$"
	group_expr = "^Grp[0-9A-Za-z]+"

	imm_expr = "^[IJAO][a-z]"
	imm_flag["Ib"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Jb"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Iw"] = "INAT_MAKE_IMM(INAT_IMM_WORD)"
	imm_flag["Id"] = "INAT_MAKE_IMM(INAT_IMM_DWORD)"
	imm_flag["Iq"] = "INAT_MAKE_IMM(INAT_IMM_QWORD)"
	imm_flag["Ap"] = "INAT_MAKE_IMM(INAT_IMM_PTR)"
	imm_flag["Iz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Jz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Iv"] = "INAT_MAKE_IMM(INAT_IMM_VWORD)"
	imm_flag["Ob"] = "INAT_MOFFSET"
	imm_flag["Ov"] = "INAT_MOFFSET"

	modrm_expr = "^([CDEGMNPQRSUVW/][a-z]+|NTA|T[012])"
	force64_expr = "\\([df]64\\)"
	rex_expr = "^REX(\\.[XRWB]+)*"
	fpu_expr = "^ESC" # TODO

	lprefix1_expr = "\\(66\\)"
	lprefix2_expr = "\\(F3\\)"
	lprefix3_expr = "\\(F2\\)"
	max_lprefix = 4

	vexok_expr = "\\(VEX\\)"
	vexonly_expr = "\\(oVEX\\)"

	prefix_expr = "\\(Prefix\\)"
	prefix_num["Operand-Size"] = "INAT_PFX_OPNDSZ"
	prefix_num["REPNE"] = "INAT_PFX_REPNE"
	prefix_num["REP/REPE"] = "INAT_PFX_REPE"
	prefix_num["LOCK"] = "INAT_PFX_LOCK"
	prefix_num["SEG=CS"] = "INAT_PFX_CS"
	prefix_num["SEG=DS"] = "INAT_PFX_DS"
	prefix_num["SEG=ES"] = "INAT_PFX_ES"
	prefix_num["SEG=FS"] = "INAT_PFX_FS"
	prefix_num["SEG=GS"] = "INAT_PFX_GS"
	prefix_num["SEG=SS"] = "INAT_PFX_SS"
	prefix_num["Address-Size"] = "INAT_PFX_ADDRSZ"
	prefix_num["2bytes-VEX"] = "INAT_PFX_VEX2"
	prefix_num["3bytes-VEX"] = "INAT_PFX_VEX3"

	clear_vars()
}

function semantic_error(msg) {
	print "Semantic error at " NR ": " msg > "/dev/stderr"
	exit 1
}

function debug(msg) {
	print "DEBUG: " msg
}

function array_size(arr,   i,c) {
	c = 0
	for (i in arr)
		c++
	return c
}

/^Table:/ {
	print "/* " $0 " */"
	if (tname != "")
		semantic_error("Hit Table: before EndTable:.");
}

/^Referrer:/ {
	if (NF != 1) {
		# escape opcode table
		ref = ""
		for (i = 2; i <= NF; i++)
			ref = ref $i
		eid = escape[ref]
		tname = sprintf("inat_escape_table_%d", eid)
	}
}

/^AVXcode:/ {
	if (NF != 1) {
		# AVX/escape opcode table
		aid = $2
		if (gaid <= aid)
			gaid = aid + 1
		if (tname == "")	# AVX only opcode table
			tname = sprintf("inat_avx_table_%d", $2)
	}
	if (aid == -1 && eid == -1)	# primary opcode table
		tname = "inat_primary_table"
}

/^GrpTable:/ {
	print "/* " $0 " */"
	if (!($2 in group))
		semantic_error("No group: " $2 )
	gid = group[$2]
	tname = "inat_group_table_" gid
}

function print_table(tbl,name,fmt,n)
{
	print "const insn_attr_t " name " = {"
	for (i = 0; i < n; i++) {
		id = sprintf(fmt, i)
		if (tbl[id])
			print "	[" id "] = " tbl[id] ","
	}
	print "};"
}

/^EndTable/ {
	if (gid != -1) {
		# print group tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1, tname "_1[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2, tname "_2[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3, tname "_3[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,3] = tname "_3"
		}
	} else {
		# print primary/escaped tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,0] = tname
			if (aid >= 0)
				atable[aid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1,tname "_1[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,1] = tname "_1"
			if (aid >= 0)
				atable[aid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2,tname "_2[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,2] = tname "_2"
			if (aid >= 0)
				atable[aid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3,tname "_3[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,3] = tname "_3"
			if (aid >= 0)
				atable[aid,3] = tname "_3"
		}
	}
	print ""
	clear_vars()
}

function add_flags(old,new) {
	if (old && new)
		return old " | " new
	else if (old)
		return old
	else
		return new
}

# convert operands to flags.
function convert_operands(count,opnd,       i,j,imm,mod)
{
	imm = null
	mod = null
	for (j = 1; j <= count; j++) {
		i = opnd[j]
		if (match(i, imm_expr) == 1) {
			if (!imm_flag[i])
				semantic_error("Unknown imm opnd: " i)
			if (imm) {
				if (i != "Ib")
					semantic_error("Second IMM error")
				imm = add_flags(imm, "INAT_SCNDIMM")
			} else
				imm = imm_flag[i]
		} else if (match(i, modrm_expr))
			mod = "INAT_MODRM"
	}
	return add_flags(imm, mod)
}

/^[0-9a-f]+\:/ {
	if (NR == 1)
		next
	# get index
	idx = "0x" substr($1, 1, index($1,":") - 1)
	if (idx in table)
		semantic_error("Redefine " idx " in " tname)

	# check if escaped opcode
	if ("escape" == $2) {
		if ($3 != "#")
			semantic_error("No escaped name")
		ref = ""
		for (i = 4; i <= NF; i++)
			ref = ref $i
		if (ref in escape)
			semantic_error("Redefine escape (" ref ")")
		escape[ref] = geid
		geid++
		table[idx] = "INAT_MAKE_ESCAPE(" escape[ref] ")"
		next
	}

	variant = null
	# converts
	i = 2
	while (i <= NF) {
		opcode = $(i++)
		delete opnds
		ext = null
		flags = null
		opnd = null
		# parse one opcode
		if (match($i, opnd_expr)) {
			opnd = $i
			count = split($(i++), opnds, ",")
			flags = convert_operands(count, opnds)
		}
		if (match($i, ext_expr))
			ext = $(i++)
		if (match($i, sep_expr))
			i++
		else if (i < NF)
			semantic_error($i " is not a separator")

		# check if group opcode
		if (match(opcode, group_expr)) {
			if (!(opcode in group)) {
				group[opcode] = ggid
				ggid++
			}
			flags = add_flags(flags, "INAT_MAKE_GROUP(" group[opcode] ")")
		}
		# check force(or default) 64bit
		if (match(ext, force64_expr))
			flags = add_flags(flags, "INAT_FORCE64")

		# check REX prefix
		if (match(opcode, rex_expr))
			flags = add_flags(flags, "INAT_MAKE_PREFIX(INAT_PFX_REX)")

		# check coprocessor escape : TODO
		if (match(opcode, fpu_expr))
			flags = add_flags(flags, "INAT_MODRM")

		# check VEX only code
		if (match(ext, vexonly_expr))
			flags = add_flags(flags, "INAT_VEXOK | INAT_VEXONLY")

		# check VEX only code
		if (match(ext, vexok_expr))
			flags = add_flags(flags, "INAT_VEXOK")

		# check prefixes
		if (match(ext, prefix_expr)) {
			if (!prefix_num[opcode])
				semantic_error("Unknown prefix: " opcode)
			flags = add_flags(flags, "INAT_MAKE_PREFIX(" prefix_num[opcode] ")")
		}
		if (length(flags) == 0)
			continue
		# check if last prefix
		if (match(ext, lprefix1_expr)) {
			lptable1[idx] = add_flags(lptable1[idx],flags)
			variant = "INAT_VARIANT"
		} else if (match(ext, lprefix2_expr)) {
			lptable2[idx] = add_flags(lptable2[idx],flags)
			variant = "INAT_VARIANT"
		} else if (match(ext, lprefix3_expr)) {
			lptable3[idx] = add_flags(lptable3[idx],flags)
			variant = "INAT_VARIANT"
		} else {
			table[idx] = add_flags(table[idx],flags)
		}
	}
	if (variant)
		table[idx] = add_flags(table[idx],variant)
}

END {
	if (awkchecked != "")
		exit 1
	# print escape opcode map's array
	print "/* Escape opcode map array */"
	print "const insn_attr_t const *inat_escape_tables[INAT_ESC_MAX + 1]" \
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < geid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (etable[i,j])
				print "	["i"]["j"] = "etable[i,j]","
	print "};\n"
	# print group opcode map's array
	print "/* Group opcode map array */"
	print "const insn_attr_t const *inat_group_tables[INAT_GRP_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < ggid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (gtable[i,j])
				print "	["i"]["j"] = "gtable[i,j]","
	print "};\n"
	# print AVX opcode map's array
	print "/* AVX opcode map array */"
	print "const insn_attr_t const *inat_avx_tables[X86_VEX_M_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < gaid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (atable[i,j])
				print "	["i"]["j"] = "atable[i,j]","
	print "};"
}

#!/bin/awk -f
# gen-insn-attr-x86.awk: Instruction attribute table generator
# Written by Masami Hiramatsu <mhiramat@redhat.com>
#
# Usage: awk -f gen-insn-attr-x86.awk x86-opcode-map.txt > inat-tables.c

# Awk implementation sanity check
function check_awk_implement() {
	if (!match("abc", "[[:lower:]]+"))
		return "Your awk doesn't support charactor-class."
	if (sprintf("%x", 0) != "0")
		return "Your awk has a printf-format problem."
	return ""
}

# Clear working vars
function clear_vars() {
	delete table
	delete lptable2
	delete lptable1
	delete lptable3
	eid = -1 # escape id
	gid = -1 # group id
	aid = -1 # AVX id
	tname = ""
}

BEGIN {
	# Implementation error checking
	awkchecked = check_awk_implement()
	if (awkchecked != "") {
		print "Error: " awkchecked > "/dev/stderr"
		print "Please try to use gawk." > "/dev/stderr"
		exit 1
	}

	# Setup generating tables
	print "/* x86 opcode map generated from x86-opcode-map.txt */"
	print "/* Do not change this code. */\n"
	ggid = 1
	geid = 1
	gaid = 0
	delete etable
	delete gtable
	delete atable

	opnd_expr = "^[[:alpha:]/]"
	ext_expr = "^\\("
	sep_expr = "^\\|$"
	group_expr = "^Grp[[:alnum:]]+"

	imm_expr = "^[IJAO][[:lower:]]"
	imm_flag["Ib"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Jb"] = "INAT_MAKE_IMM(INAT_IMM_BYTE)"
	imm_flag["Iw"] = "INAT_MAKE_IMM(INAT_IMM_WORD)"
	imm_flag["Id"] = "INAT_MAKE_IMM(INAT_IMM_DWORD)"
	imm_flag["Iq"] = "INAT_MAKE_IMM(INAT_IMM_QWORD)"
	imm_flag["Ap"] = "INAT_MAKE_IMM(INAT_IMM_PTR)"
	imm_flag["Iz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Jz"] = "INAT_MAKE_IMM(INAT_IMM_VWORD32)"
	imm_flag["Iv"] = "INAT_MAKE_IMM(INAT_IMM_VWORD)"
	imm_flag["Ob"] = "INAT_MOFFSET"
	imm_flag["Ov"] = "INAT_MOFFSET"

	modrm_expr = "^([CDEGMNPQRSUVW/][[:lower:]]+|NTA|T[012])"
	force64_expr = "\\([df]64\\)"
	rex_expr = "^REX(\\.[XRWB]+)*"
	fpu_expr = "^ESC" # TODO

	lprefix1_expr = "\\(66\\)"
	lprefix2_expr = "\\(F3\\)"
	lprefix3_expr = "\\(F2\\)"
	max_lprefix = 4

	vexok_expr = "\\(VEX\\)"
	vexonly_expr = "\\(oVEX\\)"

	prefix_expr = "\\(Prefix\\)"
	prefix_num["Operand-Size"] = "INAT_PFX_OPNDSZ"
	prefix_num["REPNE"] = "INAT_PFX_REPNE"
	prefix_num["REP/REPE"] = "INAT_PFX_REPE"
	prefix_num["LOCK"] = "INAT_PFX_LOCK"
	prefix_num["SEG=CS"] = "INAT_PFX_CS"
	prefix_num["SEG=DS"] = "INAT_PFX_DS"
	prefix_num["SEG=ES"] = "INAT_PFX_ES"
	prefix_num["SEG=FS"] = "INAT_PFX_FS"
	prefix_num["SEG=GS"] = "INAT_PFX_GS"
	prefix_num["SEG=SS"] = "INAT_PFX_SS"
	prefix_num["Address-Size"] = "INAT_PFX_ADDRSZ"
	prefix_num["2bytes-VEX"] = "INAT_PFX_VEX2"
	prefix_num["3bytes-VEX"] = "INAT_PFX_VEX3"

	clear_vars()
}

function semantic_error(msg) {
	print "Semantic error at " NR ": " msg > "/dev/stderr"
	exit 1
}

function debug(msg) {
	print "DEBUG: " msg
}

function array_size(arr,   i,c) {
	c = 0
	for (i in arr)
		c++
	return c
}

/^Table:/ {
	print "/* " $0 " */"
	if (tname != "")
		semantic_error("Hit Table: before EndTable:.");
}

/^Referrer:/ {
	if (NF != 1) {
		# escape opcode table
		ref = ""
		for (i = 2; i <= NF; i++)
			ref = ref $i
		eid = escape[ref]
		tname = sprintf("inat_escape_table_%d", eid)
	}
}

/^AVXcode:/ {
	if (NF != 1) {
		# AVX/escape opcode table
		aid = $2
		if (gaid <= aid)
			gaid = aid + 1
		if (tname == "")	# AVX only opcode table
			tname = sprintf("inat_avx_table_%d", $2)
	}
	if (aid == -1 && eid == -1)	# primary opcode table
		tname = "inat_primary_table"
}

/^GrpTable:/ {
	print "/* " $0 " */"
	if (!($2 in group))
		semantic_error("No group: " $2 )
	gid = group[$2]
	tname = "inat_group_table_" gid
}

function print_table(tbl,name,fmt,n)
{
	print "const insn_attr_t " name " = {"
	for (i = 0; i < n; i++) {
		id = sprintf(fmt, i)
		if (tbl[id])
			print "	[" id "] = " tbl[id] ","
	}
	print "};"
}

/^EndTable/ {
	if (gid != -1) {
		# print group tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1, tname "_1[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2, tname "_2[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3, tname "_3[INAT_GROUP_TABLE_SIZE]",
				    "0x%x", 8)
			gtable[gid,3] = tname "_3"
		}
	} else {
		# print primary/escaped tables
		if (array_size(table) != 0) {
			print_table(table, tname "[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,0] = tname
			if (aid >= 0)
				atable[aid,0] = tname
		}
		if (array_size(lptable1) != 0) {
			print_table(lptable1,tname "_1[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,1] = tname "_1"
			if (aid >= 0)
				atable[aid,1] = tname "_1"
		}
		if (array_size(lptable2) != 0) {
			print_table(lptable2,tname "_2[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,2] = tname "_2"
			if (aid >= 0)
				atable[aid,2] = tname "_2"
		}
		if (array_size(lptable3) != 0) {
			print_table(lptable3,tname "_3[INAT_OPCODE_TABLE_SIZE]",
				    "0x%02x", 256)
			etable[eid,3] = tname "_3"
			if (aid >= 0)
				atable[aid,3] = tname "_3"
		}
	}
	print ""
	clear_vars()
}

function add_flags(old,new) {
	if (old && new)
		return old " | " new
	else if (old)
		return old
	else
		return new
}

# convert operands to flags.
function convert_operands(count,opnd,       i,j,imm,mod)
{
	imm = null
	mod = null
	for (j = 1; j <= count; j++) {
		i = opnd[j]
		if (match(i, imm_expr) == 1) {
			if (!imm_flag[i])
				semantic_error("Unknown imm opnd: " i)
			if (imm) {
				if (i != "Ib")
					semantic_error("Second IMM error")
				imm = add_flags(imm, "INAT_SCNDIMM")
			} else
				imm = imm_flag[i]
		} else if (match(i, modrm_expr))
			mod = "INAT_MODRM"
	}
	return add_flags(imm, mod)
}

/^[0-9a-f]+\:/ {
	if (NR == 1)
		next
	# get index
	idx = "0x" substr($1, 1, index($1,":") - 1)
	if (idx in table)
		semantic_error("Redefine " idx " in " tname)

	# check if escaped opcode
	if ("escape" == $2) {
		if ($3 != "#")
			semantic_error("No escaped name")
		ref = ""
		for (i = 4; i <= NF; i++)
			ref = ref $i
		if (ref in escape)
			semantic_error("Redefine escape (" ref ")")
		escape[ref] = geid
		geid++
		table[idx] = "INAT_MAKE_ESCAPE(" escape[ref] ")"
		next
	}

	variant = null
	# converts
	i = 2
	while (i <= NF) {
		opcode = $(i++)
		delete opnds
		ext = null
		flags = null
		opnd = null
		# parse one opcode
		if (match($i, opnd_expr)) {
			opnd = $i
			count = split($(i++), opnds, ",")
			flags = convert_operands(count, opnds)
		}
		if (match($i, ext_expr))
			ext = $(i++)
		if (match($i, sep_expr))
			i++
		else if (i < NF)
			semantic_error($i " is not a separator")

		# check if group opcode
		if (match(opcode, group_expr)) {
			if (!(opcode in group)) {
				group[opcode] = ggid
				ggid++
			}
			flags = add_flags(flags, "INAT_MAKE_GROUP(" group[opcode] ")")
		}
		# check force(or default) 64bit
		if (match(ext, force64_expr))
			flags = add_flags(flags, "INAT_FORCE64")

		# check REX prefix
		if (match(opcode, rex_expr))
			flags = add_flags(flags, "INAT_MAKE_PREFIX(INAT_PFX_REX)")

		# check coprocessor escape : TODO
		if (match(opcode, fpu_expr))
			flags = add_flags(flags, "INAT_MODRM")

		# check VEX only code
		if (match(ext, vexonly_expr))
			flags = add_flags(flags, "INAT_VEXOK | INAT_VEXONLY")

		# check VEX only code
		if (match(ext, vexok_expr))
			flags = add_flags(flags, "INAT_VEXOK")

		# check prefixes
		if (match(ext, prefix_expr)) {
			if (!prefix_num[opcode])
				semantic_error("Unknown prefix: " opcode)
			flags = add_flags(flags, "INAT_MAKE_PREFIX(" prefix_num[opcode] ")")
		}
		if (length(flags) == 0)
			continue
		# check if last prefix
		if (match(ext, lprefix1_expr)) {
			lptable1[idx] = add_flags(lptable1[idx],flags)
			variant = "INAT_VARIANT"
		} else if (match(ext, lprefix2_expr)) {
			lptable2[idx] = add_flags(lptable2[idx],flags)
			variant = "INAT_VARIANT"
		} else if (match(ext, lprefix3_expr)) {
			lptable3[idx] = add_flags(lptable3[idx],flags)
			variant = "INAT_VARIANT"
		} else {
			table[idx] = add_flags(table[idx],flags)
		}
	}
	if (variant)
		table[idx] = add_flags(table[idx],variant)
}

END {
	if (awkchecked != "")
		exit 1
	# print escape opcode map's array
	print "/* Escape opcode map array */"
	print "const insn_attr_t const *inat_escape_tables[INAT_ESC_MAX + 1]" \
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < geid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (etable[i,j])
				print "	["i"]["j"] = "etable[i,j]","
	print "};\n"
	# print group opcode map's array
	print "/* Group opcode map array */"
	print "const insn_attr_t const *inat_group_tables[INAT_GRP_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < ggid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (gtable[i,j])
				print "	["i"]["j"] = "gtable[i,j]","
	print "};\n"
	# print AVX opcode map's array
	print "/* AVX opcode map array */"
	print "const insn_attr_t const *inat_avx_tables[X86_VEX_M_MAX + 1]"\
	      "[INAT_LSTPFX_MAX + 1] = {"
	for (i = 0; i < gaid; i++)
		for (j = 0; j < max_lprefix; j++)
			if (atable[i,j])
				print "	["i"]["j"] = "atable[i,j]","
	print "};"
}


Adding color to loop flow

Adding color to loop flow

git-svn-id: 392d0e5ce652024192c571322d40dcc23d6d83b0@81 8a65f31f-a56f-4bd6-98e8-2755f64920ec

print "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>";
  print "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd\">";

  for(i=0;i<numHosts;i++) {
    lookup[hosts[i]] = i;
    printf "\n", hosts[i], i;
  }

  w = (numHosts-1) * xHostSpace + leftMargin + rightMargin;
  h = numLines * yLineSpace + topMargin + bottomMargin;

  printf "<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"%d\" height=\"%d\" viewBox=\"0 0 %d %d\">\n",w,h,w,h;

  insertStyleDefs();

  ystart = 60;
  yend = h;

  printf "<text x=\"%d\" y=\"%d\" class=\"label title-text\">%s</text>\n",
    (w/2),
    ystart-35,
    title;

  for(i=0;i<numTraces;i++) {

    if (label[i] == "")
      label[i] = hosts[i];

    printf "<text x=\"%d\" y=\"%d\" class=\"label host-text\">%s</text>\n",
      leftMargin+(i*xHostSpace),
      ystart-(15*(i%2)+2),
      label[i];

    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"element-line\" />\n", leftMargin+(i*xHostSpace), ystart, leftMargin+(i*xHostSpace), yend;
  }

  printf "   <map name=\"callflowmap\" id=\"callflowmap\">\n" > "imagemap";
}

func insertStyleDefs () {
  printf "<defs>\n\n</defs>\n";
}

func arrow(x,y,d,c) {

  printf "<polygon points=\"%d,%d %d,%d %d,%d %d,%d\" class=\"arrowhead\" style=\"fill: %s; stroke: %s;\"/>\n",
  x,y,
  x+5*d, y-3,
  x+3*d, y,
  x+5*d, y+3,
  color[c], color[c];
}

func line(x1,x2,y,output, c) {

  if (x1 == x2) {
    printf "<polyline points=\"%d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d\" fill=\"none\" class=\"traceline\" style=\"stroke: %s;\"/>\n",
      x1,y-18+7,
      x1+5,y-18+7,
      x1+12,y-15+7,
      x1+15,y-10+7,
      x1+15,y-5+7,
      x1+12,y-2+7,
      x1+5,y+7,
      x1,y+7,
      color[c];
    arrow(x1,y+7,1,c);

    xtext = x1 + 18;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"  onmouseover=\"return getFrame('frames/Frame%d.html');\" onmouseout=\"return nd();\"/>\n", $3, x1, y-yLineSpace+2, x1+15, y+7+1, $3, $3 >> "imagemap"

  } else if (x1<x2) {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,-1,c);

    xtext = x1 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"  onmouseover=\"return getFrame('frames/Frame%d.html');\" onmouseout=\"return nd();\"/>\n", $3, x1, y-yLineSpace+2, x2, y+1, $3, $3 >> "imagemap"

  } else {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,1,c);

    xtext = x2 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"  onmouseover=\"return getFrame('frames/Frame%d.html');\" onmouseout=\"return nd();\"/>\n", $3, x2, y-yLineSpace+2, x1, y+1, $3, $3 >> "imagemap"
  }

  printf "\n", $3;
  printf "<text x=\"%d\" y=\"%d\" class=\"pkt-text\">%s</text>\n", xtext, y-4, output;
  printf "\n";
}

{
  y = NR;
  y = y * yLineSpace + ystart;

  if($0 ~ "^#") {
    output = "";
    split($0, A, " ")

    for (i=2; i <= length(A); i++) {

      if (A[i] == "!")
        break;
      output = output " " A[i];
    }

    link = "";
    if (A[i] == "!") {

      i++;
      for(;i <= length(A); i++) {
        link = link " " A[i];
      }
    }

    if (link != "") {

      printf("<text x=\"%d\" y=\"%d\" class=\"link\">%s</text>\n", 50, y, output);
      printf("    <area href=\"%s\" coords=\"%d,%d,%d,%d\"/>\n", link, 50, y-yLineSpace+2, w, y+1) >> "imagemap";

    } else {

      printf("<text x=\"%d\" y=\"%d\" class=\"comment-text\">%s</text>\n", 50, y, output);
    }

  } else {

    if ($6 ~ "{([0-9]+)}") {
      str = $6
      gsub("{", "", str)
      gsub("}", "", str)

      colorId = strtonum(str)
      if ( colorId <= colors ) {
        c = colorId
      } else {

        # There are more sessions than configured colors.
        # choose the first color as default...
        c = 0
      }

    } else { c = 0 }

    l1 = sprintf("%s:%s", $4,$5);
    l2 = sprintf("%s:%s", $7,$8);
    for (i=0; i<numHosts; i++) {

      if(l1 ~ hosts[i]) {
        x1 = strtonum(lookup[hosts[i]]);
      }

      if(l2 ~ hosts[i]) {
        x2 = strtonum(lookup[hosts[i]]);
      }
    }

    x1 = x1 * xHostSpace + leftMargin;
    x2 = x2 * xHostSpace + leftMargin;
    if ((x1==x2) && (noLocalLoop==1)) {
      # Do nothing
    } else {
      # Print the line
      ORS = "";
      printf ("<text x=\"%d\" y=\"%d\" class=\"pkt-text\" xml:space=\"preserve\" ", leftMargin - 10, y)
      printf ("style=\"text-anchor: end;\">%d", $3)
      if ( showTime == "yes" ) {
        printf ("    %s", $1)
      }
      printf ("</text>\n")

      output = $9" "$10;

      gsub("SIP(/SDP|/XML|) *(Status|Request): *","", output);
      gsub(", with session description *$"," w/SDP",output);

      line(x1,x2,y,output,c);
    }
  }
}

END {
  printf "   </map>\n" >> "imagemap";
  printf "</svg>\n";
}

print "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>";
  print "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd\">";

  for(i=0;i<numHosts;i++) {
    lookup[hosts[i]] = i;
    printf "\n", hosts[i], i;
  }

  w = (numHosts-1) * xHostSpace + leftMargin + rightMargin;
  h = numLines * yLineSpace + topMargin + bottomMargin;

  printf "<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"%d\" height=\"%d\" viewBox=\"0 0 %d %d\">\n",w,h,w,h;

  insertStyleDefs();

  ystart = 60;
  yend = h;

  printf "<text x=\"%d\" y=\"%d\" class=\"label title-text\">%s</text>\n",
    (w/2),
    ystart-35,
    title;

  for(i=0;i<numTraces;i++) {

    if (label[i] == "")
      label[i] = hosts[i];

    printf "<text x=\"%d\" y=\"%d\" class=\"label host-text\">%s</text>\n",
      leftMargin+(i*xHostSpace),
      ystart-(15*(i%2)+2),
      label[i];

    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"element-line\" />\n", leftMargin+(i*xHostSpace), ystart, leftMargin+(i*xHostSpace), yend;
  }

  printf "   <map name=\"callflowmap\" id=\"callflowmap\">\n" > "imagemap";
}

func insertStyleDefs () {
  printf "<defs>\n\n</defs>\n";
}

func arrow(x,y,d,c) {

  printf "<polygon points=\"%d,%d %d,%d %d,%d %d,%d\" class=\"arrowhead\" style=\"fill: %s; stroke: %s;\"/>\n",
  x,y,
  x+5*d, y-3,
  x+3*d, y,
  x+5*d, y+3,
  color[c], color[c];
}

func line(x1,x2,y,output, c) {

  if (x1 == x2) {
    printf "<polyline points=\"%d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d\" fill=\"none\" class=\"traceline\"/>\n",
      x1,y-18+7,
      x1+5,y-18+7,
      x1+12,y-15+7,
      x1+15,y-10+7,
      x1+15,y-5+7,
      x1+12,y-2+7,
      x1+5,y+7,
      x1,y+7;
    arrow(x1,y+7,1);

    xtext = x1 + 18;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"  onmouseover=\"return getFrame('frames/Frame%d.html');\" onmouseout=\"return nd();\"/>\n", $3, x1, y-yLineSpace+2, x1+15, y+7+1, $3, $3 >> "imagemap"

  } else if (x1<x2) {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,-1,c);

    xtext = x1 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"  onmouseover=\"return getFrame('frames/Frame%d.html');\" onmouseout=\"return nd();\"/>\n", $3, x1, y-yLineSpace+2, x2, y+1, $3, $3 >> "imagemap"

  } else {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,1,c);

    xtext = x2 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"  onmouseover=\"return getFrame('frames/Frame%d.html');\" onmouseout=\"return nd();\"/>\n", $3, x2, y-yLineSpace+2, x1, y+1, $3, $3 >> "imagemap"
  }

  printf "\n", $3;
  printf "<text x=\"%d\" y=\"%d\" class=\"pkt-text\">%s</text>\n", xtext, y-4, output;
  printf "\n";
}

{
  y = NR;
  y = y * yLineSpace + ystart;

  if($0 ~ "^#") {
    output = "";
    split($0, A, " ")

    for (i=2; i <= length(A); i++) {

      if (A[i] == "!")
        break;
      output = output " " A[i];
    }

    link = "";
    if (A[i] == "!") {

      i++;
      for(;i <= length(A); i++) {
        link = link " " A[i];
      }
    }

    if (link != "") {

      printf("<text x=\"%d\" y=\"%d\" class=\"link\">%s</text>\n", 50, y, output);
      printf("    <area href=\"%s\" coords=\"%d,%d,%d,%d\"/>\n", link, 50, y-yLineSpace+2, w, y+1) >> "imagemap";

    } else {

      printf("<text x=\"%d\" y=\"%d\" class=\"comment-text\">%s</text>\n", 50, y, output);
    }

  } else {

    if ($6 ~ "{([0-9]+)}") {
      str = $6
      gsub("{", "", str)
      gsub("}", "", str)

      colorId = strtonum(str)
      if ( colorId <= colors ) {
        c = colorId
      } else {

        # There are more sessions than configured colors.
        # choose the first color as default...
        c = 0
      }

    } else { c = 0 }

    l1 = sprintf("%s:%s", $4,$5);
    l2 = sprintf("%s:%s", $7,$8);
    for (i=0; i<numHosts; i++) {

      if(l1 ~ hosts[i]) {
        x1 = strtonum(lookup[hosts[i]]);
      }

      if(l2 ~ hosts[i]) {
        x2 = strtonum(lookup[hosts[i]]);
      }
    }

    x1 = x1 * xHostSpace + leftMargin;
    x2 = x2 * xHostSpace + leftMargin;
    if ((x1==x2) && (noLocalLoop==1)) {
      # Do nothing
    } else {
      # Print the line
      ORS = "";
      printf ("<text x=\"%d\" y=\"%d\" class=\"pkt-text\" xml:space=\"preserve\" ", leftMargin - 10, y)
      printf ("style=\"text-anchor: end;\">%d", $3)
      if ( showTime == "yes" ) {
        printf ("    %s", $1)
      }
      printf ("</text>\n")

      output = $9" "$10;

      gsub("SIP(/SDP|/XML|) *(Status|Request): *","", output);
      gsub(", with session description *$"," w/SDP",output);

      line(x1,x2,y,output,c);
    }
  }
}

END {
  printf "   </map>\n" >> "imagemap";
  printf "</svg>\n";
}


Move GLN testing to 2015.

Move GLN testing to 2015.

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2005 && journal year (or journal end year) is lt or eq to the current year we are testing]
  current_year = 2015
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "Mz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,2) >= "NA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "doNotProcess"
  s[11] = "doesNotExist"
  sn = 12

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "doNP"
  sc[11] = "dNoE"
  scn = 12

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2005 && journal year (or journal end year) is lt or eq to the current year we are testing]
  current_year = 2014
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "Mz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,2) >= "NA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "doNotProcess"
  s[11] = "doesNotExist"
  sn = 12

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "doNP"
  sc[11] = "dNoE"
  scn = 12

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}


aux/sys/cmd/mapr.awk: add pseudo-hashbang for syntax colouring

#! /modernish/helper/script/for/awk -f
#
# awk script belonging to modernish sys/cmd/mapr.mm
#
# This converts the input to commands for the shell to 'eval', based
# on parameters inherited from the environment.
#
# --- begin license ---
# Copyright (c) 2018 Martijn Dekker <martijn@inlv.org>, Groningen, Netherlands
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# --- end license ---

function round8(num) {
	return (num % 8 == 0) ? num : (num - num % 8 + 8);
}

# Avoid writing very long physical lines of shell code. Some shells on old systems
# can't deal with physical lines exceeding the minimum guaranteed SIZE_MAX==65535.
function printsh(arg) {
	shlen += length(arg);
	if (shlen >= 65533) {	# SIZE_MAX minus backslash minus linefeed
		print "\\\n";	# line continuation
		shlen = length(arg);
	}
	print arg;
}

BEGIN {
	RS = ENVIRON["_Msh_Mo_d"];

	# Some awk versions need a dummy calculation to convert from string to number.
	opt_s = ENVIRON["_Msh_Mo_s"] + 0;
	opt_n = ENVIRON["_Msh_Mo_n"] + 0;
	opt_c = ENVIRON["_Msh_Mo_c"] + 0;
	opt_m = ENVIRON["_Msh_Mo_m"] + 0;
	arg_max = ENVIRON["_Msh_ARG_MAX"] + 0;

	L_cmd = 0;	# count total length of fixed arguments in command line
	for (i=1; i<ARGC; i++) {
		if (opt_m) {
			L_cmd += length(ARGV[i]) + 1;
		} else {
			# macOS seems to need special argument size aligning to avoid a buffer overflow.
			L_cmd += round8(length(ARGV[i]) + 1) + 8;
		}
	}
	ARGC=1;		# nuke arguments so awk won't take them as input files

	c = 0;		# count number of arguments per command invocation
	L = L_cmd;	# count total argument length per command invocation
	tL = 0;		# count total argument length per batch of commands
	cont = 0;	# flag for continuing with multiple awk invocations

}

NR == 1 {
	NR = ENVIRON["_Msh_M_NR"] + 0;	# number of records: inherit from previous batches

	ORS = "";			# no automatic output record separation
	shlen = 0;			# count length of physical line of shell-quoted arguments
	printsh("\"$@\" ");		# print fixed command line argument(s)
}

NR <= opt_s {
	next;
}

opt_n && NR > opt_n + opt_s {
	exit 0;
}

# main:
{
	if (opt_m) {
		L += length($0) + 1;
	} else {
		# macOS seems to need special argument size aligning to avoid a buffer overflow.
		L += round8(length($0) + 1) + 8;
	}

	# Check the counters.
	if ((opt_c && c >= opt_c) || (opt_m ? L >= opt_m : L >= arg_max)) {
		# Try not to process much more than 4 megs of arguments per batch.
		if ((tL += L) >= 4194304) {
			cont = 1;
			exit 0;
		}
		printsh("||_Msh_mapr_ckE \"$@\"||break;\"$@\" ");
		c = 0;
		L = L_cmd;
	}

	c++;

	# Output the record as a shell-quoted argument.
	numsegs = split($0, seg, /'/);
	if (numsegs == 0) {
		printsh("'' ");
	} else {
		for (n = 1; n <= numsegs; n++) {
			printsh( ("'") (seg[n]) (n < numsegs ? "'\\'" : "' ") );
		}
	}
}

END {
	if (NR) {
		printsh("||_Msh_mapr_ckE \"$@\"||break");
	}
	print ("\n_Msh_M_NR=") (cont ? NR + 1 : "RET0") ("\n");
}