Update min/maxx to be inclusive

Update min/maxx to be inclusive

see: https://trello.com/c/RU4BWrYA/340-as-a-data-manager-adding-min-max-values-in-the-jts-they-should-be-inclusive

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column number"
    rule_type = "field-width-missmatch"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">=" : "<="
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column number"
    rule_type = "field-width-missmatch"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


Translators/GoogleTranslate: ignore JSON null, fix #171

####################################################################
# GoogleTranslate.awk                                              #
####################################################################
#
# Last Updated: 18 Dec 2015
# https://translate.google.com/translate/releases/twsfe_w_20151214_RC03/r/js/desktop_module_main.js
BEGIN { provides("google") }

function genRL(a, x,
               ####
               b, c, d, i, y) {
    tokenize(y, x)
    parseList(b, y)
    i = SUBSEP 0
    for (c = 0; c < length(b[i]) - 2; c += 3) {
        d = b[i][c + 2]
        d = d >= 97 ? d - 87 :
            d - 48 # convert to number
        d = b[i][c + 1] == 43 ? rshift(a, d) : lshift(a, d)
        a = b[i][c] == 43 ? and(a + d, 4294967295) : xor(a, d)
    }
    return a
}

function genTK(text,
               ####
               a, d, dLen, e, tkk, ub, vb) {
    if (TK[text]) return TK[text]

    tkk = systime() / 3600
    ub = "[43,45,51,94,43,98,43,45,102]"
    vb = "[43,45,97,94,43,54]"

    # FIXME: build a dump cache!
    dLen = dump(text, d) # convert to byte array
    a = tkk
    for (e = 1; e <= dLen; e++)
        a = genRL(a + d[e], vb)
    a = genRL(a, ub)
    0 > a && (a = and(a, 2147483647) + 2147483648)
    a %= 1e6
    TK[text] = a "." xor(a, tkk)

    l(text, "text")
    l(tkk, "tkk")
    l(TK[text], "tk")
    return TK[text]
}

function googleInit() {
    HttpProtocol = "http://"
    HttpHost = "translate.googleapis.com"
    HttpPort = 80
}

function googleRequestUrl(text, sl, tl, hl,    qc) {
    qc = Option["no-autocorrect"] ? "qc" : "qca";
    return HttpPathPrefix "/translate_a/single?client=gtx"              \
        "&ie=UTF-8&oe=UTF-8"                                            \
        "&dt=bd&dt=ex&dt=ld&dt=md&dt=rw&dt=rm&dt=ss&dt=t&dt=at"         \
        "&dt=" qc "&sl=" sl "&tl=" tl "&hl=" hl                         \
        "&q=" preprocess(text)
}

function googleTTSUrl(text, tl) {
    return HttpProtocol HttpHost "/translate_tts?ie=UTF-8&client=gtx"	\
        "&tl=" tl "&q=" preprocess(text)
}

function googleWebTranslateUrl(uri, sl, tl, hl) {
    return "https://translate.google.com/translate?"    \
        "hl=" hl "&sl=" sl "&tl=" tl "&u=" uri
}

# Get the translation of a string.
function googleTranslate(text, sl, tl, hl,
                         isVerbose, toSpeech, returnPlaylist, returnIl,
                         ####
                         r,
                         content, tokens, ast,
                         _sl, _tl, _hl, il, ils, isPhonetic,
                         article, example, explanation, ref, word,
                         translation, translations, phonetics,
                         wordClasses, words, segments, altTranslations,
                         original, oPhonetics, oWordClasses, oWords,
                         oRefs, oSynonymClasses, oSynonyms,
                         oExamples, oSeeAlso,
                         wShowOriginal, wShowOriginalPhonetics,
                         wShowTranslation, wShowTranslationPhonetics,
                         wShowPromptMessage, wShowLanguages,
                         wShowOriginalDictionary, wShowDictionary,
                         wShowAlternatives,
                         hasWordClasses, hasAltTranslations,
                         i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }

    # Convert codes or aliases to standard codes used by Google Translate
    # If the code or alias cannot be found, use as it is
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl
    content = getResponse(text, _sl, _tl, _hl)
    tokenize(tokens, content)
    parseJsonArray(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in ast) {
        if (ast[i] == "null") continue

        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "0$")
            append(translations, postprocess(literal(ast[i])))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "1$")
            append(original, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "2$")
            append(phonetics, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "3$")
            append(oPhonetics, literal(ast[i]))

        # 1 - word classes and explanations
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            wordClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]][group[3]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]]["1"][group[3]] = literal(ast[i])

        # 5 - alternative translations
        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group)) {
            segments[group[1]] = literal(ast[i])
            altTranslations[group[1]][0] = ""
        }
        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            altTranslations[group[1]][group[2]] = postprocess(literal(ast[i]))

        # 7 - autocorrection
        if (i ~ "^0" SUBSEP "7" SUBSEP "5$") {
            if (ast[i] == "true")
                w("Showing translation for:  (use -no-auto to disable autocorrect)")
            else
                w("Did you mean: "                                      \
                  ansi("bold", unparameterize(ast["0" SUBSEP "7" SUBSEP "1"])))
        }

        # 8 - identified source languages
        if (i ~ "^0" SUBSEP "8" SUBSEP "0" SUBSEP "[[:digit:]]+$" ||
            i ~ "^0" SUBSEP "2$")
            append(ils, literal(ast[i]))

        # 11 - (original) word classes and synonyms
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oSynonymClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "1$", group))
            if (ast[i]) {
                oRefs[literal(ast[i])][1] = group[1]
                oRefs[literal(ast[i])][2] = group[2]
            }
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0" SUBSEP "([[:digit:]]+)$", group))
            oSynonyms[group[1]][group[2]][group[3]] = literal(ast[i])

        # 12 - (original) word classes and explanations
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oWordClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oWords[group[1]][group[2]][0] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "1$", group))
            oWords[group[1]][group[2]][1] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2$", group))
            oWords[group[1]][group[2]][2] = postprocess(literal(ast[i]))

        # 13 - (original) examples
        if (match(i, "^0" SUBSEP "13" SUBSEP "0" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oExamples[group[1]] = postprocess(literal(ast[i]))

        # 14 - (original) see also
        if (match(i, "^0" SUBSEP "14" SUBSEP "0" SUBSEP "([[:digit:]]+)$", group))
            oSeeAlso[group[1]] = literal(ast[i])
    }
    PROCINFO["sorted_in"] = saveSortedIn

    translation = join(translations)

    returnIl[0] = il = !anything(ils) || belongsTo(sl, ils) ? sl : ils[0]
    if (Option["verbose"] < 0)
        return getList(il)

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && anything(phonetics) ?
            prettify("brief-translation-phonetics", join(phonetics)) :
            prettify("brief-translation", s(translation, tl))

        if (toSpeech) {
            returnPlaylist[0]["text"] = translation
            returnPlaylist[0]["tl"] = tl
        }

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        wShowTranslation = Option["show-translation"]
        wShowTranslationPhonetics = Option["show-translation-phonetics"]
        wShowPromptMessage = Option["show-prompt-message"]
        wShowLanguages = Option["show-languages"]
        wShowOriginalDictionary = Option["show-original-dictionary"]
        wShowDictionary = Option["show-dictionary"]
        wShowAlternatives = Option["show-alternatives"]

        if (!anything(oPhonetics)) wShowOriginalPhonetics = 0
        if (!anything(phonetics)) wShowTranslationPhonetics = 0
        if (getCode(il) == getCode(tl) &&                               \
            (isarray(oWordClasses) || isarray(oSynonymClasses) ||       \
             isarray(oExamples) || isarray(oSeeAlso))) {
            wShowOriginalDictionary = 1
            wShowTranslation = 0
        }
        hasWordClasses = exists(wordClasses)
        hasAltTranslations = exists(altTranslations[0])
        if (!hasWordClasses && !hasAltTranslations)
            wShowPromptMessage = wShowLanguages = 0
        if (!hasWordClasses) wShowDictionary = 0
        if (hasWordClasses || !hasAltTranslations) wShowAlternatives = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text & phonetics")
            r = r prettify("original", s(join(original), il))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics), il))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation & phonetics")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics), tl))
        }

        if (wShowPromptMessage || wShowLanguages)
            if (r) r = r RS
        if (wShowPromptMessage) {
            if (hasWordClasses) {
                # Display: prompt message (Definitions of ...)
                if (r) r = r RS
                r = r m("-- display prompt message (Definitions of ...)")
                if (isRTL(hl)) # home language is R-to-L
                    r = r prettify("prompt-message", s(showDefinitionsOf(hl, join(original))))
                else { # home language is L-to-R
                    split(showDefinitionsOf(hl, "\0%s\0"), group, "\0")
                    for (i = 1; i <= length(group); i++) {
                        if (group[i] == "%s")
                            r = r prettify("prompt-message-original", show(join(original), il))
                        else
                            r = r prettify("prompt-message", group[i])
                    }
                }
            } else if (hasAltTranslations) {
                # Display: prompt message (Translations of ...)
                if (r) r = r RS
                r = r m("-- display prompt message (Translations of ...)")
                if (isRTL(hl)) # home language is R-to-L
                    r = r prettify("prompt-message", s(showTranslationsOf(hl, join(original))))
                else { # home language is L-to-R
                    split(showTranslationsOf(hl, "\0%s\0"), group, "\0")
                    for (i = 1; i <= length(group); i++) {
                        if (group[i] == "%s")
                            r = r prettify("prompt-message-original", show(join(original), il))
                        else
                            r = r prettify("prompt-message", group[i])
                    }
                }
            }
        }
        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowOriginalDictionary) {
            # Display: original dictionary
            if (exists(oWordClasses)) {
                # Detailed explanations
                if (r) r = r RS
                r = r m("-- display original dictionary (detailed explanations)")
                for (i = 0; i < length(oWordClasses); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("original-dictionary-detailed-word-class", s(oWordClasses[i], hl))
                    for (j = 0; j < length(oWords[i]); j++) {
                        explanation = oWords[i][j][0]
                        ref = oWords[i][j][1]
                        example = oWords[i][j][2]

                        r = (j > 0 ? r RS : r) RS prettify("original-dictionary-detailed-explanation", ins(1, explanation, il))
                        if (example)
                            r = r RS prettify("original-dictionary-detailed-example", ins(2, "- \"" example "\"", il))
                        if (ref && isarray(oRefs[ref])) {
                            temp = prettify("original-dictionary-detailed-synonyms", ins(1, show(showSynonyms(hl), hl) ": "))
                            temp = temp prettify("original-dictionary-detailed-synonyms-item", show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][0], il))
                            for (k = 1; k < length(oSynonyms[oRefs[ref][1]][oRefs[ref][2]]); k++)
                                temp = temp prettify("original-dictionary-detailed-synonyms", ", ") \
                                    prettify("original-dictionary-detailed-synonyms-item", show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][k], il))
                            r = r RS temp
                        }
                    }
                }
            }
            if (exists(oSynonymClasses)) {
                # Synonyms
                r = r RS RS
                r = r m("-- display original dictionary (synonyms)")
                r = r prettify("original-dictionary-synonyms", s(showSynonyms(hl), hl))
                for (i = 0; i < length(oSynonymClasses); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("original-dictionary-synonyms-word-class", ins(1, oSynonymClasses[i], hl))
                    for (j = 0; j < length(oSynonyms[i]); j++) {
                        temp = prettify("original-dictionary-synonyms-synonyms", ins(2, "- "))
                        temp = temp prettify("original-dictionary-synonyms-synonyms-item", show(oSynonyms[i][j][0], il))
                        for (k = 1; k < length(oSynonyms[i][j]); k++)
                            temp = temp prettify("original-dictionary-synonyms-synonyms", ", ") \
                                prettify("original-dictionary-synonyms-synonyms-item", show(oSynonyms[i][j][k], il))
                        r = r RS temp
                    }
                }
            }
            if (exists(oExamples)) {
                # Examples
                r = r RS RS
                r = r m("-- display original dictionary (examples)")
                r = r prettify("original-dictionary-examples", s(showExamples(hl), hl))
                for (i = 0; i < length(oExamples); i++) {
                    example = oExamples[i]

                    temp = prettify("original-dictionary-examples-example", ins(1, "- "))
                    split(example, group, /(<b>|<\/b>)/)
                    if (isRTL(il)) # target language is R-to-L
                        temp = temp show(group[1] group[2] group[3], il)
                    else # target language is L-to-R
                        temp = temp prettify("original-dictionary-examples-example", group[1]) \
                            prettify("original-dictionary-examples-original", group[2]) \
                            prettify("original-dictionary-examples-example", group[3])
                    r = (i > 0 ? r RS : r) RS temp
                }
            }
            if (exists(oSeeAlso)) {
                # See also
                r = r RS RS
                r = r m("-- display original dictionary (see also)")
                r = r prettify("original-dictionary-see-also", s(showSeeAlso(hl), hl))
                temp = ins(1, prettify("original-dictionary-see-also-phrases-item", show(oSeeAlso[0], il)))
                for (k = 1; k < length(oSeeAlso); k++)
                    temp = temp prettify("original-dictionary-see-also-phrases", ", ") \
                        prettify("original-dictionary-see-also-phrases-item", show(oSeeAlso[k], il))
                r = r RS temp
            }
        }

        if (wShowDictionary) {
            # Display: dictionary entries
            if (r) r = r RS
            r = r m("-- display dictionary entries")
            for (i = 0; i < length(wordClasses); i++) {
                r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))
                for (j = 0; j < length(words[i]); j++) {
                    word = words[i][j][0]
                    article = words[i][j][4]
                    if (isRTL(il))
                        explanation = join(words[i][j][1], ", ")
                    else {
                        explanation = prettify("dictionary-explanation-item", words[i][j][1][0])
                        for (k = 1; k < length(words[i][j][1]); k++)
                            explanation = explanation prettify("dictionary-explanation", ", ") \
                                prettify("dictionary-explanation-item", words[i][j][1][k])
                    }

                    r = r RS prettify("dictionary-word", ins(1, (article ? "(" article ") " : "") word, tl))
                    if (isRTL(il))
                        r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                    else
                        r = r RS ins(2, explanation)
                }
            }
        }

        if (wShowAlternatives) {
            # Display: alternative translations
            if (r) r = r RS RS
            r = r m("-- display alternative translations")
            for (i = 0; i < length(altTranslations); i++) {
                r = (i > 0 ? r RS : r) prettify("alternatives-original", show(segments[i], il))
                if (isRTL(tl)) {
                    temp = join(altTranslations[i], ", ")
                    r = r RS prettify("alternatives-translations-item", ins(1, temp, tl))
                } else {
                    temp = prettify("alternatives-translations-item", altTranslations[i][0])
                    for (j = 1; j < length(altTranslations[i]); j++)
                        temp = temp prettify("alternatives-translations", ", ") \
                            prettify("alternatives-translations-item", altTranslations[i][j])
                    r = r RS ins(1, temp)
                }
            }
        }

        if (toSpeech) {
            if (index(showTranslationsOf(hl, "%s"), "%s") > 2) {
                returnPlaylist[0]["text"] = showTranslationsOf(hl)
                returnPlaylist[0]["tl"] = hl
                returnPlaylist[1]["text"] = join(original)
                returnPlaylist[1]["tl"] = il
            } else {
                returnPlaylist[0]["text"] = join(original)
                returnPlaylist[0]["tl"] = il
                returnPlaylist[1]["text"] = showTranslationsOf(hl)
                returnPlaylist[1]["tl"] = hl
            }
            returnPlaylist[2]["text"] = translation
            returnPlaylist[2]["tl"] = tl
        }
    }

    return r
}

####################################################################
# GoogleTranslate.awk                                              #
####################################################################
#
# Last Updated: 18 Dec 2015
# https://translate.google.com/translate/releases/twsfe_w_20151214_RC03/r/js/desktop_module_main.js
BEGIN { provides("google") }

function genRL(a, x,
               ####
               b, c, d, i, y) {
    tokenize(y, x)
    parseList(b, y)
    i = SUBSEP 0
    for (c = 0; c < length(b[i]) - 2; c += 3) {
        d = b[i][c + 2]
        d = d >= 97 ? d - 87 :
            d - 48 # convert to number
        d = b[i][c + 1] == 43 ? rshift(a, d) : lshift(a, d)
        a = b[i][c] == 43 ? and(a + d, 4294967295) : xor(a, d)
    }
    return a
}

function genTK(text,
               ####
               a, d, dLen, e, tkk, ub, vb) {
    if (TK[text]) return TK[text]

    tkk = systime() / 3600
    ub = "[43,45,51,94,43,98,43,45,102]"
    vb = "[43,45,97,94,43,54]"

    # FIXME: build a dump cache!
    dLen = dump(text, d) # convert to byte array
    a = tkk
    for (e = 1; e <= dLen; e++)
        a = genRL(a + d[e], vb)
    a = genRL(a, ub)
    0 > a && (a = and(a, 2147483647) + 2147483648)
    a %= 1e6
    TK[text] = a "." xor(a, tkk)

    l(text, "text")
    l(tkk, "tkk")
    l(TK[text], "tk")
    return TK[text]
}

function googleInit() {
    HttpProtocol = "http://"
    HttpHost = "translate.googleapis.com"
    HttpPort = 80
}

function googleRequestUrl(text, sl, tl, hl,    qc) {
    qc = Option["no-autocorrect"] ? "qc" : "qca";
    return HttpPathPrefix "/translate_a/single?client=gtx"              \
        "&ie=UTF-8&oe=UTF-8"                                            \
        "&dt=bd&dt=ex&dt=ld&dt=md&dt=rw&dt=rm&dt=ss&dt=t&dt=at"         \
        "&dt=" qc "&sl=" sl "&tl=" tl "&hl=" hl                         \
        "&q=" preprocess(text)
}

function googleTTSUrl(text, tl) {
    return HttpProtocol HttpHost "/translate_tts?ie=UTF-8&client=gtx"	\
        "&tl=" tl "&q=" preprocess(text)
}

function googleWebTranslateUrl(uri, sl, tl, hl) {
    return "https://translate.google.com/translate?"    \
        "hl=" hl "&sl=" sl "&tl=" tl "&u=" uri
}

# Get the translation of a string.
function googleTranslate(text, sl, tl, hl,
                         isVerbose, toSpeech, returnPlaylist, returnIl,
                         ####
                         r,
                         content, tokens, ast,
                         _sl, _tl, _hl, il, ils, isPhonetic,
                         article, example, explanation, ref, word,
                         translation, translations, phonetics,
                         wordClasses, words, segments, altTranslations,
                         original, oPhonetics, oWordClasses, oWords,
                         oRefs, oSynonymClasses, oSynonyms,
                         oExamples, oSeeAlso,
                         wShowOriginal, wShowOriginalPhonetics,
                         wShowTranslation, wShowTranslationPhonetics,
                         wShowPromptMessage, wShowLanguages,
                         wShowOriginalDictionary, wShowDictionary,
                         wShowAlternatives,
                         hasWordClasses, hasAltTranslations,
                         i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }

    # Convert codes or aliases to standard codes used by Google Translate
    # If the code or alias cannot be found, use as it is
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl
    content = getResponse(text, _sl, _tl, _hl)
    tokenize(tokens, content)
    parseJsonArray(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in ast) {
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "0$")
            append(translations, postprocess(literal(ast[i])))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "1$")
            append(original, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "2$")
            append(phonetics, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "3$")
            append(oPhonetics, literal(ast[i]))

        # 1 - word classes and explanations
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            wordClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]][group[3]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]]["1"][group[3]] = literal(ast[i])

        # 5 - alternative translations
        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group)) {
            segments[group[1]] = literal(ast[i])
            altTranslations[group[1]][0] = ""
        }
        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            altTranslations[group[1]][group[2]] = postprocess(literal(ast[i]))

        # 7 - autocorrection
        if (i ~ "^0" SUBSEP "7" SUBSEP "5$") {
            if (ast[i] == "true")
                w("Showing translation for:  (use -no-auto to disable autocorrect)")
            else
                w("Did you mean: "                                      \
                  ansi("bold", unparameterize(ast["0" SUBSEP "7" SUBSEP "1"])))
        }

        # 8 - identified source languages
        if (i ~ "^0" SUBSEP "8" SUBSEP "0" SUBSEP "[[:digit:]]+$" ||
            i ~ "^0" SUBSEP "2$")
            append(ils, literal(ast[i]))

        # 11 - (original) word classes and synonyms
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oSynonymClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "1$", group))
            if (ast[i]) {
                oRefs[literal(ast[i])][1] = group[1]
                oRefs[literal(ast[i])][2] = group[2]
            }
        if (match(i, "^0" SUBSEP "11" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0" SUBSEP "([[:digit:]]+)$", group))
            oSynonyms[group[1]][group[2]][group[3]] = literal(ast[i])

        # 12 - (original) word classes and explanations
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oWordClasses[group[1]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oWords[group[1]][group[2]][0] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "1$", group))
            oWords[group[1]][group[2]][1] = literal(ast[i])
        if (match(i, "^0" SUBSEP "12" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2$", group))
            oWords[group[1]][group[2]][2] = postprocess(literal(ast[i]))

        # 13 - (original) examples
        if (match(i, "^0" SUBSEP "13" SUBSEP "0" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            oExamples[group[1]] = postprocess(literal(ast[i]))

        # 14 - (original) see also
        if (match(i, "^0" SUBSEP "14" SUBSEP "0" SUBSEP "([[:digit:]]+)$", group))
            oSeeAlso[group[1]] = literal(ast[i])
    }
    PROCINFO["sorted_in"] = saveSortedIn

    translation = join(translations)

    returnIl[0] = il = !anything(ils) || belongsTo(sl, ils) ? sl : ils[0]
    if (Option["verbose"] < 0)
        return getList(il)

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && anything(phonetics) ?
            prettify("brief-translation-phonetics", join(phonetics)) :
            prettify("brief-translation", s(translation, tl))

        if (toSpeech) {
            returnPlaylist[0]["text"] = translation
            returnPlaylist[0]["tl"] = tl
        }

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        wShowTranslation = Option["show-translation"]
        wShowTranslationPhonetics = Option["show-translation-phonetics"]
        wShowPromptMessage = Option["show-prompt-message"]
        wShowLanguages = Option["show-languages"]
        wShowOriginalDictionary = Option["show-original-dictionary"]
        wShowDictionary = Option["show-dictionary"]
        wShowAlternatives = Option["show-alternatives"]

        if (!anything(oPhonetics)) wShowOriginalPhonetics = 0
        if (!anything(phonetics)) wShowTranslationPhonetics = 0
        if (getCode(il) == getCode(tl) &&                               \
            (isarray(oWordClasses) || isarray(oSynonymClasses) ||       \
             isarray(oExamples) || isarray(oSeeAlso))) {
            wShowOriginalDictionary = 1
            wShowTranslation = 0
        }
        hasWordClasses = exists(wordClasses)
        hasAltTranslations = exists(altTranslations[0])
        if (!hasWordClasses && !hasAltTranslations)
            wShowPromptMessage = wShowLanguages = 0
        if (!hasWordClasses) wShowDictionary = 0
        if (hasWordClasses || !hasAltTranslations) wShowAlternatives = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text & phonetics")
            r = r prettify("original", s(join(original), il))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics), il))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation & phonetics")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics), tl))
        }

        if (wShowPromptMessage || wShowLanguages)
            if (r) r = r RS
        if (wShowPromptMessage) {
            if (hasWordClasses) {
                # Display: prompt message (Definitions of ...)
                if (r) r = r RS
                r = r m("-- display prompt message (Definitions of ...)")
                if (isRTL(hl)) # home language is R-to-L
                    r = r prettify("prompt-message", s(showDefinitionsOf(hl, join(original))))
                else { # home language is L-to-R
                    split(showDefinitionsOf(hl, "\0%s\0"), group, "\0")
                    for (i = 1; i <= length(group); i++) {
                        if (group[i] == "%s")
                            r = r prettify("prompt-message-original", show(join(original), il))
                        else
                            r = r prettify("prompt-message", group[i])
                    }
                }
            } else if (hasAltTranslations) {
                # Display: prompt message (Translations of ...)
                if (r) r = r RS
                r = r m("-- display prompt message (Translations of ...)")
                if (isRTL(hl)) # home language is R-to-L
                    r = r prettify("prompt-message", s(showTranslationsOf(hl, join(original))))
                else { # home language is L-to-R
                    split(showTranslationsOf(hl, "\0%s\0"), group, "\0")
                    for (i = 1; i <= length(group); i++) {
                        if (group[i] == "%s")
                            r = r prettify("prompt-message-original", show(join(original), il))
                        else
                            r = r prettify("prompt-message", group[i])
                    }
                }
            }
        }
        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowOriginalDictionary) {
            # Display: original dictionary
            if (exists(oWordClasses)) {
                # Detailed explanations
                if (r) r = r RS
                r = r m("-- display original dictionary (detailed explanations)")
                for (i = 0; i < length(oWordClasses); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("original-dictionary-detailed-word-class", s(oWordClasses[i], hl))
                    for (j = 0; j < length(oWords[i]); j++) {
                        explanation = oWords[i][j][0]
                        ref = oWords[i][j][1]
                        example = oWords[i][j][2]

                        r = (j > 0 ? r RS : r) RS prettify("original-dictionary-detailed-explanation", ins(1, explanation, il))
                        if (example)
                            r = r RS prettify("original-dictionary-detailed-example", ins(2, "- \"" example "\"", il))
                        if (ref && isarray(oRefs[ref])) {
                            temp = prettify("original-dictionary-detailed-synonyms", ins(1, show(showSynonyms(hl), hl) ": "))
                            temp = temp prettify("original-dictionary-detailed-synonyms-item", show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][0], il))
                            for (k = 1; k < length(oSynonyms[oRefs[ref][1]][oRefs[ref][2]]); k++)
                                temp = temp prettify("original-dictionary-detailed-synonyms", ", ") \
                                    prettify("original-dictionary-detailed-synonyms-item", show(oSynonyms[oRefs[ref][1]][oRefs[ref][2]][k], il))
                            r = r RS temp
                        }
                    }
                }
            }
            if (exists(oSynonymClasses)) {
                # Synonyms
                r = r RS RS
                r = r m("-- display original dictionary (synonyms)")
                r = r prettify("original-dictionary-synonyms", s(showSynonyms(hl), hl))
                for (i = 0; i < length(oSynonymClasses); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("original-dictionary-synonyms-word-class", ins(1, oSynonymClasses[i], hl))
                    for (j = 0; j < length(oSynonyms[i]); j++) {
                        temp = prettify("original-dictionary-synonyms-synonyms", ins(2, "- "))
                        temp = temp prettify("original-dictionary-synonyms-synonyms-item", show(oSynonyms[i][j][0], il))
                        for (k = 1; k < length(oSynonyms[i][j]); k++)
                            temp = temp prettify("original-dictionary-synonyms-synonyms", ", ") \
                                prettify("original-dictionary-synonyms-synonyms-item", show(oSynonyms[i][j][k], il))
                        r = r RS temp
                    }
                }
            }
            if (exists(oExamples)) {
                # Examples
                r = r RS RS
                r = r m("-- display original dictionary (examples)")
                r = r prettify("original-dictionary-examples", s(showExamples(hl), hl))
                for (i = 0; i < length(oExamples); i++) {
                    example = oExamples[i]

                    temp = prettify("original-dictionary-examples-example", ins(1, "- "))
                    split(example, group, /(<b>|<\/b>)/)
                    if (isRTL(il)) # target language is R-to-L
                        temp = temp show(group[1] group[2] group[3], il)
                    else # target language is L-to-R
                        temp = temp prettify("original-dictionary-examples-example", group[1]) \
                            prettify("original-dictionary-examples-original", group[2]) \
                            prettify("original-dictionary-examples-example", group[3])
                    r = (i > 0 ? r RS : r) RS temp
                }
            }
            if (exists(oSeeAlso)) {
                # See also
                r = r RS RS
                r = r m("-- display original dictionary (see also)")
                r = r prettify("original-dictionary-see-also", s(showSeeAlso(hl), hl))
                temp = ins(1, prettify("original-dictionary-see-also-phrases-item", show(oSeeAlso[0], il)))
                for (k = 1; k < length(oSeeAlso); k++)
                    temp = temp prettify("original-dictionary-see-also-phrases", ", ") \
                        prettify("original-dictionary-see-also-phrases-item", show(oSeeAlso[k], il))
                r = r RS temp
            }
        }

        if (wShowDictionary) {
            # Display: dictionary entries
            if (r) r = r RS
            r = r m("-- display dictionary entries")
            for (i = 0; i < length(wordClasses); i++) {
                r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))
                for (j = 0; j < length(words[i]); j++) {
                    word = words[i][j][0]
                    article = words[i][j][4]
                    if (isRTL(il))
                        explanation = join(words[i][j][1], ", ")
                    else {
                        explanation = prettify("dictionary-explanation-item", words[i][j][1][0])
                        for (k = 1; k < length(words[i][j][1]); k++)
                            explanation = explanation prettify("dictionary-explanation", ", ") \
                                prettify("dictionary-explanation-item", words[i][j][1][k])
                    }

                    r = r RS prettify("dictionary-word", ins(1, (article ? "(" article ") " : "") word, tl))
                    if (isRTL(il))
                        r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                    else
                        r = r RS ins(2, explanation)
                }
            }
        }

        if (wShowAlternatives) {
            # Display: alternative translations
            if (r) r = r RS RS
            r = r m("-- display alternative translations")
            for (i = 0; i < length(altTranslations); i++) {
                r = (i > 0 ? r RS : r) prettify("alternatives-original", show(segments[i], il))
                if (isRTL(tl)) {
                    temp = join(altTranslations[i], ", ")
                    r = r RS prettify("alternatives-translations-item", ins(1, temp, tl))
                } else {
                    temp = prettify("alternatives-translations-item", altTranslations[i][0])
                    for (j = 1; j < length(altTranslations[i]); j++)
                        temp = temp prettify("alternatives-translations", ", ") \
                            prettify("alternatives-translations-item", altTranslations[i][j])
                    r = r RS ins(1, temp)
                }
            }
        }

        if (toSpeech) {
            if (index(showTranslationsOf(hl, "%s"), "%s") > 2) {
                returnPlaylist[0]["text"] = showTranslationsOf(hl)
                returnPlaylist[0]["tl"] = hl
                returnPlaylist[1]["text"] = join(original)
                returnPlaylist[1]["tl"] = il
            } else {
                returnPlaylist[0]["text"] = join(original)
                returnPlaylist[0]["tl"] = il
                returnPlaylist[1]["text"] = showTranslationsOf(hl)
                returnPlaylist[1]["tl"] = hl
            }
            returnPlaylist[2]["text"] = translation
            returnPlaylist[2]["tl"] = tl
        }
    }

    return r
}


Initialization in BEGIN block.

Initialization in BEGIN block.

function printUrlQueryString(queryString)
{
    split(queryString, pairs, "&", seps)
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=", seps2)
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\"%s\": \"%s\",\n", k, urlDecode(urlDecode(v))
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

BEGIN {
    logLineSeen = 0
    print "{";
}

END {
    print "}"
}

/^#Version: / {
    printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
}

/^#Fields: / {
    rest = substr($0, length("#Fields: ") + 1)
    split(rest, fields, " ")
    numFields = length(fields)
}

/^#/ {
    # ignore anything else starting with "#"
}

! /^#/ {
    if (logLineSeen == 1) {
        printf ",\n"
    }
    split($0, fieldsThisLine, " ")
    if (NF == numFields) {
        print "\t{"
        for(i = 1; i < NF; i++)
        {
            printf "\t\t\"%s\": \"%s\"", fields[i], fieldsThisLine[i]
            if (i + 1 < NF) {
                printf ","
            }
            printf "\n"
        }
        printf "\t}"
    }
    logLineSeen = 1;
}

function printUrlQueryString(queryString)
{
    split(queryString, pairs, "&", seps)
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=", seps2)
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\"%s\": \"%s\",\n", k, urlDecode(urlDecode(v))
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

BEGIN {
    print "{";
}

END {
    print "}"
}

/^#Version: / {
    printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
    logLineSeen = 0
}

/^#Fields: / {
    rest = substr($0, length("#Fields: ") + 1)
    split(rest, fields, " ")
    numFields = length(fields)
}

/^#/ {
    # ignore anything else starting with "#"
}

! /^#/ {
    if (logLineSeen == 1) {
        printf ",\n"
    }
    split($0, fieldsThisLine, " ")
    if (NF == numFields) {
        print "\t{"
        for(i = 1; i < NF; i++)
        {
            printf "\t\t\"%s\": \"%s\"", fields[i], fieldsThisLine[i]
            if (i + 1 < NF) {
                printf ","
            }
            printf "\n"
        }
        printf "\t}"
    }
    logLineSeen = 1;
}


Another tentative waypoint before any sensible commits begin. This introduces splines through the intersection points of the graphviz edge and node circles.


print full inheritance

print full inheritance

Change-Id: Ia53fcbde6b3ff4d38b477069a1f956c9aef84f78

#!/usr/bin/gawk -f
# -*- Mode: awk; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
#
# Copyright 2012 LibreOffice contributors.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Usage: gawk -f list-dateacceptancepattern.awk *.xml [--html]
#
# Outputs three lists of locales, one with DateAcceptancePattern elements
# defined, one with inherited LC_FORMAT elements and thus date patterns, and
# one where no DateAcceptancePattern are defined.
#
# If --html is given as the last parameter, format output suitable for
# inclusion in HTML.

BEGIN {
    html = 0
    if (ARGV[ARGC-1] == "--html") {
        html = 1
        --ARGC
    }
    file = ""
    offlocale = 0
    offpatterns = 1
    offinherit = 2
    inheritedcount = 0
    if (html)
        print "
"
    else
        print ""
    print "Locales with explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
}

file != FILENAME {
    if (file)
        endFile()
    file = FILENAME
    patterns = 0
    inherited = ""
}

/<DateAcceptancePattern>/ {
    split( $0, a, /<|>/ )
    pattern[patterns++] = a[3]
}

# pattern inherited as well
/<LC_FORMAT[^>]* ref="[^>"]+"[^>]*>/ {
    split( $0, a, /.* ref="|"/ )
    inherited = a[2]
}

END {
    if (file)
        endFile()

    PROCINFO["sorted_in"] = "@ind_str_asc"

    fillAllInherited()

    if (html)
    {
        print "
"
        print "\n
"
    }
    else
        print "\n"

    print "Locales inheriting patterns:"
    if (html)
    {
        print "
"
        for (i in LocaleList)
        {
            if (LocaleList[i][offinherit] && LocaleList[i][offpatterns])
                print getInheritance( LocaleList[i][offlocale], LocaleList[i][offlocale]) "    "
        }
        print "\n
"
    }
    else
    {
        for (i in LocaleList)
        {
            if (LocaleList[i][offinherit] && LocaleList[i][offpatterns])
                print getInheritance( LocaleList[i][offlocale], LocaleList[i][offlocale])
        }
        print "\n"
    }

    print "Locales without explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
    print "(one implicit full date pattern is always generated)"
    if (html)
        print "
"
    if (html)
    {
        for (i in LocaleList)
        {
            if (!LocaleList[i][offpatterns])
                print getInheritance( LocaleList[i][offlocale], LocaleList[i][offlocale]) "    "
        }
    }
    else
    {
        for (i in LocaleList)
        {
            if (!LocaleList[i][offpatterns])
                print getInheritance( LocaleList[i][offlocale], LocaleList[i][offlocale])
        }
    }
}

function endFile(       locale ) {
    locale =  getLocale( file)
    LocaleList[locale][offlocale] = locale
    LocaleList[locale][offpatterns] = patterns
    if (patterns)
    {
        if (html)
        {
            print "
 " locale ":"
            print "
"
            for ( i=0; i<patterns; ++i )
            {
                print "
 " pattern[i]
            }
            print "
"
        }
        else
        {
            print locale ":"
            for ( i=0; i<patterns; ++i )
            {
                print "    " pattern[i]
            }
        }
    }
    else if (inherited)
        LocaleList[locale][offinherit] = inherited
}

function getLocale( file,       tmp ) {
    tmp = file
    gsub( /.*\//, "", tmp )
    gsub( /\.xml/, "", tmp )
    return tmp
}

function fillInherited( locale ) {
    if (!LocaleList[locale][offpatterns] && LocaleList[locale][offinherit])
        LocaleList[locale][offpatterns] = fillInherited( LocaleList[locale][offinherit])
    return LocaleList[locale][offpatterns]
}

function fillAllInherited(      i ) {
    for (i in LocaleList)
    {
        if (!LocaleList[i][offpatterns] && LocaleList[i][offinherit])
            LocaleList[i][offpatterns] = fillInherited( LocaleList[i][offinherit])
    }
}

function getInheritance( str, locale ) {
    if (LocaleList[locale][offinherit])
        str = getInheritance( str " = " LocaleList[locale][offinherit], LocaleList[locale][offinherit])
    return str
}

# vim:set shiftwidth=4 softtabstop=4 expandtab:

#!/usr/bin/gawk -f
# -*- Mode: awk; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
#
# Copyright 2012 LibreOffice contributors.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Usage: gawk -f list-dateacceptancepattern.awk *.xml [--html]
#
# Outputs three lists of locales, one with DateAcceptancePattern elements
# defined, one with inherited LC_FORMAT elements and thus date patterns, and
# one where no DateAcceptancePattern are defined.
#
# If --html is given as the last parameter, format output suitable for
# inclusion in HTML.

BEGIN {
    html = 0
    if (ARGV[ARGC-1] == "--html") {
        html = 1
        --ARGC
    }
    file = ""
    offlocale = 0
    offpatterns = 1
    offinherit = 2
    inheritedcount = 0
    if (html)
        print "
"
    else
        print ""
    print "Locales with explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
}

file != FILENAME {
    if (file)
        endFile()
    file = FILENAME
    patterns = 0
    inherited = ""
}

/<DateAcceptancePattern>/ {
    split( $0, a, /<|>/ )
    pattern[patterns++] = a[3]
}

# pattern inherited as well
/<LC_FORMAT[^>]* ref="[^>"]+"[^>]*>/ {
    split( $0, a, /.* ref="|"/ )
    inherited = a[2]
}

END {
    if (file)
        endFile()

    PROCINFO["sorted_in"] = "@ind_str_asc"

    fillAllInherited()

    if (html)
    {
        print "
"
        print "\n
"
    }
    else
        print "\n"

    print "Locales inheriting patterns:"
    if (html)
    {
        print "
"
        for (i in LocaleList)
        {
            if (LocaleList[i][offinherit] && LocaleList[i][offpatterns])
                print LocaleList[i][offlocale] " = " LocaleList[i][offinherit] "    "
        }
        print "\n
"
    }
    else
    {
        for (i in LocaleList)
        {
            if (LocaleList[i][offinherit] && LocaleList[i][offpatterns])
                print LocaleList[i][offlocale] " = " LocaleList[i][offinherit]
        }
        print "\n"
    }

    print "Locales without explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
    print "(one implicit full date pattern is always generated)"
    if (html)
        print "
"
    if (html)
    {
        for (i in LocaleList)
        {
            if (!LocaleList[i][offpatterns])
                print LocaleList[i][offlocale] "    "
        }
    }
    else
    {
        for (i in LocaleList)
        {
            if (!LocaleList[i][offpatterns])
                print LocaleList[i][offlocale]
        }
    }
}

function endFile() {
    locale =  getLocale( file)
    LocaleList[locale][offlocale] = locale
    LocaleList[locale][offpatterns] = patterns
    if (patterns)
    {
        if (html)
        {
            print "
 " locale ":"
            print "
"
            for ( i=0; i<patterns; ++i )
            {
                print "
 " pattern[i]
            }
            print "
"
        }
        else
        {
            print locale ":"
            for ( i=0; i<patterns; ++i )
            {
                print "    " pattern[i]
            }
        }
    }
    else if (inherited)
        LocaleList[locale][offinherit] = inherited
}

function getLocale( file,       tmp ) {
    tmp = file
    gsub( /.*\//, "", tmp )
    gsub( /\.xml/, "", tmp )
    return tmp
}

function fillInherited( locale ) {
    if (!LocaleList[locale][offpatterns] && LocaleList[locale][offinherit])
        LocaleList[locale][offpatterns] = fillInherited( LocaleList[locale][offinherit])
    return LocaleList[locale][offpatterns]
}

function fillAllInherited(      i ) {
    for (i in LocaleList)
    {
        if (!LocaleList[i][offpatterns] && LocaleList[i][offinherit])
            LocaleList[i][offpatterns] = fillInherited( LocaleList[i][offinherit])
    }
}

# vim:set shiftwidth=4 softtabstop=4 expandtab:


add subnet statistics

add subnet statistics

BEGIN {
    num_hosts = 0;
    num_subnets = 0;
    num_users = 0;
}
{
    if ($3 != "RECV" && $3 != "SENT") {
	if (num_hosts == 0) {
	    StartDate = $1;
	    StartTime = $2;
	} else {
	    EndDate = $1;
	    EndTime = $2;
	}
	if (AttemptedSends[$9] == 0 && AttemptedRecvs[$9] == 0) {
	    Hosts[num_hosts] = $9;
	    num_hosts++;
	}
	split($9, ip_tmp, ".");
	Subnet = sprintf("%s.%s.%s", ip_tmp[1], ip_tmp[2], ip_tmp[3]);
	if (AttemptedSends[Subnet] == 0 && AttemptedRecvs[Subnet] == 0) {
	    Subnets[num_subnets] = Subnet;
	    num_subnets++;
	}
	split($10, user_tmp, "\@");
	User = user_tmp[1];
	if (AttemptedSends[User] == 0 && AttemptedRecvs[User] == 0) {
	    Users[num_users] = User;
	    num_users++;
	}
	if ($3 == "S") {
	    AttemptedSends[$9]++;
	    AttemptedSends[Subnet]++;
	    AttemptedSends[User]++;
	    if ($4 != "F") {
		CompletedSends[$9]++;
		CompletedSends[Subnet]++;
		CompletedSends[User]++;
		BytesSent[$9] += $5;
		BytesSent[Subnet] += $5;
		BytesSent[User] += $5;
		TimeSending[$9] += $7;
		TimeSending[Subnet] += $7;
		TimeSending[User] += $7;
	    }
	} else {
	    AttemptedRecvs[$9]++;
	    AttemptedRecvs[Subnet]++;
	    AttemptedRecvs[User]++;
	    if ($4 != "F") {
		CompletedRecvs[$9]++;
		CompletedRecvs[Subnet]++;
		CompletedRecvs[User]++;
		BytesRecvd[$9] += $5;
		BytesRecvd[Subnet] += $5;
		BytesRecvd[User] += $5;
		TimeReceiving[$9] += $7;
		TimeReceiving[Subnet] += $7;
		TimeReceiving[User] += $7;
	    }
	}
    }
}
END {
    qsort(Hosts, 0, num_hosts-1);
    qsort(Subnets, 0, num_subnets-1);
    qsort(Users, 0, num_users-1);
    printf("\nCheckpoint Server Transfer Log Data from %s %s to %s %s\n\n",
	   StartDate, StartTime, EndDate, EndTime);
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Send", "Send", "Receive", "Receive", "Total", "Total");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Bandwidth", "Success", "Bandwidth", "Success", "Bandwidth",
	   "Success");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n\n",
	   "User/Host", "(Mb/s)", "Rate", "(Mb/s)", "Rate", "(Mb/s)", "Rate");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	Bytes = BytesSent[User] + BytesRecvd[User];
	Time = TimeSending[User] + TimeReceiving[User];
	Attempted = AttemptedSends[User] + AttemptedRecvs[User];
	Completed = CompletedSends[User] + CompletedRecvs[User];
	if (TimeSending[User] > 0 && TimeReceiving[User] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   User,
		   BytesSent[User]*8/TimeSending[User]/1024000,
		   CompletedSends[User]/AttemptedSends[User]*100,
		   BytesRecvd[User]*8/TimeReceiving[User]/1024000,
		   CompletedRecvs[User]/AttemptedRecvs[User]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
    }
    printf("\n");
    for (i=0; i < num_hosts; i++) {
	Host = Hosts[i];
	Bytes = BytesSent[Host] + BytesRecvd[Host];
	Time = TimeSending[Host] + TimeReceiving[Host];
	Attempted = AttemptedSends[Host] + AttemptedRecvs[Host];
	Completed = CompletedSends[Host] + CompletedRecvs[Host];
	if (TimeSending[Host] > 0 && TimeReceiving[Host] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   Host,
		   BytesSent[Host]*8/TimeSending[Host]/1024000,
		   CompletedSends[Host]/AttemptedSends[Host]*100,
		   BytesRecvd[Host]*8/TimeReceiving[Host]/1024000,
		   CompletedRecvs[Host]/AttemptedRecvs[Host]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
	TotalBytesSent += BytesSent[Host];
	TotalBytesRecvd += BytesRecvd[Host];
	TotalTimeSending += TimeSending[Host];
	TotalTimeReceiving += TimeReceiving[Host];
	TotalAttemptedSends += AttemptedSends[Host];
	TotalAttemptedRecvs += AttemptedRecvs[Host];
	TotalCompletedSends += CompletedSends[Host];
	TotalCompletedRecvs += CompletedRecvs[Host];
    }
    printf("\n");
    for (i=0; i < num_subnets; i++) {
	Subnet = Subnets[i];
	Bytes = BytesSent[Subnet] + BytesRecvd[Subnet];
	Time = TimeSending[Subnet] + TimeReceiving[Subnet];
	Attempted = AttemptedSends[Subnet] + AttemptedRecvs[Subnet];
	Completed = CompletedSends[Subnet] + CompletedRecvs[Subnet];
	if (TimeSending[Subnet] > 0 && TimeReceiving[Subnet] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   Subnet,
		   BytesSent[Subnet]*8/TimeSending[Subnet]/1024000,
		   CompletedSends[Subnet]/AttemptedSends[Subnet]*100,
		   BytesRecvd[Subnet]*8/TimeReceiving[Subnet]/1024000,
		   CompletedRecvs[Subnet]/AttemptedRecvs[Subnet]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
    }
    TotalBytes = TotalBytesSent + TotalBytesRecvd;
    TotalTime = TotalTimeSending + TotalTimeReceiving;
    TotalAttempted = TotalAttemptedSends + TotalAttemptedRecvs;
    TotalCompleted = TotalCompletedSends + TotalCompletedRecvs;
    if (TotalTimeSending > 0 && TotalTimeReceiving > 0) {
	printf("\n%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n\n",
	       "Overall",
	       TotalBytesSent*8/TotalTimeSending/1024000,
	       TotalCompletedSends/TotalAttemptedSends*100,
	       TotalBytesRecvd*8/TotalTimeReceiving/1024000,
	       TotalCompletedRecvs/TotalAttemptedRecvs*100,
	       TotalBytes*8/TotalTime/1024000,
	       TotalCompleted/TotalAttempted*100);
    }
    printf("\n%-16.16s %25.25s %25.25s\n\n", "User", "MB Sent", "MB Received");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	printf("%-16.16s %25.0f %25.0f\n",
	       User, BytesSent[User]/1024000, BytesRecvd[User]/1024000);
    }
    printf("\n%-16.16s %25.0f %25.0f\n",
	   "Total", TotalBytesSent/1024000, TotalBytesRecvd/1024000);
}
function qsort(v, left, right) {
  if (left >= right) return;
  swap(v, left, int((left + right)/2));
  last = left;
  for (i=left+1; i <= right; i++) {
    if (v[i] < v[left]) swap(v, ++last, i);
  }
  swap(v, left, last);
  qsort(v, left, last-1);
  qsort(v, last+1, right);
}
function swap(v, i, j) {
  temp = v[i];
  v[i] = v[j];
  v[j] = temp;
}

BEGIN {
    num_hosts = 0;
    num_users = 0;
}
{
    if ($3 != "RECV" && $3 != "SENT") {
	if (num_hosts == 0) {
	    StartDate = $1;
	    StartTime = $2;
	} else {
	    EndDate = $1;
	    EndTime = $2;
	}
	if (AttemptedSends[$9] == 0 && AttemptedRecvs[$9] == 0) {
	    Hosts[num_hosts] = $9;
	    num_hosts++;
	}
	split($10, user_tmp, "\@");
	User = user_tmp[1];
	if (AttemptedSends[User] == 0 && AttemptedRecvs[User] == 0) {
	    Users[num_users] = User;
	    num_users++;
	}
	if ($3 == "S") {
	    AttemptedSends[$9]++;
	    AttemptedSends[User]++;
	    if ($4 != "F") {
		CompletedSends[$9]++;
		CompletedSends[User]++;
		BytesSent[$9] += $5;
		BytesSent[User] += $5;
		TimeSending[$9] += $7;
		TimeSending[User] += $7;
	    }
	} else {
	    AttemptedRecvs[$9]++;
	    AttemptedRecvs[User]++;
	    if ($4 != "F") {
		CompletedRecvs[$9]++;
		CompletedRecvs[User]++;
		BytesRecvd[$9] += $5;
		BytesRecvd[User] += $5;
		TimeReceiving[$9] += $7;
		TimeReceiving[User] += $7;
	    }
	}
    }
}
END {
    printf("\nCheckpoint Server Transfer Log Data from %s %s to %s %s\n\n",
	   StartDate, StartTime, EndDate, EndTime);
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Send", "Send", "Receive", "Receive", "Total", "Total");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Bandwidth", "Success", "Bandwidth", "Success", "Bandwidth",
	   "Success");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n\n",
	   "User/Host", "(Mb/s)", "Rate", "(Mb/s)", "Rate", "(Mb/s)", "Rate");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	Bytes = BytesSent[User] + BytesRecvd[User];
	Time = TimeSending[User] + TimeReceiving[User];
	Attempted = AttemptedSends[User] + AttemptedRecvs[User];
	Completed = CompletedSends[User] + CompletedRecvs[User];
	if (TimeSending[User] > 0 && TimeReceiving[User] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   User,
		   BytesSent[User]*8/TimeSending[User]/1024000,
		   CompletedSends[User]/AttemptedSends[User]*100,
		   BytesRecvd[User]*8/TimeReceiving[User]/1024000,
		   CompletedRecvs[User]/AttemptedRecvs[User]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
    }
    printf("\n");
    for (i=0; i < num_hosts; i++) {
	Host = Hosts[i];
	Bytes = BytesSent[Host] + BytesRecvd[Host];
	Time = TimeSending[Host] + TimeReceiving[Host];
	Attempted = AttemptedSends[Host] + AttemptedRecvs[Host];
	Completed = CompletedSends[Host] + CompletedRecvs[Host];
	if (TimeSending[Host] > 0 && TimeReceiving[Host] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   Host,
		   BytesSent[Host]*8/TimeSending[Host]/1024000,
		   CompletedSends[Host]/AttemptedSends[Host]*100,
		   BytesRecvd[Host]*8/TimeReceiving[Host]/1024000,
		   CompletedRecvs[Host]/AttemptedRecvs[Host]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
	TotalBytesSent += BytesSent[Host];
	TotalBytesRecvd += BytesRecvd[Host];
	TotalTimeSending += TimeSending[Host];
	TotalTimeReceiving += TimeReceiving[Host];
	TotalAttemptedSends += AttemptedSends[Host];
	TotalAttemptedRecvs += AttemptedRecvs[Host];
	TotalCompletedSends += CompletedSends[Host];
	TotalCompletedRecvs += CompletedRecvs[Host];
    }
    TotalBytes = TotalBytesSent + TotalBytesRecvd;
    TotalTime = TotalTimeSending + TotalTimeReceiving;
    TotalAttempted = TotalAttemptedSends + TotalAttemptedRecvs;
    TotalCompleted = TotalCompletedSends + TotalCompletedRecvs;
    if (TotalTimeSending > 0 && TotalTimeReceiving > 0) {
	printf("\n%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n\n",
	       "Overall",
	       TotalBytesSent*8/TotalTimeSending/1024000,
	       TotalCompletedSends/TotalAttemptedSends*100,
	       TotalBytesRecvd*8/TotalTimeReceiving/1024000,
	       TotalCompletedRecvs/TotalAttemptedRecvs*100,
	       TotalBytes*8/TotalTime/1024000,
	       TotalCompleted/TotalAttempted*100);
    }
    printf("\n%-16.16s %25.25s %25.25s\n\n", "User", "MB Sent", "MB Received");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	printf("%-16.16s %25.0f %25.0f\n",
	       User, BytesSent[User]/1024000, BytesRecvd[User]/1024000);
    }
    printf("\n%-16.16s %25.0f %25.0f\n",
	   "Total", TotalBytesSent/1024000, TotalBytesRecvd/1024000);
}


r11552/sofa-dev : missing double-slashes in filter-qmake-pro.awk

Former-commit-id: 1325c1b6b009cc1e080200429d891be5c8aaa556

#!/usr/bin/awk -f
# This script removes parts flagged by a given FILTER_TAG inside qmake project files.
# The file with the relevant parts removed is output to the standard output,
# while files mentionned inside removed parts are output to the standard error.
# This can be used to remove then from the repository by svn rm'ing them.

# If you never used awk, a good introduction is available at http://www.cs.hmc.edu/qref/awk.html

# init
BEGIN {
    infilelist=0;
    blk_before=0;
    blk_after=0;
    blk_filtered=0;
    if (plugins == "")
        plugins = "applications/plugins";
    if (FILTER_TAG == "")
        FILTER_TAG = "SOFA_RELEASE";
}

{ blk_before=blk_after; }

/[{}]/ {
  split($0,a,"#")
  if (a[1]!="") {
    l = a[1];
    beg[1]=""; nbeg = split(l,beg,"{")
    end[1]=""; nend = split(l,end,"}")
    blk_after = blk_before + nbeg - nend
  }
}

END {
    if (blk_after != 0) print "# ERROR: unmatched brackets"
#    print "# Filtered with " FILTER_TAG
}

# Match blocks conditionnally included depending on the SOFA_DEV flag
# Note that we now count the brackets in the source file to find the
# SOFA_DEV closing one instead of relying on the presence of a comment

blk_filtered==0 && $0 ~ "contains[ \t]*\\([ \t]*DEFINES[ \t]*,[ \t]*"FILTER_TAG"[ \t]*\\)[ \t]*{" {
    blk_filtered=blk_after;
    infilelist=0;
}

blk_filtered>0 && blk_after < blk_filtered {
    blk_filtered=0;
    next;
}

/^[ \t#]*declare/ {
  name=$0;
  gsub(/.*\(/,"",name); gsub(/,.*$/,"",name); gsub(/ /,"",name); gsub(/\t/,"",name);
  path=$0;
  gsub(/.*\([^,]*,/,"",path); gsub(/,.*$/,"",path); gsub(/\).*$/,"",path); gsub(/ /,"",path); gsub(/\t/,"",path);
#  gsub(/\).*$/,"",path); gsub(/,$/,"",path); gsub(/ /,"",path);
  #print "#PROJECT <<<",name,"|",path,">>>";
  projects[name] = path;
}
blk_filtered>0 && /^[ \t#]*enable/ {
  name=$0;
  gsub(/.*\(/,"",name); gsub(/,.*$/,"",name); gsub(/\).*$/,"",name); gsub(/ /,"",name); gsub(/\t/,"",name);
  path = projects[name];
  #print "#DEV PROJECT <<<",name,"|",path,">>>";
  print path > "/dev/stderr";
  next;
}
blk_filtered>0 && /^[ \t#]*usePlugin/ {
  name=$0;
  gsub(/.*\(/,"",name); gsub(/,.*$/,"",name); gsub(/\).*$/,"",name); gsub(/ /,"",name); gsub(/\t/,"",name);
  #print "#DEV PLUGIN <<<",name,">>>";
  print plugins "/" name > "/dev/stderr";
  next;
}

blk_filtered>0 {
    #print "# " $0
    # look for a filename
    # we assume files and directories are listed inside variables containing SUBDIRS, SOURCES or HEADERS
    f0=1;
    nf=NF;
    if ($nf~/\r/) {
      #print "#CR <" $nf ">"
      gsub(/[\r\n]/,"",$nf);
      #print "#CRF <" $nf ">"
      if ($nf=="") nf--;
    }
    if (infilelist==0) {
	if (($1~"SUBDIRS" || $1~"SOURCES" || $1~"HEADERS") && ($2=="=" || $2=="+=")) {
	    #print "#infilelist " $1
	    infilelist=1;
	    f0=3; # first file is in 3rd field
	}
	else if ($1~"###") {
	    #print "#infilelist " $1
	    infilelist=1;
	    f0=2; # first file is in 3rd field
	}
    }
    if (infilelist) {
	for(f=f0;f<=nf && !(f==nf && $f~/^[:space:]*\\[:space:]*$/);f++) {
	    fname=$f
	    gsub(/[\r\n]/,"",fname);
	    gsub(/\\[:space:]*$/,"",fname);
	    gsub(/^##*/,"",fname);
	    if (fname != "") print fname > "/dev/stderr";
	}
	if (f>nf && (nf==0 || $nf!~/\\[:space:]*$/)) { # no "\\" is put at the end of the line -> end of list
	    #print "#end"
	    infilelist=0;
	}
    }
    next;
}

# Match lines mentionning FILTER_TAG, such as the point where it is defined
$0 ~ FILTER_TAG { next; }

# other: simply print the line
#{ print blk_before "<" $0 ">" blk_after; }
{ print }

#!/usr/bin/awk -f
# This script removes parts flagged by a given FILTER_TAG inside qmake project files.
# The file with the relevant parts removed is output to the standard output,
# while files mentionned inside removed parts are output to the standard error.
# This can be used to remove then from the repository by svn rm'ing them.

# If you never used awk, a good introduction is available at http://www.cs.hmc.edu/qref/awk.html

# init
BEGIN {
    infilelist=0;
    blk_before=0;
    blk_after=0;
    blk_filtered=0;
    if (plugins == "")
        plugins = "applications/plugins";
    if (FILTER_TAG == "")
        FILTER_TAG = "SOFA_RELEASE";
}

{ blk_before=blk_after; }

/[{}]/ {
  split($0,a,"#")
  if (a[1]!="") {
    l = a[1];
    beg[1]=""; nbeg = split(l,beg,"{")
    end[1]=""; nend = split(l,end,"}")
    blk_after = blk_before + nbeg - nend
  }
}

END {
    if (blk_after != 0) print "# ERROR: unmatched brackets"
#    print "# Filtered with " FILTER_TAG
}

# Match blocks conditionnally included depending on the SOFA_DEV flag
# Note that we now count the brackets in the source file to find the
# SOFA_DEV closing one instead of relying on the presence of a comment

blk_filtered==0 && $0 ~ "contains[ \t]*\([ \t]*DEFINES[ \t]*,[ \t]*"FILTER_TAG"[ \t]*\)[ \t]*{" {
    blk_filtered=blk_after;
    infilelist=0;
}

blk_filtered>0 && blk_after < blk_filtered {
    blk_filtered=0;
    next;
}

/^[ \t#]*declare/ {
  name=$0;
  gsub(/.*\(/,"",name); gsub(/,.*$/,"",name); gsub(/ /,"",name); gsub(/\t/,"",name);
  path=$0;
  gsub(/.*\([^,]*,/,"",path); gsub(/,.*$/,"",path); gsub(/\).*$/,"",path); gsub(/ /,"",path); gsub(/\t/,"",path);
#  gsub(/\).*$/,"",path); gsub(/,$/,"",path); gsub(/ /,"",path);
  #print "#PROJECT <<<",name,"|",path,">>>";
  projects[name] = path;
}
blk_filtered>0 && /^[ \t#]*enable/ {
  name=$0;
  gsub(/.*\(/,"",name); gsub(/,.*$/,"",name); gsub(/\).*$/,"",name); gsub(/ /,"",name); gsub(/\t/,"",name);
  path = projects[name];
  #print "#DEV PROJECT <<<",name,"|",path,">>>";
  print path > "/dev/stderr";
  next;
}
blk_filtered>0 && /^[ \t#]*usePlugin/ {
  name=$0;
  gsub(/.*\(/,"",name); gsub(/,.*$/,"",name); gsub(/\).*$/,"",name); gsub(/ /,"",name); gsub(/\t/,"",name);
  #print "#DEV PLUGIN <<<",name,">>>";
  print plugins "/" name > "/dev/stderr";
  next;
}

blk_filtered>0 {
    #print "# " $0
    # look for a filename
    # we assume files and directories are listed inside variables containing SUBDIRS, SOURCES or HEADERS
    f0=1;
    nf=NF;
    if ($nf~/\r/) {
      #print "#CR <" $nf ">"
      gsub(/[\r\n]/,"",$nf);
      #print "#CRF <" $nf ">"
      if ($nf=="") nf--;
    }
    if (infilelist==0) {
	if (($1~"SUBDIRS" || $1~"SOURCES" || $1~"HEADERS") && ($2=="=" || $2=="+=")) {
	    #print "#infilelist " $1
	    infilelist=1;
	    f0=3; # first file is in 3rd field
	}
	else if ($1~"###") {
	    #print "#infilelist " $1
	    infilelist=1;
	    f0=2; # first file is in 3rd field
	}
    }
    if (infilelist) {
	for(f=f0;f<=nf && !(f==nf && $f~/^[:space:]*\\[:space:]*$/);f++) {
	    fname=$f
	    gsub(/[\r\n]/,"",fname);
	    gsub(/\\[:space:]*$/,"",fname);
	    gsub(/^##*/,"",fname);
	    if (fname != "") print fname > "/dev/stderr";
	}
	if (f>nf && (nf==0 || $nf!~/\\[:space:]*$/)) { # no "\\" is put at the end of the line -> end of list
	    #print "#end"
	    infilelist=0;
	}
    }
    next;
}

# Match lines mentionning FILTER_TAG, such as the point where it is defined
$0 ~ FILTER_TAG { next; }

# other: simply print the line
#{ print blk_before "<" $0 ">" blk_after; }
{ print }


TestUtils: mark as gawk5-compatible (close #297)

@include "include/Utils"

BEGIN {
    START_TEST("Utils.awk")

    T("GawkVersion", 1)
    {
        initGawk()
        assertTrue(GawkVersion ~ "^(4|5).")
    }

    T("Rlwrap", 1)
    {
        initRlwrap()
        assertEqual(Rlwrap, "rlwrap")
    }

    T("Emacs", 1)
    {
        initEmacs()
        assertEqual(Emacs, "emacs")
    }

    T("newerVersion()", 5)
    {
        assertTrue(newerVersion("0.9", "0.8"))
        assertTrue(newerVersion("0.9.0.1", "0.9.0"))
        assertTrue(newerVersion("1.0", "0.9.9999"))
        assertTrue(newerVersion("1.9.9999", "1.9.10"))
        assertTrue(newerVersion("2", "1.9.9999"))
    }

    if (yn(ENVIRON["NETWORK_ACCESS"])) { # if network access enabled
        T("curl()", 1)
        {
            delete tokens; delete ast
            tokenize(tokens, curl("https://httpbin.org/get"))
            parseJson(ast, tokens)
            assertEqual(unparameterize(ast[0 SUBSEP "url"]),
                        "https://httpbin.org/get")
        }

        T("curlPost()", 1)
        {
            delete tokens; delete ast
            tokenize(tokens, curlPost("https://httpbin.org/post", "fizz=buzz"))
            parseJson(ast, tokens)
            assertEqual(unparameterize(ast[0 SUBSEP "url"]),
                        "https://httpbin.org/post")
        }
    }

    T("dump()", 3)
    {
        delete group
        assertEqual(dump("a", group), 1)

        delete group
        assertEqual(dump("Århus", group), 6)

        delete group
        assertEqual(dump("안녕하세요 세계", group), 22)
    }

    T("base64()", 1)
    {
        assertEqual(base64("ninja"), "bmluamE=")
    }

    T("uprintf", 1)
    {
        assertEqual(uprintf("Ma\\u00f1ana"), "Mañana")
    }

    END_TEST()
}

@include "include/Utils"

BEGIN {
    START_TEST("Utils.awk")

    T("GawkVersion", 1)
    {
        initGawk()
        assertTrue(GawkVersion ~ "^4.")
    }

    T("Rlwrap", 1)
    {
        initRlwrap()
        assertEqual(Rlwrap, "rlwrap")
    }

    T("Emacs", 1)
    {
        initEmacs()
        assertEqual(Emacs, "emacs")
    }

    T("newerVersion()", 5)
    {
        assertTrue(newerVersion("0.9", "0.8"))
        assertTrue(newerVersion("0.9.0.1", "0.9.0"))
        assertTrue(newerVersion("1.0", "0.9.9999"))
        assertTrue(newerVersion("1.9.9999", "1.9.10"))
        assertTrue(newerVersion("2", "1.9.9999"))
    }

    if (yn(ENVIRON["NETWORK_ACCESS"])) { # if network access enabled
        T("curl()", 1)
        {
            delete tokens; delete ast
            tokenize(tokens, curl("https://httpbin.org/get"))
            parseJson(ast, tokens)
            assertEqual(unparameterize(ast[0 SUBSEP "url"]),
                        "https://httpbin.org/get")
        }

        T("curlPost()", 1)
        {
            delete tokens; delete ast
            tokenize(tokens, curlPost("https://httpbin.org/post", "fizz=buzz"))
            parseJson(ast, tokens)
            assertEqual(unparameterize(ast[0 SUBSEP "url"]),
                        "https://httpbin.org/post")
        }
    }

    T("dump()", 3)
    {
        delete group
        assertEqual(dump("a", group), 1)

        delete group
        assertEqual(dump("Århus", group), 6)

        delete group
        assertEqual(dump("안녕하세요 세계", group), 22)
    }

    T("base64()", 1)
    {
        assertEqual(base64("ninja"), "bmluamE=")
    }

    T("uprintf", 1)
    {
        assertEqual(uprintf("Ma\\u00f1ana"), "Mañana")
    }

    END_TEST()
}